github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Core memory management fixes for 7.4-rc2

Browse Source 
This includes fixes to:
 
  - THP (Transparent Huge Pages)
  - GUP (Get User Pages)
  - KSM (Kernel Samepage Merging)
  - USERFAULTFD
  - SHARED MEMORY (shmem/tmpfs/memfd)
  - HUGETLB
  - Core MM, PAGE CACHE and MEMORY RECLAIM
 
 The main changes are:
 
  - Prevent information leaks through memfd pinning, and out-of
    bounds writes in GUP.
 
  - Fix data loss in shmem, THP collapse, KSM and memfd pinning.
 
  - Fix corruption and reference-counting errors with huge PMD/PUD
    mappings, and incorrect TLB flushing and unmapping.
 
  - Fix various use-after-free bugs, deadlocks and unkillable loops,
    witha a focus on shmem swap handling.
 
  - Fix execution-seal enforcement for memfd (incl. hugetlb).
 
 Other changes fix accounting, quota and userfaultfd (un)registration
 handling.
 
 Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQRSU98w6emufXtGDgPH6csYRVW7eQUCauw3uAAKCRDH6csYRVW7
 eWVOAP9Z6aWCsj7SVZ76tC3pBnBl/HDoxTRCBdQLAnBVE9IqowD/e9KHgbg0N8ES
 hs6/IKHmZWN12pyxuhC0gzq5M+4wdAE=
 =HYHR
 -----END PGP SIGNATURE-----

Merge tag 'mm-core-7.4-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core

Pull core memory management fixes from David Hildenbrand:
 "Here's a bigger set of MM fixes that we didn't manage to have ready
  for -rc1. All fixes are stable-worthy; the majority fixes nasty
  long-standing bugs instead of being purely hotfixes.

  We have some more of these kinds of stable-worthy fixes in the
  pipeline, but we'll focus on the really urgent fixes as we get closer
  to the release.

  (Leaving urgent MM fixes lurking for various weeks is not good, but
  having too many changes close to the release is also not good, it's
  tricky ...)

  Most issues were reported by Anthropic. Others were found while
  reworking proposed fixes, and of course some of the fixes were just
  reported and fixed on the mailing list.

  All fixes were in linux-next for several days without changes. Some
  commit IDs changed recently when I had to drop some non-stable fixes,
  rebased to -rc1 and picked up more trailers.

  This includes fixes to:
   - THP (Transparent Huge Pages)
   - GUP (Get User Pages)
   - KSM (Kernel Samepage Merging)
   - USERFAULTFD
   - SHARED MEMORY (shmem/tmpfs/memfd)
   - HUGETLB
   - Core MM, PAGE CACHE and MEMORY RECLAIM

  The main changes are:

   - Prevent information leaks through memfd pinning, and out-of bounds
     writes in GUP.

   - Fix data loss in shmem, THP collapse, KSM and memfd pinning.

   - Fix corruption and reference-counting errors with huge PMD/PUD
     mappings, and incorrect TLB flushing and unmapping.

   - Fix various use-after-free bugs, deadlocks and unkillable loops,
     witha a focus on shmem swap handling.

   - Fix execution-seal enforcement for memfd (incl. hugetlb).

  Other changes fix accounting, quota and userfaultfd (un)registration
  handling"

* tag 'mm-core-7.4-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/mm/core: (61 commits)
  mm/gup: check folio->mapping under folio lock in memfd_pin_folios()
  khugepaged: fix readahead lock inversion in collapse_file()
  mm/huge_memory: set the mmu_gather page size in zap_huge_pud()
  mm/memory-tiers: unregister empty tiers outside memory_tier_lock
  mm/cma: don't release CMA pages still in use
  mm/vma: don't leak the charge for stack growth
  mm/huge_memory: split huge PUDs when adjusting VMA boundaries
  mm/vma: call ->may_split() before a partial merge
  mm/ksm: check large swap-cache folios before mapping them
  mm/ksm: fix use of a freed anon_vma at swap-in
  mm/ksm: reset the unstable trees of all nodes
  mm: fix deadlock of mremap() against UFFDIO_MOVE
  mm: fix unkillable fault loop on MAP_DROPPABLE memory
  mm: fix unmap_mapping_range() beyond 4 GiB on 32-bit
  mm/memory: reject remote COW faults on PFN mappings
  mm/madvise: fix data loss from MADV_FREE on a KSM page
  mm/mremap: fix moving PUD leaves
  mm/mremap: recheck PUDs under the page table lock
  mm/mremap: use pudp_huge_get_and_clear() when moving huge PUDs
  mm/mmu_notifier: set ->mm only after mmgrab()
  ...
This commit is contained in:
Linus Torvalds 2026-11-05 13:13:35 -08:00
commit d68ce4e712
35 changed files with 717 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fs/hugetlbfs/inode.c
View File

@ -28,6 +28,7 @@
#include <linux/folio_batch.h>
#include <linux/fs_parser.h>
#include <linux/mman.h>
#include <linux/memfd.h>
#include <linux/slab.h>
#include <linux/dnotify.h>
#include <linux/statfs.h>
@ -840,6 +841,13 @@ static int hugetlbfs_setattr(const struct mnt_idmap *idmap,
if (error)
return error;
if (S_ISREG(inode->i_mode) && (ia_valid & ATTR_MODE)) {
error = memfd_check_seals_mode(info->seals, inode->i_mode,
attr->ia_mode);
if (error)
return error;
}
trace_hugetlbfs_setattr(inode, dentry, attr);
if (ia_valid & ATTR_SIZE) {

3
include/asm-generic/tlb.h
View File

@ -350,8 +350,7 @@ struct mmu_gather {
void tlb_flush_mmu(struct mmu_gather *tlb);
static inline void __tlb_adjust_range(struct mmu_gather *tlb,
unsigned long address,
unsigned int range_size)
unsigned long address, unsigned long range_size)
{
tlb->start = min(tlb->start, address);
tlb->end = max(tlb->end, address + range_size);

5
include/linux/huge_mm.h
View File

@ -12,9 +12,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
bool huge_pmd_set_accessed(struct vm_fault *vmf);
int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
struct vm_area_struct *vma);
int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
pud_t *dst_pud, pud_t *src_pud, unsigned long addr);
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);

6
include/linux/ksm.h
View File

@ -91,8 +91,8 @@ static inline void ksm_exit(struct mm_struct *mm)
* We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE,
* but what if the vma was unmerged while the page was swapped out?
*/
struct folio *ksm_might_need_to_copy(struct folio *folio,
struct vm_area_struct *vma, unsigned long addr);
struct folio *ksm_might_need_to_copy(struct folio *folio, struct page *page,
struct vm_area_struct *vma, unsigned long addr);
void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc);
void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);
@ -145,7 +145,7 @@ static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
}
static inline struct folio *ksm_might_need_to_copy(struct folio *folio,
struct vm_area_struct *vma, unsigned long addr)
struct page *page, struct vm_area_struct *vma, unsigned long addr)
{
return folio;
}

9
include/linux/memfd.h
View File

@ -2,10 +2,19 @@
#ifndef __LINUX_MEMFD_H
#define __LINUX_MEMFD_H
#include <linux/fcntl.h>
#include <linux/file.h>
#define MEMFD_ANON_NAME "[memfd]"
static inline int memfd_check_seals_mode(unsigned int seals, umode_t old_mode,
umode_t new_mode)
{
if ((seals & F_SEAL_EXEC) && ((old_mode ^ new_mode) & 0111))
return -EPERM;
return 0;
}
#ifdef CONFIG_MEMFD_CREATE
extern long memfd_fcntl(struct file *file, unsigned int cmd, unsigned int arg);
struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx);

3
include/linux/mmu_notifier.h
View File

@ -483,7 +483,8 @@ mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
/*
* This version of mmu_notifier_invalidate_range_start() avoids blocking, but it
* can return an error if a notifier can't proceed without blocking, in which
* case you're not allowed to modify PTEs in the specified range.
* case you're not allowed to modify PTEs in the specified range or call
* mmu_notifier_invalidate_range_end().
*
* This is mainly intended for OOM handling.
*/

15
include/linux/shmem_fs.h
View File

@ -117,8 +117,14 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags);
extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts);
#ifdef CONFIG_SHMEM
bool shmem_file(struct file *file);
bool shmem_mapping(const struct address_space *mapping);
#else
static inline bool shmem_file(struct file *file)
{
return false;
}
static inline bool shmem_mapping(const struct address_space *mapping)
{
return false;
@ -193,15 +199,6 @@ static inline struct page *shmem_read_mapping_page(
mapping_gfp_mask(mapping));
}
static inline bool shmem_file(struct file *file)
{
if (!IS_ENABLED(CONFIG_SHMEM))
return false;
if (!file || !file->f_mapping)
return false;
return shmem_mapping(file->f_mapping);
}
/* Must be called with inode lock taken exclusive. */
static inline void shmem_freeze(struct inode *inode, bool freeze)
{

2
include/linux/userfaultfd_k.h
View File

@ -153,8 +153,6 @@ extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
unsigned long len, bool enable_rwp);
/* move_pages */
void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2);
void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2);
int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval,
struct vm_area_struct *dst_vma,
struct vm_area_struct *src_vma,

20
kernel/fork.c
View File

@ -1533,12 +1533,19 @@ static struct mm_struct *dup_mm(struct task_struct *tsk,
if (!mm)
goto fail_nomem;
uprobe_start_dup_mmap();
if (mmap_write_lock_killable(oldmm)) {
free_mm(mm);
goto fail_uprobe;
}
memcpy(mm, oldmm, sizeof(*mm));
if (!mm_init(mm, tsk))
goto fail_nomem;
if (!mm_init(mm, tsk)) {
mmap_write_unlock(oldmm);
goto fail_uprobe;
}
uprobe_start_dup_mmap();
err = dup_mmap(mm, oldmm);
if (err)
goto free_pt;
@ -1557,9 +1564,10 @@ static struct mm_struct *dup_mm(struct task_struct *tsk,
mm->binfmt = NULL;
mm_init_owner(mm, NULL);
mmput(mm);
if (err)
uprobe_end_dup_mmap();
if (!err)
return NULL;
fail_uprobe:
uprobe_end_dup_mmap();
fail_nomem:
return NULL;
}

10
mm/cma.c
View File

@ -1024,7 +1024,7 @@ bool cma_release(struct cma *cma, const struct page *pages,
unsigned long count)
{
struct cma_memrange *cmr;
unsigned long ret = 0;
unsigned long leaked = 0;
unsigned long i, pfn;
cmr = find_cma_memrange(cma, pages, count);
@ -1033,9 +1033,13 @@ bool cma_release(struct cma *cma, const struct page *pages,
pfn = page_to_pfn(pages);
for (i = 0; i < count; i++, pfn++)
ret += !put_page_testzero(pfn_to_page(pfn));
leaked += !page_ref_freeze(pfn_to_page(pfn), 1);
WARN(ret, "%lu pages are still in use!\n", ret);
if (leaked) {
pr_warn_ratelimited("%lu pages are still in use, not freeing CMA region!\n",
leaked);
return true;
}
__cma_release_frozen(cma, cmr, pages, count);

84
mm/filemap.c
View File

@ -227,18 +227,6 @@ void __filemap_remove_folio(struct folio *folio, void *shadow)
page_cache_delete(mapping, folio, shadow);
}
static void filemap_free_folio(const struct address_space *mapping,
struct folio *folio)
{
void (*free_folio)(struct folio *);
free_folio = mapping->a_ops->free_folio;
if (free_folio)
free_folio(folio);
folio_put_refs(folio, folio_nr_pages(folio));
}
/**
* filemap_remove_folio - Remove folio from page cache.
* @folio: The folio.
@ -250,6 +238,7 @@ static void filemap_free_folio(const struct address_space *mapping,
void filemap_remove_folio(struct folio *folio)
{
struct address_space *mapping = folio->mapping;
void (*free_folio)(struct folio *folio);
BUG_ON(!folio_test_locked(folio));
spin_lock(&mapping->host->i_lock);
@ -258,9 +247,12 @@ void filemap_remove_folio(struct folio *folio)
xa_unlock_irq(&mapping->i_pages);
if (mapping_shrinkable(mapping))
inode_lru_list_add(mapping->host);
free_folio = mapping->a_ops->free_folio;
spin_unlock(&mapping->host->i_lock);
filemap_free_folio(mapping, folio);
if (free_folio)
free_folio(folio);
folio_put_refs(folio, folio_nr_pages(folio));
}
/*
@ -320,6 +312,7 @@ static void page_cache_delete_batch(struct address_space *mapping,
void delete_from_page_cache_batch(struct address_space *mapping,
struct folio_batch *fbatch)
{
void (*free_folio)(struct folio *folio);
int i;
if (!folio_batch_count(fbatch))
@ -337,10 +330,16 @@ void delete_from_page_cache_batch(struct address_space *mapping,
xa_unlock_irq(&mapping->i_pages);
if (mapping_shrinkable(mapping))
inode_lru_list_add(mapping->host);
free_folio = mapping->a_ops->free_folio;
spin_unlock(&mapping->host->i_lock);
for (i = 0; i < folio_batch_count(fbatch); i++)
filemap_free_folio(mapping, fbatch->folios[i]);
for (i = 0; i < folio_batch_count(fbatch); i++) {
struct folio *folio = fbatch->folios[i];
if (free_folio)
free_folio(folio);
folio_put_refs(folio, folio_nr_pages(folio));
}
}
int filemap_check_errors(struct address_space *mapping)
@ -1374,10 +1373,22 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
* return value based on that state without races.
*
* Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive
* waiter, but an exclusive one requires WQ_FLAG_DONE.
* waiter, but an exclusive one must also acquire the bit.
*/
if (behavior == EXCLUSIVE)
return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR;
if (behavior == EXCLUSIVE) {
if (wait->flags & WQ_FLAG_DONE)
return 0;
/*
* We might have just been woken up and the waker will not
* wake up anybody else. If we ignore that now, the remaining
* waiters may not get woken up: take the bit now if it is
* still free.
*/
if ((wait->flags & WQ_FLAG_WOKEN) &&
!test_and_set_bit(bit_nr, folio_flags(folio, 0)))
return 0;
return -EINTR;
}
return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR;
}
@ -3297,11 +3308,11 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,
* @folio - the folio to lock.
* @fpin - the pointer to the file we may pin (or is already pinned).
*
* This works similar to lock_folio_or_retry in that it can drop the
* mmap_lock. It differs in that it actually returns the folio locked
* if it returns 1 and 0 if it couldn't lock the folio. If we did have
* to drop the mmap_lock then fpin will point to the pinned file and
* needs to be fput()'ed at a later point.
* Returns 1 with the folio locked, or 0 if locking failed.
*
* On both entry and return, a non-NULL *fpin means the fault lock has
* already been dropped and the caller must eventually fput() the file.
* Otherwise the fault lock is still held, including on failure.
*/
static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio,
struct file **fpin)
@ -3318,21 +3329,9 @@ static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio,
return 0;
*fpin = maybe_unlock_mmap_for_io(vmf, *fpin);
if (vmf->flags & FAULT_FLAG_KILLABLE) {
if (__folio_lock_killable(folio)) {
/*
* We didn't have the right flags to drop the
* fault lock, but all fault_handlers only check
* for fatal signals if we return VM_FAULT_RETRY,
* so we need to drop the fault lock here and
* return 0 if we don't have a fpin.
*/
if (*fpin == NULL)
release_fault_lock(vmf);
return 0;
}
} else
__folio_lock(folio);
if (vmf->flags & FAULT_FLAG_KILLABLE)
return !__folio_lock_killable(folio);
__folio_lock(folio);
return 1;
}
@ -3561,7 +3560,7 @@ static vm_fault_t filemap_fault_recheck_pte_none(struct vm_fault *vmf)
* vma->vm_mm->mmap_lock must be held on entry.
*
* If our return value has VM_FAULT_RETRY set, it's because the mmap_lock
* may be dropped before doing I/O or by lock_folio_maybe_drop_mmap().
* may be dropped before doing I/O or when aborting an interrupted wait.
*
* If our return value does not have VM_FAULT_RETRY set, the mmap_lock
* has not been released.
@ -3717,9 +3716,8 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
out_retry:
/*
* We dropped the mmap_lock, we need to return to the fault handler to
* re-find the vma and come back and find our hopefully still populated
* page.
* With fpin set, the fault lock was already dropped. Otherwise it is
* still held. Release invalidate_lock before dropping either protection.
*/
if (!IS_ERR(folio))
folio_put(folio);
@ -3727,6 +3725,8 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
filemap_invalidate_unlock_shared(mapping);
if (fpin)
fput(fpin);
else if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
release_fault_lock(vmf);
return ret | VM_FAULT_RETRY;
}
EXPORT_SYMBOL(filemap_fault);

68
mm/gup.c
View File

@ -658,9 +658,6 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma,
assert_spin_locked(pud_lockptr(mm, pudp));
if (!pud_present(pud))
return NULL;
if ((flags & FOLL_WRITE) &&
!can_follow_write_pud(pud, pfn_to_page(pfn), vma, flags))
return NULL;
@ -896,15 +893,17 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
}
static struct page *follow_pmd_mask(struct vm_area_struct *vma,
unsigned long address, pud_t *pudp,
unsigned int flags,
unsigned long *page_mask)
unsigned long address, pud_t *pudp, pud_t pud,
unsigned int flags, unsigned long *page_mask)
{
pmd_t *pmd, pmdval;
spinlock_t *ptl;
struct page *page;
struct mm_struct *mm = vma->vm_mm;
if (unlikely(pud_bad(pud)))
return no_page_table(vma, flags, address);
pmd = pmd_offset(pudp, address);
pmdval = pmdp_get_lockless(pmd);
if (pmd_none(pmdval))
@ -953,18 +952,23 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
pud = pudp_get(pudp);
if (!pud_present(pud))
return no_page_table(vma, flags, address);
if (pud_leaf(pud)) {
ptl = pud_lock(mm, pudp);
page = follow_huge_pud(vma, address, pudp, flags, page_mask);
if (!pud_leaf(pud))
return follow_pmd_mask(vma, address, pudp, pud, flags, page_mask);
ptl = pud_lock(mm, pudp);
pud = *pudp;
if (unlikely(!pud_present(pud))) {
spin_unlock(ptl);
if (page)
return page;
return no_page_table(vma, flags, address);
}
if (unlikely(pud_bad(pud)))
return no_page_table(vma, flags, address);
if (unlikely(!pud_leaf(pud))) {
spin_unlock(ptl);
return follow_pmd_mask(vma, address, pudp, pud, flags, page_mask);
}
return follow_pmd_mask(vma, address, pudp, flags, page_mask);
page = follow_huge_pud(vma, address, pudp, flags, page_mask);
spin_unlock(ptl);
return page;
}
static struct page *follow_p4d_mask(struct vm_area_struct *vma,
@ -3211,6 +3215,9 @@ static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
FOLL_PCI_P2PDMA | FOLL_HONOR_NUMA_FAULT)))
return -EINVAL;
if (unlikely(!nr_pages))
return 0;
if (gup_flags & FOLL_PIN)
mm_set_has_pinned_flag(current->mm);
@ -3218,8 +3225,8 @@ static int gup_fast_fallback(unsigned long start, unsigned long nr_pages,
might_lock_read(&current->mm->mmap_lock);
start = untagged_addr(start) & PAGE_MASK;
len = nr_pages << PAGE_SHIFT;
if (check_add_overflow(start, len, &end))
if (check_shl_overflow(nr_pages, PAGE_SHIFT, &len) ||
check_add_overflow(start, len, &end))
return -EOVERFLOW;
if (end > TASK_SIZE_MAX)
return -EFAULT;
@ -3464,7 +3471,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end,
{
unsigned int flags, nr_folios, nr_found;
unsigned int i, pgshift = PAGE_SHIFT;
pgoff_t start_idx, end_idx;
pgoff_t start_idx, end_idx, retry_idx;
struct folio *folio = NULL;
struct folio_batch fbatch;
struct hstate *h;
@ -3505,6 +3512,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end,
* reason, we try to allocate them and add them to the
* page cache.
*/
retry_idx = start_idx;
nr_found = filemap_get_folios_contig(memfd->f_mapping,
&start_idx,
end_idx,
@ -3517,12 +3525,38 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end,
for (i = 0; i < nr_found; i++) {
folio = fbatch.folios[i];
/*
* The folio might have been removed from the
* page cache (e.g., truncated or collapsed)
* after the lookup. Recheck under the folio
* lock.
*/
folio_lock(folio);
if (unlikely(folio->mapping != memfd->f_mapping)) {
folio_unlock(folio);
start_idx = retry_idx;
break;
}
if (try_grab_folio(folio, 1, FOLL_PIN)) {
folio_unlock(folio);
folio_batch_release(&fbatch);
ret = -EINVAL;
goto err;
}
/*
* fallocate() can leave folios in the
* page cache that were never cleared.
*/
if (!folio_test_uptodate(folio)) {
folio_zero_range(folio, 0, folio_size(folio));
flush_dcache_folio(folio);
folio_mark_uptodate(folio);
}
retry_idx = folio_next_index(folio);
folio_unlock(folio);
if (nr_folios == 0)
*offset = offset_in_folio(folio, start);

146
mm/huge_memory.c
View File

@ -2072,11 +2072,13 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr,
update_mmu_cache_pud(vma, addr, pud);
}
int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
struct vm_area_struct *vma)
int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
pud_t *dst_pud, pud_t *src_pud, unsigned long addr)
{
struct mm_struct *dst_mm = dst_vma->vm_mm;
struct mm_struct *src_mm = src_vma->vm_mm;
spinlock_t *dst_ptl, *src_ptl;
struct page *page;
pud_t pud;
int ret;
@ -2089,11 +2091,20 @@ int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
if (unlikely(!pud_trans_huge(pud)))
goto out_unlock;
/*
* TODO: once we support anonymous pages, use
* folio_try_dup_anon_rmap_*() and split if duplicating fails.
*/
if (vma_is_cow_mapping(vma) && pud_write(pud)) {
page = vm_normal_page_pud(src_vma, addr, pud);
if (page) {
struct folio *folio = page_folio(page);
folio_get(folio);
/*
* TODO: once we support anonymous pages, use
* folio_try_dup_anon_rmap_*() and split if duplicating fails.
*/
folio_add_file_rmap_pud(folio, page, dst_vma);
add_mm_counter(dst_mm, mm_counter_file(folio), HPAGE_PUD_NR);
}
if (vma_is_cow_mapping(src_vma) && pud_write(pud)) {
pudp_set_wrprotect(src_mm, addr, src_pud);
pud = pud_wrprotect(pud);
}
@ -2482,11 +2493,7 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
static void zap_huge_pmd_folio(struct mm_struct *mm, struct vm_area_struct *vma,
pmd_t pmdval, struct folio *folio, bool is_present)
{
const bool is_device_private = folio_is_device_private(folio);
/* Present and device private folios are rmappable. */
if (is_present || is_device_private)
folio_remove_rmap_pmd(folio, &folio->page, vma);
const bool is_device_private = pmd_is_device_private_entry(pmdval);
if (folio_test_anon(folio)) {
add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR);
@ -2501,6 +2508,13 @@ static void zap_huge_pmd_folio(struct mm_struct *mm, struct vm_area_struct *vma,
folio_mark_accessed(folio);
}
/*
* Present and device private folios are rmappable. Remove the
* rmap after transferring young+dirty to the folio.
*/
if (is_present || is_device_private)
folio_remove_rmap_pmd(folio, &folio->page, vma);
/* Device private folios are pinned. */
if (is_device_private)
folio_put(folio);
@ -2632,15 +2646,11 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
return false;
}
/*
* We don't have to worry about the ordering of src and dst
* ptlocks because exclusive mmap_lock prevents deadlock.
*/
old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
if (old_ptl) {
new_ptl = pmd_lockptr(mm, new_pmd);
if (new_ptl != old_ptl)
spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
old_ptl = pmd_lockptr(mm, old_pmd);
new_ptl = pmd_lockptr(mm, new_pmd);
double_pt_lock(old_ptl, new_ptl);
if (likely(pmd_is_huge(*old_pmd))) {
pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
if (pmd_present(pmd))
force_flush = true;
@ -2669,11 +2679,10 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
set_pmd_at(mm, new_addr, new_pmd, pmd);
if (force_flush)
flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
if (new_ptl != old_ptl)
spin_unlock(new_ptl);
spin_unlock(old_ptl);
double_pt_unlock(old_ptl, new_ptl);
return true;
}
double_pt_unlock(old_ptl, new_ptl);
return false;
}
@ -3040,6 +3049,8 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
spinlock_t *ptl;
pud_t orig_pud;
tlb_change_page_size(tlb, HPAGE_PUD_SIZE);
ptl = __pud_trans_huge_lock(pud, vma);
if (!ptl)
return 0;
@ -3495,16 +3506,66 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
__split_huge_pmd(vma, pmd, address, freeze);
}
static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
static bool pud_split_required(const struct vm_area_struct *vma,
unsigned long address)
{
if (!IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD))
return false;
if (IS_ALIGNED(address, HPAGE_PUD_SIZE))
return false;
/*
* If the new address isn't hpage aligned and it could previously
* contain an hugepage: check if we need to split an huge pmd.
* Only selected VMAs support mapping PUDs for now. Lift the
* restriction once this changes.
*/
if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) &&
range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),
ALIGN(address, HPAGE_PMD_SIZE)))
split_huge_pmd_address(vma, address, false);
if (!vma_is_dax(vma) && !vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT))
return false;
return range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PUD_SIZE),
ALIGN(address, HPAGE_PUD_SIZE));
}
static bool pmd_split_required(const struct vm_area_struct *vma,
unsigned long address)
{
if (IS_ALIGNED(address, HPAGE_PMD_SIZE))
return false;
return range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),
ALIGN(address, HPAGE_PMD_SIZE));
}
static void split_huge_leaf(struct vm_area_struct *vma,
unsigned long address)
{
const bool split_pud = pud_split_required(vma, address);
const bool split_pmd = pmd_split_required(vma, address);
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
/*
* mmap, VMA and rmap write locks protect the walk; the split helpers
* take the page table locks.
*/
if (!split_pud && !split_pmd)
return;
pgd = pgd_offset(vma->vm_mm, address);
if (!pgd_present(*pgd))
return;
p4d = p4d_offset(pgd, address);
if (!p4d_present(*p4d))
return;
pud = pud_offset(p4d, address);
if (pud_leaf(*pud))
__split_huge_pud(vma, pud, address);
if (!split_pmd || !pud_present(*pud))
return;
pmd = pmd_offset(pud, address);
__split_huge_pmd(vma, pmd, address, false);
}
void vma_adjust_trans_huge(struct vm_area_struct *vma,
@ -3513,14 +3574,14 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma,
struct vm_area_struct *next)
{
/* Check if we need to split start first. */
split_huge_pmd_if_needed(vma, start);
split_huge_leaf(vma, start);
/* Check if we need to split end next. */
split_huge_pmd_if_needed(vma, end);
split_huge_leaf(vma, end);
/* If we're incrementing next->vm_start, we might need to split it. */
if (next)
split_huge_pmd_if_needed(next, end);
split_huge_leaf(next, end);
}
/*
@ -4522,8 +4583,7 @@ int split_folio_to_list(struct folio *folio, struct list_head *list)
*
* It is unsafe to call folio_unqueue_deferred_split() until folio refcount is
* zero: because even when the list_lru lock is held, a non-empty
* _deferred_list might be in use on deferred_split_scan()'s unlocked
* on-stack list.
* _deferred_list might be in use on deferred_split_scan()'s on-stack list.
*
* The list_lru sublist is determined by folio's memcg: it is therefore
* important to unqueue deferred split before changing folio memcg.
@ -4665,10 +4725,22 @@ static unsigned long deferred_split_scan(struct shrinker *shrink,
deferred_split_isolate, &dispose);
list_for_each_entry_safe(folio, next, &dispose, _deferred_list) {
struct mem_cgroup *memcg;
struct list_lru_one *lru;
bool did_split = false;
bool underused = false;
/*
* Sync with deferred_split_folio(), which can requeue the
* folio once list_empty() is true, and sets PG_partially_mapped.
*/
rcu_read_lock();
memcg = folio_memcg(folio);
lru = list_lru_lock_irq(&deferred_split_lru, folio_nid(folio),
&memcg);
list_del_init(&folio->_deferred_list);
list_lru_unlock_irq(lru);
rcu_read_unlock();
if (!folio_test_partially_mapped(folio)) {
/*

14
mm/hugetlb.c
View File

@ -5157,15 +5157,9 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
spinlock_t *src_ptl, *dst_ptl;
pte_t pte;
dst_ptl = huge_pte_lock(h, mm, dst_pte);
dst_ptl = huge_pte_lockptr(h, mm, dst_pte);
src_ptl = huge_pte_lockptr(h, mm, src_pte);
/*
* We don't have to worry about the ordering of src and dst ptlocks
* because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock.
*/
if (src_ptl != dst_ptl)
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
double_pt_lock(dst_ptl, src_ptl);
pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz);
@ -5194,9 +5188,7 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
}
if (src_ptl != dst_ptl)
spin_unlock(src_ptl);
spin_unlock(dst_ptl);
double_pt_unlock(dst_ptl, src_ptl);
}
unsigned long move_hugetlb_page_tables(struct vm_area_struct *vma,

28
mm/internal.h
View File

@ -626,6 +626,34 @@ static inline void force_page_cache_readahead(struct address_space *mapping,
force_page_cache_ra(&ractl, nr_to_read);
}
/*
* Take two page table locks at the same level in address order such that
* we cannot deadlock.
*/
static inline void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2)
__acquires(ptl1)
__acquires(ptl2)
{
if (ptl1 > ptl2)
swap(ptl1, ptl2);
spin_lock(ptl1);
if (ptl1 != ptl2)
spin_lock_nested(ptl2, SINGLE_DEPTH_NESTING);
else
__acquire(ptl2);
}
static inline void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2)
__releases(ptl1)
__releases(ptl2)
{
spin_unlock(ptl1);
if (ptl1 != ptl2)
spin_unlock(ptl2);
else
__release(ptl2);
}
unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,

65
mm/khugepaged.c
View File

@ -1904,7 +1904,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
result = SCAN_FAIL;
start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl);
if (!start_pte) /* mmap_lock + page lock should prevent this */
if (!start_pte)
goto drop_folio;
/* step 1: check all mapped PTEs are to the right huge page */
@ -1935,6 +1935,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
}
pte_unmap_unlock(start_pte, ptl);
ptl = NULL;
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
haddr, haddr + HPAGE_PMD_SIZE);
mmu_notifier_invalidate_range_start(&range);
@ -1952,7 +1953,7 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
pml = pmd_lock(mm, pmd);
start_pte = pte_offset_map_rw_nolock(mm, pmd, haddr, &pgt_pmd, &ptl);
if (!start_pte) /* mmap_lock + page lock should prevent this */
if (!start_pte)
goto abort;
if (!pml)
spin_lock(ptl);
@ -1994,9 +1995,10 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
/*
* Must clear entry, or a racing truncate may re-remove it.
* TLB flush can be left until pmdp_collapse_flush() does it.
* PTE dirty? Shmem page is already dirty; file is read-only.
*/
clear_ptes(mm, addr, pte, nr_batch_ptes);
ptent = get_and_clear_ptes(mm, addr, pte, nr_batch_ptes);
if (pte_dirty(ptent))
folio_mark_dirty(folio);
folio_remove_rmap_ptes(folio, page, nr_batch_ptes, vma);
nr_mapped_ptes += nr_batch_ptes;
}
@ -2013,12 +2015,11 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
/* step 4: remove empty page table */
if (!pml) {
pml = pmd_lock(mm, pmd);
if (ptl != pml) {
if (ptl != pml)
spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);
if (unlikely(!pmd_same(pgt_pmd, pmdp_get_lockless(pmd)))) {
flush_tlb_mm(mm);
goto unlock;
}
if (unlikely(!pmd_same(pgt_pmd, pmdp_get_lockless(pmd)))) {
flush_tlb_mm(mm);
goto unlock;
}
}
pgt_pmd = pmdp_collapse_flush(vma, haddr, pmd);
@ -2230,6 +2231,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
struct file *file, pgoff_t start, struct collapse_control *cc)
{
struct address_space *mapping = file->f_mapping;
const bool is_shmem = shmem_mapping(mapping);
struct page *dst;
struct folio *folio, *tmp, *new_folio;
unsigned long new_pfn = -1;
@ -2238,7 +2240,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
enum scan_result result = SCAN_SUCCEED;
int nr_none = 0;
bool is_shmem = shmem_file(file);
bool do_readahead = false;
bool readahead_done = false;
/*
* MADV_COLLAPSE ignores shmem huge config, so do not check shmem
@ -2256,6 +2259,10 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
mapping_set_update(&xas, mapping);
retry:
result = SCAN_SUCCEED;
do_readahead = false;
xas_set_order(&xas, start, HPAGE_PMD_ORDER);
__folio_set_locked(new_folio);
if (is_shmem)
__folio_set_swapbacked(new_folio);
@ -2320,17 +2327,9 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
}
} else { /* !is_shmem */
if (!folio || xa_is_value(folio)) {
xas_unlock_irq(&xas);
page_cache_sync_readahead(mapping, &file->f_ra,
file, index,
end - index);
/* drain lru cache to help folio_isolate_lru() */
lru_add_drain();
folio = filemap_lock_folio(mapping, index);
if (IS_ERR(folio)) {
result = SCAN_FAIL;
goto xa_unlocked;
}
do_readahead = !readahead_done;
result = SCAN_FAIL;
goto xa_locked;
} else if (folio_test_dirty(folio)) {
/*
* This page is dirty because it hasn't
@ -2657,6 +2656,30 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
new_folio->mapping = NULL;
folio_unlock(new_folio);
/* Readahead may take invalidate_lock; source folios are unlocked. */
if (do_readahead) {
for (index = start; index < end;) {
folio = filemap_lock_folio(mapping, index);
if (IS_ERR(folio)) {
page_cache_sync_readahead(mapping, &file->f_ra,
file, index, end - index);
folio = filemap_lock_folio(mapping, index);
if (IS_ERR(folio)) {
result = SCAN_FAIL;
goto out_put;
}
}
index = folio_next_index(folio);
folio_unlock(folio);
folio_put(folio);
}
/* Drain the local LRU batch before retrying isolation. */
lru_add_drain();
readahead_done = true;
goto retry;
}
out_put:
folio_put(new_folio);
out:
VM_BUG_ON(!list_empty(&pagelist));

56
mm/ksm.c
View File

@ -476,6 +476,7 @@ static void advisor_stop_scan(void)
#ifdef CONFIG_NUMA
/* Zeroed when merging across nodes is not allowed */
static unsigned int ksm_merge_across_nodes = 1;
/* Number of allocated stable and unstable tree roots. */
static int ksm_nr_node_ids = 1;
#else
#define ksm_merge_across_nodes 1U
@ -3146,24 +3147,52 @@ void __ksm_exit(struct mm_struct *mm)
trace_ksm_exit(mm);
}
struct folio *ksm_might_need_to_copy(struct folio *folio,
struct vm_area_struct *vma, unsigned long addr)
static bool folio_anon_rmap_compatible(const struct folio *folio,
const struct page *page, struct vm_area_struct *vma,
unsigned long addr)
{
const struct anon_vma *anon_vma = folio_anon_vma(folio);
struct anon_vma_chain *avc;
VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
VM_WARN_ON_ONCE_FOLIO(page_folio(page) != folio, folio);
VM_WARN_ON_ONCE_FOLIO(folio_test_ksm(folio), folio);
vma_assert_stabilised(vma);
if (!anon_vma)
return true;
if (page_pgoff(folio, page) != linear_anon_page_index(vma, addr))
return false;
/*
* Careful: anon_vma might have been freed, so we cannot safely
* dereference it. Add a fast path before we walk the anon_vma chain.
*/
if (anon_vma == vma->anon_vma || anon_vma == vma->anon_vma->root)
return true;
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) {
if (avc->anon_vma == anon_vma)
return true;
}
return false;
}
struct folio *ksm_might_need_to_copy(struct folio *folio, struct page *page,
struct vm_area_struct *vma, unsigned long addr)
{
struct page *page = folio_page(folio, 0);
struct anon_vma *anon_vma = folio_anon_vma(folio);
struct folio *new_folio;
if (folio_test_large(folio))
return folio;
/*
* A KSM folio is small, but the slot that it left behind can be swapped
* in as a part of a large folio by one of the processes that shared it.
* So test, and copy, the page of the slot.
*/
if (folio_test_ksm(folio)) {
if (folio_stable_node(folio) &&
!(ksm_run & KSM_RUN_UNMERGE))
return folio; /* no need to copy it */
} else if (!anon_vma) {
return folio; /* no need to copy it */
} else if (folio->index == linear_anon_page_index(vma, addr) &&
anon_vma->root == vma->anon_vma->root) {
} else if (folio_anon_rmap_compatible(folio, page, vma, addr)) {
return folio; /* still no need to copy it */
}
if (PageHWPoison(page))
@ -3646,14 +3675,13 @@ static ssize_t merge_across_nodes_store(struct kobject *kobj,
else {
root_stable_tree = buf;
root_unstable_tree = buf + nr_node_ids;
ksm_nr_node_ids = nr_node_ids;
/* Stable tree is empty but not the unstable */
root_unstable_tree[0] = one_unstable_tree[0];
}
}
if (!err) {
if (!err)
ksm_merge_across_nodes = knob;
ksm_nr_node_ids = knob ? 1 : nr_node_ids;
}
}
mutex_unlock(&ksm_thread_mutex);

9
mm/madvise.c
View File

@ -756,6 +756,15 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
if (folio_test_swapcache(folio) || folio_test_dirty(folio)) {
if (!folio_trylock(folio))
continue;
/*
* KSM folios can gain new mappings at any time, so we must
* not clear their dirty flag.
*/
if (folio_test_ksm(folio)) {
folio_unlock(folio);
continue;
}
/*
* If we have a large folio at this point, we know it is
* fully mapped so if its mapcount is the same as its

2
mm/memfd.c
View File

@ -482,6 +482,8 @@ struct file *memfd_alloc_file(const char *name, unsigned int flags)
}
file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
/* a regular file like any other: POSIX.1-2008/SUSv4 Section XSI 2.9.7 */
file->f_mode |= FMODE_ATOMIC_POS;
file->f_flags |= O_LARGEFILE;
if (flags & MFD_NOEXEC_SEAL) {

29
mm/memory-tiers.c
View File

@ -572,21 +572,14 @@ static struct memory_tier *set_node_memory_tier(int node)
return memtier;
}
static void destroy_memory_tier(struct memory_tier *memtier)
static struct memory_tier *clear_node_memory_tier(int node)
{
list_del(&memtier->list);
device_unregister(&memtier->dev);
}
static bool clear_node_memory_tier(int node)
{
bool cleared = false;
pg_data_t *pgdat;
struct memory_tier *memtier;
pgdat = NODE_DATA(node);
if (!pgdat)
return false;
return NULL;
/*
* Make sure that anybody looking at NODE_DATA who finds
@ -604,14 +597,10 @@ static bool clear_node_memory_tier(int node)
synchronize_rcu();
memtype = node_memory_types[node].memtype;
node_clear(node, memtype->nodes);
if (nodes_empty(memtype->nodes)) {
if (nodes_empty(memtype->nodes))
list_del_init(&memtype->tier_sibling);
if (list_empty(&memtier->memory_types))
destroy_memory_tier(memtier);
}
cleared = true;
}
return cleared;
return memtier;
}
static void release_memtype(struct kref *kref)
@ -884,13 +873,21 @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
{
struct memory_tier *memtier;
struct node_notify *nn = _arg;
bool empty = false;
switch (action) {
case NODE_REMOVED_LAST_MEMORY:
mutex_lock(&memory_tier_lock);
if (clear_node_memory_tier(nn->nid))
memtier = clear_node_memory_tier(nn->nid);
if (memtier) {
empty = list_empty(&memtier->memory_types);
if (empty)
list_del(&memtier->list);
establish_demotion_targets();
}
mutex_unlock(&memory_tier_lock);
if (empty)
device_unregister(&memtier->dev);
break;
case NODE_ADDED_FIRST_MEMORY:
mutex_lock(&memory_tier_lock);

30
mm/memory.c
View File

@ -1463,7 +1463,6 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
unsigned long end)
{
struct mm_struct *dst_mm = dst_vma->vm_mm;
struct mm_struct *src_mm = src_vma->vm_mm;
pud_t *src_pud, *dst_pud;
unsigned long next;
@ -1477,8 +1476,7 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
int err;
VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
err = copy_huge_pud(dst_mm, src_mm,
dst_pud, src_pud, addr, src_vma);
err = copy_huge_pud(dst_vma, src_vma, dst_pud, src_pud, addr);
if (err == -ENOMEM)
return -ENOMEM;
if (!err)
@ -3768,6 +3766,13 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src,
return 0;
}
/*
* __copy_from_user_inatomic() only works as expected for local faults.
* Reject remote faults.
*/
if (unlikely(mm != current->mm))
return -EFAULT;
/*
* If the source page was a PFN mapping, we don't have
* a "struct page" for it. We do a best-effort copy by
@ -4093,6 +4098,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
folio_put(old_folio);
delayacct_wpcopy_end();
if (err == -EFAULT)
return VM_FAULT_SIGBUS;
return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
}
kmsan_copy_page_meta(&new_folio->page, vmf->page);
@ -4650,8 +4657,8 @@ EXPORT_SYMBOL_GPL(unmap_mapping_pages);
void unmap_mapping_range(struct address_space *mapping,
loff_t const holebegin, loff_t const holelen, int even_cows)
{
pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
pgoff_t hba = (u64)holebegin >> PAGE_SHIFT;
pgoff_t hlen = ((u64)holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
/* Check for overflow. */
if (sizeof(holelen) > sizeof(hlen)) {
@ -5125,7 +5132,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
* folio->index of non-ksm folios would be nonlinear inside the
* anon VMA -- the ksm flag is lost on actual swapout.
*/
folio = ksm_might_need_to_copy(folio, vma, vmf->address);
folio = ksm_might_need_to_copy(folio, page, vma, vmf->address);
if (unlikely(!folio)) {
ret = VM_FAULT_OOM;
folio = swapcache;
@ -5306,7 +5313,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
if (unlikely(folio != swapcache)) {
folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
folio_add_lru_vma(folio, vma);
folio_put_swap(swapcache, NULL);
folio_put_swap(swapcache,
folio_file_page(swapcache, swp_offset(entry)));
} else if (!folio_test_anon(folio)) {
/*
* We currently only expect !anon folios that are fully
@ -6983,8 +6991,12 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
lru_gen_exit_fault();
/* If the mapping is droppable, then errors due to OOM aren't fatal. */
if (is_droppable)
/*
* If the mapping is droppable, then errors due to OOM aren't fatal: the
* task returns to user mode, handles its signals and tries again. A
* fault from kernel mode has no such return, so it must see the error.
*/
if (is_droppable && (flags & FAULT_FLAG_USER))
ret &= ~VM_FAULT_OOM;
if (flags & FAULT_FLAG_USER) {

4
mm/mempolicy.c
View File

@ -3372,11 +3372,11 @@ void __init numa_policy_init(void)
policy_cache = kmem_cache_create("numa_policy",
sizeof(struct mempolicy),
0, SLAB_PANIC, NULL);
0, SLAB_PANIC | SLAB_ACCOUNT, NULL);
sn_cache = kmem_cache_create("shared_policy_node",
sizeof(struct sp_node),
0, SLAB_PANIC, NULL);
0, SLAB_PANIC | SLAB_ACCOUNT, NULL);
for_each_node(nid) {
preferred_node_policy[nid] = (struct mempolicy) {

8
mm/mmap.c
View File

@ -1713,8 +1713,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
LIST_HEAD(uf);
VMA_ITERATOR(vmi, mm, 0);
if (mmap_write_lock_killable(oldmm))
return -EINTR;
mmap_assert_write_locked(oldmm);
flush_cache_dup_mm(oldmm);
uprobe_dup_mmap(oldmm, mm);
/*
@ -1725,11 +1724,6 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
/* No ordering required: file already has been exposed. */
dup_mm_exe_file(mm, oldmm);
mm->total_vm = oldmm->total_vm;
mm->data_vm = oldmm->data_vm;
mm->exec_vm = oldmm->exec_vm;
mm->stack_vm = oldmm->stack_vm;
/* Use __mt_dup() to efficiently build an identical maple tree. */
retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL);
if (unlikely(retval))

24
mm/mmu_notifier.c
View File

@ -569,16 +569,25 @@ int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
{
struct mmu_notifier_subscriptions *subscriptions =
range->mm->notifier_subscriptions;
int ret;
const bool has_itree = subscriptions->has_itree;
int ret = 0;
if (subscriptions->has_itree) {
if (has_itree) {
ret = mn_itree_invalidate(subscriptions, range);
if (ret)
return ret;
}
if (!hlist_empty(&subscriptions->list))
return mn_hlist_invalidate_range_start(subscriptions, range);
return 0;
ret = mn_hlist_invalidate_range_start(subscriptions, range);
/*
* On error, the non-blocking caller is not allowed to call
* invalidate_range_end(), as in mn_itree_invalidate().
*/
if (ret && has_itree && !mmu_notifier_range_blockable(range))
mn_itree_inv_end(subscriptions);
return ret;
}
static void
@ -938,7 +947,6 @@ static int __mmu_interval_notifier_insert(
struct mmu_notifier_subscriptions *subscriptions, unsigned long start,
unsigned long length, const struct mmu_interval_notifier_ops *ops)
{
interval_sub->mm = mm;
interval_sub->ops = ops;
RB_CLEAR_NODE(&interval_sub->interval_tree.rb);
interval_sub->interval_tree.start = start;
@ -955,8 +963,12 @@ static int __mmu_interval_notifier_insert(
if (WARN_ON(atomic_read(&mm->mm_users) <= 0))
return -EINVAL;
/* pairs with mmdrop in mmu_interval_notifier_remove() */
/*
* Pairs with mmdrop in mmu_interval_notifier_remove(). Drivers take a
* non-NULL ->mm for "registered", so set it only with the reference.
*/
mmgrab(mm);
interval_sub->mm = mm;
/*
* If some invalidate_range_start/end region is going on in parallel

103
mm/mremap.c
View File

@ -87,7 +87,7 @@ static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
return NULL;
pud = pud_offset(p4d, addr);
if (pud_none_or_clear_bad(pud))
if (!pud_trans_huge(*pud) && pud_none_or_clear_bad(pud))
return NULL;
return pud;
@ -202,7 +202,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
struct mm_struct *mm = vma->vm_mm;
pte_t *old_ptep, *new_ptep;
pte_t old_pte, pte;
pmd_t dummy_pmdval;
pmd_t old_pmdval, new_pmdval;
spinlock_t *old_ptl, *new_ptl;
bool force_flush = false;
unsigned long old_addr = pmc->old_addr;
@ -234,31 +234,30 @@ static int move_ptes(struct pagetable_move_control *pmc,
if (pmc->need_rmap_locks)
take_rmap_locks(vma);
/*
* We don't have to worry about the ordering of src and dst
* pte locks because exclusive mmap_lock prevents deadlock.
*/
old_ptep = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
old_ptep = pte_offset_map_rw_nolock(mm, old_pmd, old_addr, &old_pmdval,
&old_ptl);
if (!old_ptep) {
err = -EAGAIN;
goto out;
}
/*
* Now new_pte is none, so collapse_scan_file() path can not find
* this by traversing file->f_mapping, so there is no concurrency with
* retract_page_tables(). In addition, we already hold the exclusive
* mmap_lock, so this new_pte page is stable, so there is no need to get
* pmdval and do pmd_same() check.
*/
new_ptep = pte_offset_map_rw_nolock(mm, new_pmd, new_addr, &dummy_pmdval,
&new_ptl);
new_ptep = pte_offset_map_rw_nolock(mm, new_pmd, new_addr, &new_pmdval,
&new_ptl);
if (!new_ptep) {
pte_unmap_unlock(old_ptep, old_ptl);
pte_unmap(old_ptep);
err = -EAGAIN;
goto out;
}
if (new_ptl != old_ptl)
spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
double_pt_lock(old_ptl, new_ptl);
if (unlikely(!pmd_same(old_pmdval, pmdp_get_lockless(old_pmd)) ||
!pmd_same(new_pmdval, pmdp_get_lockless(new_pmd)))) {
double_pt_unlock(old_ptl, new_ptl);
pte_unmap(new_ptep);
pte_unmap(old_ptep);
err = -EAGAIN;
goto out;
}
flush_tlb_batched_pending(vma->vm_mm);
lazy_mmu_mode_enable();
@ -317,10 +316,9 @@ static int move_ptes(struct pagetable_move_control *pmc,
lazy_mmu_mode_disable();
if (force_flush)
flush_tlb_range(vma, old_end - len, old_end);
if (new_ptl != old_ptl)
spin_unlock(new_ptl);
double_pt_unlock(old_ptl, new_ptl);
pte_unmap(new_ptep - 1);
pte_unmap_unlock(old_ptep - 1, old_ptl);
pte_unmap(old_ptep - 1);
out:
if (pmc->need_rmap_locks)
drop_rmap_locks(vma);
@ -395,14 +393,9 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc,
if (WARN_ON_ONCE(!pmd_none(*new_pmd)))
return false;
/*
* We don't have to worry about the ordering of src and dst
* ptlocks because exclusive mmap_lock prevents deadlock.
*/
old_ptl = pmd_lock(mm, old_pmd);
old_ptl = pmd_lockptr(mm, old_pmd);
new_ptl = pmd_lockptr(mm, new_pmd);
if (new_ptl != old_ptl)
spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
double_pt_lock(old_ptl, new_ptl);
pmd = *old_pmd;
@ -418,9 +411,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc,
pmd_populate(mm, new_pmd, pmd_pgtable(pmd));
flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE);
out_unlock:
if (new_ptl != old_ptl)
spin_unlock(new_ptl);
spin_unlock(old_ptl);
double_pt_unlock(old_ptl, new_ptl);
return res;
}
@ -452,26 +443,24 @@ static bool move_normal_pud(struct pagetable_move_control *pmc,
if (WARN_ON_ONCE(!pud_none(*new_pud)))
return false;
/*
* We don't have to worry about the ordering of src and dst
* ptlocks because exclusive mmap_lock prevents deadlock.
*/
old_ptl = pud_lock(mm, old_pud);
old_ptl = pud_lockptr(mm, old_pud);
new_ptl = pud_lockptr(mm, new_pud);
if (new_ptl != old_ptl)
spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
double_pt_lock(old_ptl, new_ptl);
pud = *old_pud;
if (unlikely(pud_none(pud)))
goto out_unlock;
/* Clear the pud */
pud = *old_pud;
pud_clear(old_pud);
VM_BUG_ON(!pud_none(*new_pud));
pud_populate(mm, new_pud, pud_pgtable(pud));
flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE);
if (new_ptl != old_ptl)
spin_unlock(new_ptl);
spin_unlock(old_ptl);
out_unlock:
double_pt_unlock(old_ptl, new_ptl);
return true;
}
@ -499,18 +488,17 @@ static bool move_huge_pud(struct pagetable_move_control *pmc,
if (WARN_ON_ONCE(!pud_none(*new_pud)))
return false;
/*
* We don't have to worry about the ordering of src and dst
* ptlocks because exclusive mmap_lock prevents deadlock.
*/
old_ptl = pud_lock(mm, old_pud);
old_ptl = pud_lockptr(mm, old_pud);
new_ptl = pud_lockptr(mm, new_pud);
if (new_ptl != old_ptl)
spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
double_pt_lock(old_ptl, new_ptl);
if (unlikely(!pud_trans_huge(*old_pud))) {
double_pt_unlock(old_ptl, new_ptl);
return false;
}
/* Clear the pud */
pud = *old_pud;
pud_clear(old_pud);
pud = pudp_huge_get_and_clear(mm, pmc->old_addr, old_pud);
VM_BUG_ON(!pud_none(*new_pud));
@ -518,9 +506,7 @@ static bool move_huge_pud(struct pagetable_move_control *pmc,
/* mark soft_ditry when we add pud level soft dirty support */
set_pud_at(mm, pmc->new_addr, new_pud, pud);
flush_pud_tlb_range(vma, pmc->old_addr, pmc->old_addr + HPAGE_PUD_SIZE);
if (new_ptl != old_ptl)
spin_unlock(new_ptl);
spin_unlock(old_ptl);
double_pt_unlock(old_ptl, new_ptl);
return true;
}
@ -842,11 +828,10 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc)
if (!new_pud)
break;
if (pud_trans_huge(*old_pud)) {
if (extent == HPAGE_PUD_SIZE) {
move_pgt_entry(pmc, HPAGE_PUD, old_pud, new_pud);
/* We ignore and continue on error? */
if (extent == HPAGE_PUD_SIZE &&
move_pgt_entry(pmc, HPAGE_PUD, old_pud, new_pud))
continue;
}
split_huge_pud(pmc->old, old_pud, pmc->old_addr);
} else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) {
if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud))
continue;

2
mm/nommu.c
View File

@ -1887,7 +1887,7 @@ subsys_initcall(init_admin_reserve);
int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
{
mmap_write_lock(oldmm);
mmap_assert_write_locked(oldmm);
dup_mm_exe_file(mm, oldmm);
mmap_write_unlock(oldmm);
return 0;

23
mm/rmap.c
View File

@ -924,6 +924,7 @@ static bool folio_referenced_one(struct folio *folio,
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
int ptes = 0, referenced = 0;
unsigned int nr;
pte_t pteval;
while (page_vma_mapped_walk(&pvmw)) {
address = pvmw.address;
@ -968,10 +969,12 @@ static bool folio_referenced_one(struct folio *folio,
return false;
}
if (pvmw.pte && folio_test_large(folio)) {
if (pvmw.pte)
pteval = ptep_get(pvmw.pte);
if (pvmw.pte && pte_present(pteval) && folio_test_large(folio)) {
const unsigned long end_addr = pmd_addr_end(address, vma->vm_end);
const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
pte_t pteval = ptep_get(pvmw.pte);
nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr);
}
@ -981,11 +984,23 @@ static bool folio_referenced_one(struct folio *folio,
* are. —they could be on active/inactive lists or on MGLRU. So the
* simplest approach is to disable this look-around optimization.
*/
if (lru_gen_enabled() && !lru_gen_switching() && pvmw.pte) {
if (lru_gen_enabled() && !lru_gen_switching() && pvmw.pte &&
pte_present(pteval)) {
if (lru_gen_look_around(&pvmw, nr))
referenced++;
} else if (pvmw.pte) {
if (clear_flush_young_ptes_notify(vma, address, pvmw.pte, nr))
bool young = false;
/*
* PFN swap PTEs, such as device-exclusive ones, that
* actually map pages are "old" from a CPU perspective.
* The MMU notifier takes care of any device aspects.
*/
if (pte_present(pteval))
young = clear_flush_young_ptes(vma, address, pvmw.pte, nr);
young |= mmu_notifier_clear_flush_young(vma->vm_mm, address,
address + nr * PAGE_SIZE);
if (young)
referenced++;
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,

173
mm/shmem.c
View File

@ -77,6 +77,7 @@ static struct vfsmount *shm_mnt __ro_after_init;
#include <linux/magic.h>
#include <linux/syscalls.h>
#include <linux/fcntl.h>
#include <linux/memfd.h>
#include <uapi/linux/memfd.h>
#include <linux/rmap.h>
#include <linux/uuid.h>
@ -275,6 +276,11 @@ static const struct vm_operations_struct shmem_vm_ops;
static const struct vm_operations_struct shmem_anon_vm_ops;
static struct file_system_type shmem_fs_type;
bool shmem_file(struct file *file)
{
return file && file_inode(file)->i_fop == &shmem_file_operations;
}
bool shmem_mapping(const struct address_space *mapping)
{
return mapping->a_ops == &shmem_aops;
@ -1176,12 +1182,11 @@ static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
}
/*
* Remove swap entry from page cache, free the swap and its page cache. Returns
* the number of pages being freed. 0 means entry not found in XArray (0 pages
* being freed).
* Remove a swap entry or swapin-error marker wholly within [index, end].
* Return whether it was removed, counting freed swap pages in nr_swaps_freed.
*/
static long shmem_free_swap(struct address_space *mapping,
pgoff_t index, pgoff_t end, void *radswap)
static bool shmem_free_swap(struct address_space *mapping,
pgoff_t index, pgoff_t end, void *radswap, long *nr_swaps_freed)
{
XA_STATE(xas, &mapping->i_pages, index);
const softleaf_t swp = radix_to_swp_entry(radswap);
@ -1202,8 +1207,10 @@ static long shmem_free_swap(struct address_space *mapping,
xas_unlock_irq(&xas);
/* A swapin-error marker holds no swap slot, so just drop it. */
if (nr_pages && softleaf_is_swap(swp))
if (nr_pages && softleaf_is_swap(swp)) {
swap_put_entries_direct(swp, nr_pages);
*nr_swaps_freed += nr_pages;
}
return nr_pages;
}
@ -1297,12 +1304,10 @@ void shmem_unlock_mapping(struct address_space *mapping)
static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
{
const gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
struct folio *folio;
/*
* At first avoid shmem_get_folio(,,,SGP_READ): that fails
* beyond i_size, and reports fallocated folios as holes.
*/
retry:
folio = filemap_get_entry(inode->i_mapping, index);
if (!folio)
return folio;
@ -1313,13 +1318,14 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
/* The folio has been swapped out */
folio_unlock(folio);
folio_put(folio);
goto retry;
}
/*
* But read a folio back from swap if any of it is within i_size
* (although in some cases this is just a waste of time).
* Truncation can look up a large swap entry at or beyond i_size.
* Swap it in directly to bypass the size check in shmem_get_folio().
*/
folio = NULL;
shmem_get_folio(inode, index, 0, &folio, SGP_READ);
if (shmem_swapin_folio(inode, index, &folio, SGP_READ, gfp, NULL, NULL) == -EEXIST)
goto retry;
return folio;
}
@ -1358,8 +1364,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
if (xa_is_value(folio)) {
if (unfalloc)
continue;
nr_swaps_freed += shmem_free_swap(mapping, indices[i],
end - 1, folio);
shmem_free_swap(mapping, indices[i], end - 1,
folio, &nr_swaps_freed);
continue;
}
@ -1422,32 +1428,31 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
if (xa_is_value(folio)) {
int order;
long swaps_freed;
if (unfalloc)
continue;
swaps_freed = shmem_free_swap(mapping, indices[i],
end - 1, folio);
if (!swaps_freed) {
if (!shmem_free_swap(mapping, indices[i], end - 1,
folio, &nr_swaps_freed)) {
pgoff_t base = indices[i];
order = shmem_confirm_swap(mapping, indices[i],
radix_to_swp_entry(folio));
/*
* If found a large swap entry cross the end or start
* border, skip it as the truncate_inode_partial_folio
* above should have at least zerod its content once.
* Skip boundary entries when restarting
* the scan to avoid endless loops after
* swapin failure.
*/
if (order > 0) {
base = round_down(base, 1 << order);
if (base < start || base + (1 << order) > end)
if (base < start || base + (1 << order) > end) {
start = max(start, base + (1 << order));
continue;
}
}
/* Swap was replaced by page or extended, retry */
index = base;
break;
}
nr_swaps_freed += swaps_freed;
continue;
}
@ -1463,23 +1468,35 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
VM_BUG_ON_FOLIO(folio_test_writeback(folio),
folio);
if (!folio_test_large(folio)) {
/*
* A shared mapping can have dirty PTEs while
* the folio is still clean. Mark the folio
* dirty so a failed split cannot discard data
* outside the range.
*/
if (!unfalloc && folio_test_large(folio))
folio_mark_dirty(folio);
if (unfalloc || !folio_test_large(folio)) {
truncate_inode_folio(mapping, folio);
} else if (truncate_inode_partial_folio(folio,
lstart, lend, NULL, NULL)) {
/*
* If we split a page, reset the loop so
* that we pick up the new sub pages.
* Otherwise the THP was entirely
* dropped or the target range was
* zeroed, so just continue the loop as
* is.
* The folio was split. Retry the scan
* to find the split folios.
*/
if (!folio_test_large(folio)) {
folio_unlock(folio);
index = start;
break;
}
folio_unlock(folio);
index = start;
break;
} else if (folio->mapping) {
/*
* Splitting failed, but the part in the
* range is already zeroed. The folio
* was not dropped, so skip it when
* restarting the scan to avoid retrying
* it forever.
*/
start = max(start, folio_next_index(folio));
}
}
folio_unlock(folio);
@ -1546,10 +1563,11 @@ static int shmem_setattr(const struct mnt_idmap *idmap,
if (error)
return error;
if ((info->seals & F_SEAL_EXEC) && (attr->ia_valid & ATTR_MODE)) {
if ((inode->i_mode ^ attr->ia_mode) & 0111) {
return -EPERM;
}
if (attr->ia_valid & ATTR_MODE) {
error = memfd_check_seals_mode(info->seals, inode->i_mode,
attr->ia_mode);
if (error)
return error;
}
if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
@ -1578,7 +1596,8 @@ static int shmem_setattr(const struct mnt_idmap *idmap,
if (oldsize > holebegin)
unmap_mapping_range(inode->i_mapping,
holebegin, 0, 1);
if (info->alloced)
/* Swapin-error markers are not counted in alloced. */
if (info->alloced || !mapping_empty(inode->i_mapping))
shmem_truncate_range(inode,
newsize, (loff_t)-1);
/* unmap again to remove racily COWed private pages */
@ -1614,6 +1633,17 @@ static int shmem_setattr(const struct mnt_idmap *idmap,
return error;
}
static void shmem_disallow_eviction(struct shmem_inode_info *info)
{
atomic_inc(&info->stop_eviction);
}
static void shmem_allow_eviction(struct shmem_inode_info *info)
{
if (atomic_dec_and_test(&info->stop_eviction))
wake_up_var(&info->stop_eviction);
}
static void shmem_evict_inode(struct inode *inode)
{
struct shmem_inode_info *info = SHMEM_I(inode);
@ -1626,16 +1656,21 @@ static void shmem_evict_inode(struct inode *inode)
mapping_set_exiting(inode->i_mapping);
shmem_truncate_range(inode, 0, (loff_t)-1);
shmem_unused_huge_del(inode);
while (!list_empty(&info->swaplist)) {
/* Wait while shmem_unuse() is scanning this inode... */
/*
* Writeout may still use the inode after swapoff has removed it
* from the swaplist, so always wait at least once.
*/
do {
wait_var_event(&info->stop_eviction,
!atomic_read(&info->stop_eviction));
if (list_empty(&info->swaplist))
break;
spin_lock(&shmem_swaplist_lock);
/* ...but beware of the race if we peeked too early */
if (!atomic_read(&info->stop_eviction))
list_del_init(&info->swaplist);
spin_unlock(&shmem_swaplist_lock);
}
} while (!list_empty(&info->swaplist));
}
simple_xattrs_free(&sbinfo->xa_cache, &info->xattrs, sbinfo->max_inodes ? &freed : NULL);
@ -1778,15 +1813,14 @@ int shmem_unuse(unsigned int type)
* remove placeholder inode from swaplist, nor let it be freed
* (igrab() would protect from unlink, but not from unmount).
*/
atomic_inc(&info->stop_eviction);
shmem_disallow_eviction(info);
spin_unlock(&shmem_swaplist_lock);
error = shmem_unuse_inode(&info->vfs_inode, type);
cond_resched();
spin_lock(&shmem_swaplist_lock);
if (atomic_dec_and_test(&info->stop_eviction))
wake_up_var(&info->stop_eviction);
shmem_allow_eviction(info);
if (error)
break;
if (list_empty(&info->swaplist))
@ -1917,6 +1951,13 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
spin_unlock(&shmem_swaplist_lock);
}
/*
* Once the folio has left the page cache, its lock no longer
* keeps the inode from being evicted and freed: hold eviction
* off, in case the folio has to be put back below.
*/
shmem_disallow_eviction(info);
folio_dup_swap(folio, NULL);
shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));
@ -1924,6 +1965,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
error = swap_writeout(ctx, folio);
if (error != AOP_WRITEPAGE_ACTIVATE) {
/* folio has been unlocked */
shmem_allow_eviction(info);
return error;
}
@ -1940,6 +1982,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
shmem_recalc_inode(inode, 0, -nr_pages);
folio_put_swap(folio, NULL);
}
shmem_allow_eviction(info);
/*
* The swap_cache_del_folio() below could be left for
@ -2374,11 +2417,10 @@ static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
folio_put_swap(folio, NULL);
swap_cache_del_folio(folio);
/*
* Don't treat swapin error folio as alloced. Otherwise inode->i_blocks
* won't be 0 when inode is released and thus trigger WARN_ON(i_blocks)
* in shmem_evict_inode().
* Let shmem_recalc_inode() reduce alloced and release the block charge
* for the freed swap pages.
*/
shmem_recalc_inode(inode, -nr_pages, -nr_pages);
shmem_recalc_inode(inode, 0, -nr_pages);
}
static int shmem_split_large_entry(struct inode *inode, pgoff_t index,
@ -2519,6 +2561,8 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
else
error = -ENOMEM;
folio = NULL;
/* Make sure to validate the original value. */
swap = index_entry;
goto failed;
}
if (fault_type) {
@ -5408,12 +5452,35 @@ static const struct file_operations shmem_file_operations = {
#endif
};
#ifdef CONFIG_TMPFS_POSIX_ACL
static int shmem_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
struct posix_acl *acl, int type)
{
struct inode *inode = d_inode(dentry);
umode_t mode = inode->i_mode;
int error;
if (type == ACL_TYPE_ACCESS) {
error = posix_acl_equiv_mode(acl, &mode);
if (error < 0)
return error;
error = memfd_check_seals_mode(SHMEM_I(inode)->seals,
inode->i_mode, mode);
if (error)
return error;
}
return simple_set_acl(idmap, dentry, acl, type);
}
#else
#define shmem_set_acl simple_set_acl
#endif
static const struct inode_operations shmem_inode_operations = {
.getattr = shmem_getattr,
.setattr = shmem_setattr,
#ifdef CONFIG_TMPFS_XATTR
.listxattr = shmem_listxattr,
.set_acl = simple_set_acl,
.set_acl = shmem_set_acl,
.fileattr_get = shmem_fileattr_get,
.fileattr_set = shmem_fileattr_set,
#endif

28
mm/shmem_quota.c
View File

@ -239,7 +239,7 @@ static int shmem_acquire_dquot(struct dquot *dquot)
return ret;
}
static bool shmem_is_empty_dquot(struct dquot *dquot)
static bool shmem_is_default_dquot(struct dquot *dquot)
{
struct shmem_sb_info *sbinfo = dquot->dq_sb->s_fs_info;
qsize_t bhardlimit;
@ -253,19 +253,21 @@ static bool shmem_is_empty_dquot(struct dquot *dquot)
ihardlimit = sbinfo->qlimits.grpquota_ihardlimit;
}
if (test_bit(DQ_FAKE_B, &dquot->dq_flags) ||
(dquot->dq_dqb.dqb_curspace == 0 &&
dquot->dq_dqb.dqb_curinodes == 0 &&
dquot->dq_dqb.dqb_bhardlimit == bhardlimit &&
dquot->dq_dqb.dqb_ihardlimit == ihardlimit))
return true;
return false;
/*
* Defaults are zero usage and soft limits, and the mount's hard limits.
* DQ_FAKE_B only means that *all* limits are zero, which need not match
* the mount defaults.
*/
return dquot->dq_dqb.dqb_curspace == 0 &&
dquot->dq_dqb.dqb_curinodes == 0 &&
dquot->dq_dqb.dqb_bsoftlimit == 0 &&
dquot->dq_dqb.dqb_isoftlimit == 0 &&
dquot->dq_dqb.dqb_bhardlimit == bhardlimit &&
dquot->dq_dqb.dqb_ihardlimit == ihardlimit;
}
/*
* Store limits from dquot in the tree unless it's fake. If it is fake
* remove the id from the tree since there is no useful information in
* there.
* Remove the ID from the tree if it has no usage and its limits match
* the mount defaults. Otherwise, save its limits for the next acquire.
*/
static int shmem_release_dquot(struct dquot *dquot)
{
@ -300,7 +302,7 @@ static int shmem_release_dquot(struct dquot *dquot)
return -ENOENT;
found:
if (shmem_is_empty_dquot(dquot)) {
if (shmem_is_default_dquot(dquot)) {
/* Remove entry from the tree */
rb_erase(&entry->node, info->dqi_priv);
kfree(entry);

3
mm/swapfile.c
View File

@ -2514,7 +2514,8 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
return 0;
swapcache = folio;
folio = ksm_might_need_to_copy(folio, vma, addr);
folio = ksm_might_need_to_copy(folio,
folio_file_page(folio, swp_offset(entry)), vma, addr);
if (unlikely(!folio))
return -ENOMEM;
else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {

13
mm/truncate.c
View File

@ -475,8 +475,11 @@ void truncate_inode_pages_range(struct address_space *mapping,
}
}
if (start >= end)
return;
index = start;
while (index < end) {
while (true) {
cond_resched();
if (!find_get_entries(mapping, &index, end - 1, &fbatch,
indices)) {
@ -499,6 +502,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
folio_lock(folio);
VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio);
folio_wait_writeback(folio);
/* Do not handle errors, we'll do another lookup */
truncate_inode_folio(mapping, folio);
folio_unlock(folio);
}
@ -760,7 +764,12 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
folio_lock(folio);
if (unlikely(folio->mapping != mapping)) {
folio_unlock(folio);
continue;
/*
* Folio might have been replaced: look up
* again.
*/
index = indices[i];
break;
}
VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio);
folio_wait_writeback(folio);

75
mm/userfaultfd.c
View File

@ -1206,33 +1206,6 @@ int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
return 0;
}
void double_pt_lock(spinlock_t *ptl1,
spinlock_t *ptl2)
__acquires(ptl1)
__acquires(ptl2)
{
if (ptl1 > ptl2)
swap(ptl1, ptl2);
/* lock in virtual address order to avoid lock inversion */
spin_lock(ptl1);
if (ptl1 != ptl2)
spin_lock_nested(ptl2, SINGLE_DEPTH_NESTING);
else
__acquire(ptl2);
}
void double_pt_unlock(spinlock_t *ptl1,
spinlock_t *ptl2)
__releases(ptl1)
__releases(ptl2)
{
spin_unlock(ptl1);
if (ptl1 != ptl2)
spin_unlock(ptl2);
else
__release(ptl2);
}
static inline bool is_pte_pages_stable(pte_t *dst_pte, pte_t *src_pte,
pte_t orig_dst_pte, pte_t orig_src_pte,
pmd_t *dst_pmd, pmd_t dst_pmdval)
@ -2270,6 +2243,9 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
return ret;
}
static void wake_userfault_range(struct userfaultfd_ctx *ctx,
unsigned long start, unsigned long end);
/* Assumes mmap write lock taken, and mm_struct pinned. */
static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
struct vm_area_struct *vma,
@ -2287,6 +2263,9 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
prev = vma;
for_each_vma_range(vmi, vma, end) {
const bool any_mode_removed = vma->vm_flags & __VM_UFFD_FLAGS & ~vm_flags;
const vma_flags_t curr_vma_flags = vma_flags_and_mask(&vma->flags, __VMA_UFFD_FLAGS);
cond_resched();
VM_WARN_ON_ONCE(!vma_can_userfault(vma, vm_flags, wp_async));
@ -2299,7 +2278,7 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
* userfaultfd and with the right tracking mode too.
*/
if (vma->vm_userfaultfd_ctx.ctx == ctx &&
vma_test_all_mask(vma, vma_flags))
vma_flags_same_pair(&curr_vma_flags, &vma_flags))
goto skip;
/*
@ -2332,6 +2311,10 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
*/
userfaultfd_set_ctx(vma, ctx, vm_flags);
/* Retry faults for removed modes while the VMA is write-locked. */
if (any_mode_removed)
wake_userfault_range(ctx, start, vma_end);
if (vma_is_hugetlb(vma) && uffd_disable_huge_pmd_share(vma))
hugetlb_unshare_all_pmds(vma);
@ -3648,6 +3631,17 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
__wake_userfault(ctx, range);
}
static void wake_userfault_range(struct userfaultfd_ctx *ctx,
unsigned long start, unsigned long end)
{
struct userfaultfd_wake_range range = {
.start = start,
.len = end - start,
};
wake_userfault(ctx, &range);
}
static __always_inline int validate_unaligned_range(
struct mm_struct *mm, __u64 start, __u64 len)
{
@ -3977,6 +3971,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
ret = 0;
for_each_vma_range(vmi, vma, end) {
struct userfaultfd_wake_range range;
cond_resched();
/* VMA not registered with userfaultfd. */
@ -3991,18 +3987,17 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
start = vma->vm_start;
vma_end = min(end, vma->vm_end);
if (userfaultfd_missing(vma)) {
/*
* Wake any concurrent pending userfault while
* we unregister, so they will not hang
* permanently and it avoids userland to call
* UFFDIO_WAKE explicitly.
*/
struct userfaultfd_wake_range range;
range.start = start;
range.len = vma_end - start;
wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
}
/* Prevent new userfaults from being queued after the wakeup. */
vma_start_write(vma);
/*
* Wake any concurrent pending userfault while we unregister,
* so they will not hang permanently and it avoids userland to
* call UFFDIO_WAKE explicitly.
*/
range.start = start;
range.len = vma_end - start;
wake_userfault(ctx, &range);
vma = userfaultfd_clear_vma(&vmi, prev, vma,
start, vma_end);

13
mm/util.c
View File

@ -697,6 +697,19 @@ void *vcalloc_noprof(size_t n, size_t size)
}
EXPORT_SYMBOL(vcalloc_noprof);
/**
* folio_anon_vma - Get a folio's raw anon_vma pointer.
* @folio: The folio.
*
* The returned pointer may be stale for an unmapped folio. The caller
* must ensure the anon_vma stays alive before dereferencing it. A folio
* reference or lock alone is insufficient; use folio_get_anon_vma() when
* the anon_vma's lifetime is not otherwise protected. Note that
* folio_get_anon_vma() is only useful for folios that are still mapped.
*
* Return: The raw anon_vma pointer for an anonymous, non-KSM folio, or
* %NULL otherwise.
*/
struct anon_vma *folio_anon_vma(const struct folio *folio)
{
unsigned long mapping = (unsigned long)folio->mapping;

24
mm/vma.c
View File

@ -516,6 +516,15 @@ void unmap_region(struct unmap_desc *unmap)
tlb_finish_mmu(&tlb);
}
static int check_split(struct vm_area_struct *vma, unsigned long addr)
{
if (!vma->vm_ops)
return 0;
if (!vma->vm_ops->may_split)
return 0;
return vma->vm_ops->may_split(vma, addr);
}
/*
* __split_vma() bypasses sysctl_max_map_count checking. We use this where it
* has already been checked or doesn't make sense to fail.
@ -532,11 +541,9 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
WARN_ON(vma->vm_start >= addr);
WARN_ON(vma->vm_end <= addr);
if (vma->vm_ops && vma->vm_ops->may_split) {
err = vma->vm_ops->may_split(vma, addr);
if (err)
return err;
}
err = check_split(vma, addr);
if (err)
return err;
vma_start_write(vma);
new = vm_area_dup(vma);
@ -969,6 +976,10 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
if (vmg->__remove_middle && !can_merge_remove_vma(middle))
return NULL;
/* If middle isn't removed then it's resized, i.e. splits. */
if (!vmg->__remove_middle && check_split(middle, left_side ? end : start))
return NULL;
/*
* If we merge both VMAs, then next is also deleted. This implies
* merge_will_delete_vma also.
@ -3406,7 +3417,8 @@ static int acct_stack_growth(struct vm_area_struct *vma,
* Overcommit.. This must be the final test, as it will
* update security statistics.
*/
if (security_vm_enough_memory_mm(mm, grow))
if (vma_test(vma, VMA_ACCOUNT_BIT) &&
security_vm_enough_memory_mm(mm, grow))
return -ENOMEM;
return 0;

4
mm/vmscan.c
View File

@ -3911,7 +3911,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
walk->mm_stats[MM_NONLEAF_FOUND]++;
if (!walk_pte_range(&val, addr, next, args))
if (!walk_pte_range(pmd + i, addr, next, args))
continue;
walk->mm_stats[MM_NONLEAF_ADDED]++;
@ -3947,7 +3947,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
continue;
walk_pmd_range(&val, addr, next, args);
walk_pmd_range(pud + i, addr, next, args);
if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
end = (addr | ~PUD_MASK) + 1;