diff --git a/Documentation/mm/vmemmap_dedup.rst b/Documentation/mm/vmemmap_dedup.rst index 8c287ae3f86c..5e792b7a27f0 100644 --- a/Documentation/mm/vmemmap_dedup.rst +++ b/Documentation/mm/vmemmap_dedup.rst @@ -130,8 +130,19 @@ pages of ``struct page`` (page 1 to page 7) are tail pages. The optimization is only applied when the size of the struct page is a power of 2. In this case, all tail pages of the same order are identical. See compound_head(). This allows us to remap the tail pages of the vmemmap to a -shared, read-only page. The head page is also remapped to a new page. This -allows the original vmemmap pages to be freed. +shared page. The head page is also remapped to a new page. This allows the +original vmemmap pages to be freed. + +The shared page is mapped writable. A lockless get of a page reference can +read a non-zero refcount before the remap and attempt its atomic update +after it. The refcount of a tail page is zero, so the update fails without +changing the page. Nothing else may write to the shared page: it is shared +by all HugeTLB pages of the same size in the zone, and a stray write would +corrupt all of them. + +HugeTLB pages whose vmemmap is optimized when the memmap is first set up at +boot keep a read-only mapping of the shared page. Nothing can do a lockless +get on them before they are handed to HugeTLB. Here is how things look after remapping:: diff --git a/MAINTAINERS b/MAINTAINERS index 93abbb315065..d63d8b215277 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6868,11 +6868,13 @@ R: Muchun Song L: cgroups@vger.kernel.org L: linux-mm@kvack.org S: Maintained +F: Documentation/admin-guide/cgroup-v1/memory.rst F: include/linux/memcontrol.h F: include/linux/page_counter.h -F: mm/memcontrol.c +F: include/trace/events/memcg.h F: mm/memcontrol-v1.c F: mm/memcontrol-v1.h +F: mm/memcontrol.c F: mm/page_counter.c F: samples/cgroup/* F: tools/testing/selftests/cgroup/memcg_protection.m @@ -12376,6 +12378,7 @@ R: David Hildenbrand L: linux-mm@kvack.org S: Maintained F: Documentation/ABI/testing/sysfs-kernel-mm-hugepages +F: Documentation/admin-guide/cgroup-v1/hugetlb.rst F: Documentation/admin-guide/mm/hugetlbpage.rst F: Documentation/mm/hugetlbfs_reserv.rst F: Documentation/mm/vmemmap_dedup.rst @@ -12387,6 +12390,7 @@ F: mm/hugetlb.c F: mm/hugetlb_cgroup.c F: mm/hugetlb_cma.c F: mm/hugetlb_cma.h +F: mm/hugetlb_internal.h F: mm/hugetlb_sysctl.c F: mm/hugetlb_sysfs.c F: mm/hugetlb_vmemmap.c @@ -12406,6 +12410,7 @@ M: Miaohe Lin R: Naoya Horiguchi L: linux-mm@kvack.org S: Maintained +F: Documentation/ABI/testing/sysfs-memory-page-offline F: include/linux/memory-failure.h F: include/trace/events/memory-failure.h F: mm/hwpoison-inject.c @@ -17456,21 +17461,6 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git F: drivers/devfreq/tegra30-devfreq.c -MEMORY HOT(UN)PLUG -M: David Hildenbrand -M: Oscar Salvador -L: linux-mm@kvack.org -L: linux-cxl@vger.kernel.org -S: Maintained -F: Documentation/admin-guide/mm/memory-hotplug.rst -F: Documentation/core-api/memory-hotplug.rst -F: drivers/base/memory.c -F: include/linux/memory_hotplug.h -F: include/linux/memremap.h -F: mm/memremap.c -F: mm/memory_hotplug.c -F: tools/testing/selftests/memory-hotplug/ - MEMORY MANAGEMENT M: Andrew Morton L: linux-mm@kvack.org @@ -17494,18 +17484,15 @@ F: mm/alloc_tag.c F: tools/testing/selftests/alloc_tag/ MEMORY MANAGEMENT - BALLOON -M: Andrew Morton M: David Hildenbrand L: linux-mm@kvack.org L: virtualization@lists.linux.dev S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: include/linux/balloon.h F: mm/balloon.c MEMORY MANAGEMENT - CORE -M: Andrew Morton M: David Hildenbrand R: Lorenzo Stoakes R: Liam R. Howlett @@ -17515,27 +17502,34 @@ R: Suren Baghdasaryan R: Michal Hocko L: linux-mm@kvack.org S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: Documentation/admin-guide/sysctl/vm.rst +F: Documentation/core-api/gfp_mask-from-fs-io.rst +F: fs/proc/meminfo.c +F: include/linux/cacheflush.h F: include/linux/folio_batch.h F: include/linux/gfp.h F: include/linux/gfp_types.h +F: include/linux/highmem-internal.h F: include/linux/highmem.h F: include/linux/leafops.h F: include/linux/memory.h F: include/linux/mm.h F: include/linux/mm_*.h -F: include/linux/mmzone.h F: include/linux/mmdebug.h F: include/linux/mmu_notifier.h +F: include/linux/mmzone.h F: include/linux/pagewalk.h +F: include/linux/pfn.h F: include/linux/pgalloc.h F: include/linux/pgtable.h F: include/linux/ptdump.h +F: include/linux/sched/mm.h +F: include/linux/vm_event_item.h F: include/linux/vmpressure.h F: include/linux/vmstat.h -F: fs/proc/meminfo.c +F: include/trace/events/mmflags.h +F: include/trace/events/pagemap.h F: kernel/fork.c F: mm/Kconfig F: mm/debug.c @@ -17554,6 +17548,7 @@ F: mm/ptdump.c F: mm/util.c F: mm/vmpressure.c F: mm/vmstat.c +F: scripts/gdb/linux/mm.py N: include\/linux\/page[-_][a-zA-Z]* MEMORY MANAGEMENT - EXECMEM @@ -17565,15 +17560,14 @@ F: include/linux/execmem.h F: mm/execmem.c MEMORY MANAGEMENT - GUP (GET USER PAGES) -M: Andrew Morton M: David Hildenbrand R: Jason Gunthorpe R: John Hubbard R: Peter Xu L: linux-mm@kvack.org S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next +F: Documentation/core-api/pin_user_pages.rst F: mm/gup.c F: mm/gup_test.c F: mm/gup_test.h @@ -17582,14 +17576,12 @@ F: tools/testing/selftests/mm/gup.c F: tools/testing/selftests/mm/gup_longterm.c MEMORY MANAGEMENT - KSM (Kernel Samepage Merging) -M: Andrew Morton M: David Hildenbrand R: Xu Xin R: Chengming Zhou L: linux-mm@kvack.org S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: Documentation/ABI/testing/sysfs-kernel-mm-ksm F: Documentation/admin-guide/mm/ksm.rst F: Documentation/mm/ksm.rst @@ -17598,13 +17590,30 @@ F: include/trace/events/ksm.h F: mm/ksm.c F: mm/mm_slot.h +MEMORY MANAGEMENT - MEMORY HOT(UN)PLUG +M: David Hildenbrand +M: Oscar Salvador +L: linux-mm@kvack.org +L: linux-cxl@vger.kernel.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next +F: Documentation/ABI/testing/sysfs-devices-memory +F: Documentation/admin-guide/mm/memory-hotplug.rst +F: Documentation/core-api/memory-hotplug.rst +F: drivers/base/memory.c +F: include/linux/memory_hotplug.h +F: include/linux/memremap.h +F: mm/memory_hotplug.c +F: mm/memremap.c +F: tools/testing/selftests/memory-hotplug/ + MEMORY MANAGEMENT - MEMORY MAP M: Muchun Song M: Mike Rapoport M: David Hildenbrand L: linux-mm@kvack.org S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git memmap +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: Documentation/mm/memory-model.rst F: arch/*/include/asm/sparsemem.h F: include/asm-generic/memory_model.h @@ -17636,7 +17645,6 @@ F: mm/vmscan.c F: mm/workingset.c MEMORY MANAGEMENT - MIGRATION -M: Andrew Morton M: David Hildenbrand R: Zi Yan R: Matthew Brost @@ -17647,10 +17655,10 @@ R: Ying Huang R: Alistair Popple L: linux-mm@kvack.org S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: include/linux/migrate.h F: include/linux/migrate_mode.h +F: include/trace/events/migrate.h F: mm/migrate.c F: mm/migrate_device.c @@ -17674,15 +17682,20 @@ F: Documentation/admin-guide/mm/ F: Documentation/core-api/memory-allocation.rst F: Documentation/core-api/mm-api.rst F: Documentation/mm/ -X: Documentation/admin-guide/mm/damon/ -X: Documentation/mm/damon/ -F: mm/char-mem.c +F: fs/proc/page.c +F: include/linux/backing-dev-defs.h +F: include/linux/backing-dev.h F: include/linux/cma.h F: include/linux/dmapool.h F: include/linux/ioremap.h +F: include/linux/kernel-page-flags.h F: include/linux/page_idle.h +F: include/trace/events/cma.h +F: include/uapi/linux/kernel-page-flags.h F: mm/backing-dev.c +F: mm/char-mem.c F: mm/cma.c +F: mm/cma.h F: mm/cma_debug.c F: mm/cma_sysfs.c F: mm/dmapool.c @@ -17695,7 +17708,11 @@ F: mm/page_idle.c F: mm/pgalloc-track.h F: mm/process_vm_access.c F: tools/lib/mm/ +F: tools/mm/ F: tools/testing/selftests/mm/ +X: Documentation/ABI/testing/sysfs-kernel-mm-damon +X: Documentation/admin-guide/mm/damon/ +X: Documentation/mm/damon/ MEMORY MANAGEMENT - NUMA MEMBLOCKS AND NUMA EMULATION M: Andrew Morton @@ -17703,13 +17720,13 @@ M: Mike Rapoport L: linux-mm@kvack.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git +F: include/linux/numa.h F: include/linux/numa_memblks.h F: mm/numa.c F: mm/numa_emulation.c F: mm/numa_memblks.c MEMORY MANAGEMENT - NUMA PLACEMENT -M: Andrew Morton M: David Hildenbrand M: Gregory Price R: Zi Yan @@ -17718,11 +17735,11 @@ R: Rakie Kim R: Ying Huang L: linux-mm@kvack.org S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers F: Documentation/ABI/testing/sysfs-kernel-mm-mempolicy F: Documentation/ABI/testing/sysfs-kernel-mm-mempolicy-weighted-interleave +F: Documentation/admin-guide/numastat.rst F: include/linux/memory-tiers.h F: include/linux/mempolicy.h F: include/uapi/linux/mempolicy.h @@ -17741,19 +17758,22 @@ F: include/uapi/linux/oom.h F: mm/oom_kill.c MEMORY MANAGEMENT - PAGE ALLOCATOR -M: Andrew Morton M: Vlastimil Babka +M: Johannes Weiner R: Suren Baghdasaryan R: Michal Hocko R: Brendan Jackman -R: Johannes Weiner R: Zi Yan L: linux-mm@kvack.org S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/page_alloc.git page_alloc/for-next F: include/linux/compaction.h F: include/linux/gfp.h F: include/linux/page-isolation.h F: include/linux/pageblock-flags.h +F: include/trace/events/compaction.h +F: include/trace/events/page_isolation.h +F: include/trace/events/page_ref.h F: mm/compaction.c F: mm/debug_page_alloc.c F: mm/debug_page_ref.c @@ -17770,6 +17790,9 @@ F: mm/page_reporting.h F: mm/show_mem.c F: mm/shuffle.c F: mm/shuffle.h +F: scripts/gdb/linux/page_owner.py +F: tools/mm/page_owner_filter.c +F: tools/mm/page_owner_sort.c MEMORY MANAGEMENT - RECLAIM M: Andrew Morton @@ -17781,11 +17804,11 @@ R: Shakeel Butt R: Lorenzo Stoakes L: linux-mm@kvack.org S: Maintained +F: include/trace/events/vmscan.h F: mm/vmscan.c F: mm/workingset.c MEMORY MANAGEMENT - RMAP (REVERSE MAPPING) -M: Andrew Morton M: David Hildenbrand M: Lorenzo Stoakes R: Rik van Riel @@ -17796,6 +17819,7 @@ R: Jann Horn R: Lance Yang L: linux-mm@kvack.org S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: include/linux/rmap.h F: mm/interval_tree.c F: mm/page_vma_mapped.c @@ -17810,6 +17834,54 @@ S: Maintained F: include/linux/secretmem.h F: mm/secretmem.c +MEMORY MANAGEMENT - SHARED MEMORY (SHMEM/TMPFS/MEMFD) +M: Hugh Dickins +M: David Hildenbrand +R: Baolin Wang +L: linux-mm@kvack.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next +F: Documentation/filesystems/tmpfs.rst +F: Documentation/mm/shmfs.rst +F: Documentation/userspace-api/mfd_noexec.rst +F: include/linux/memfd.h +F: include/linux/shmem_fs.h +F: include/uapi/linux/memfd.h +F: mm/memfd.c +F: mm/shmem.c +F: mm/shmem_quota.c +F: tools/testing/selftests/memfd/ +F: tools/testing/selftests/tmpfs/ + +MEMORY MANAGEMENT - SLAB ALLOCATOR +M: Vlastimil Babka +M: Harry Yoo +R: Hao Li +R: Christoph Lameter +R: David Rientjes +R: Roman Gushchin +L: linux-mm@kvack.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/slab.git slab/for-next +F: Documentation/ABI/testing/sysfs-kernel-slab +F: Documentation/admin-guide/mm/slab.rst +F: Documentation/mm/slab.rst +F: include/linux/mempool.h +F: include/linux/slab.h +F: include/trace/events/kmem.h +F: lib/tests/slub_kunit.c +F: mm/failslab.c +F: mm/mempool.c +F: mm/slab.h +F: mm/slab_common.c +F: mm/slub.c +F: scripts/gdb/linux/slab.py +F: tools/cgroup/memcg_slabinfo.py +F: tools/include/linux/slab.h +F: tools/lib/slab.c +F: tools/mm/slabinfo-gnuplot.sh +F: tools/mm/slabinfo.c + MEMORY MANAGEMENT - SWAP M: Andrew Morton M: Chris Li @@ -17829,14 +17901,13 @@ F: include/linux/swapfile.h F: include/linux/swapops.h F: mm/page_io.c F: mm/swap.h -F: mm/swap_table.h F: mm/swap_state.c +F: mm/swap_table.h F: mm/swapfile.c MEMORY MANAGEMENT - THP (TRANSPARENT HUGE PAGE) -M: Andrew Morton M: David Hildenbrand -R: Zi Yan +M: Zi Yan R: Baolin Wang R: Liam R. Howlett R: Nico Pache @@ -17848,17 +17919,18 @@ R: Usama Arif R: Kiryl Shutsemau L: linux-mm@kvack.org S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/core.git for-next F: Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage F: Documentation/admin-guide/mm/transhuge.rst F: include/linux/huge_mm.h F: include/linux/khugepaged.h F: include/trace/events/huge_memory.h +F: include/trace/events/thp.h F: mm/collapse.h F: mm/huge_memory.c F: mm/khugepaged.c F: mm/mm_slot.h +F: tools/mm/thpmaps F: tools/testing/selftests/mm/khugepaged.c F: tools/testing/selftests/mm/split_huge_page_test.c F: tools/testing/selftests/mm/transhuge-stress.c @@ -17902,10 +17974,16 @@ L: linux-mm@kvack.org S: Maintained W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm -F: mm/char-mem.c -F: include/trace/events/mmap.h +F: Documentation/core-api/protection-keys.rst +F: Documentation/filesystems/mmap_prepare.rst +F: Documentation/userspace-api/mseal.rst F: fs/proc/task_mmu.c F: fs/proc/task_nommu.c +F: include/linux/mman.h +F: include/linux/pkeys.h +F: include/trace/events/mmap.h +F: include/uapi/linux/mman.h +F: mm/char-mem.c F: mm/mincore.c F: mm/mlock.c F: mm/mmap.c @@ -18743,6 +18821,7 @@ M: Peter Zijlstra L: linux-arch@vger.kernel.org L: linux-mm@kvack.org S: Maintained +F: Documentation/core-api/cachetlb.rst F: arch/*/include/asm/tlb.h F: include/asm-generic/tlb.h F: include/trace/events/tlb.h @@ -21853,6 +21932,7 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git F: arch/*/include/asm/percpu.h F: include/linux/percpu*.h +F: include/trace/events/percpu.h F: lib/percpu*.c F: mm/percpu*.c F: mm/percpu-internal.h @@ -25579,6 +25659,7 @@ F: include/linux/shrinker.h F: mm/list_lru.c F: mm/shrinker.c F: mm/shrinker_debug.c +F: tools/cgroup/memcg_shrinker.py SH_VOU V4L2 OUTPUT DRIVER L: linux-media@vger.kernel.org @@ -25800,34 +25881,6 @@ S: Maintained F: Documentation/devicetree/bindings/nvmem/layouts/kontron,sl28-vpd.yaml F: drivers/nvmem/layouts/sl28vpd.c -SLAB ALLOCATOR -M: Vlastimil Babka -M: Harry Yoo -M: Andrew Morton -R: Hao Li -R: Christoph Lameter -R: David Rientjes -R: Roman Gushchin -L: linux-mm@kvack.org -S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/slab.git -F: Documentation/admin-guide/mm/slab.rst -F: Documentation/mm/slab.rst -F: include/linux/mempool.h -F: include/linux/slab.h -F: lib/tests/slub_kunit.c -F: mm/failslab.c -F: mm/mempool.c -F: mm/slab.h -F: mm/slab_common.c -F: mm/slub.c -F: scripts/gdb/linux/slab.py -F: tools/cgroup/memcg_slabinfo.py -F: tools/include/linux/slab.h -F: tools/lib/slab.c -F: tools/mm/slabinfo-gnuplot.sh -F: tools/mm/slabinfo.c - SLCAN CAN NETWORK DRIVER M: Dario Binacchi L: linux-can@vger.kernel.org @@ -28245,17 +28298,6 @@ S: Maintained F: Documentation/hwmon/tmp513.rst F: drivers/hwmon/tmp513.c -TMPFS (SHMEM FILESYSTEM) -M: Hugh Dickins -R: Baolin Wang -L: linux-mm@kvack.org -S: Maintained -F: include/linux/memfd.h -F: include/linux/shmem_fs.h -F: mm/memfd.c -F: mm/shmem.c -F: mm/shmem_quota.c - TOMOYO SECURITY MODULE M: Kentaro Takeda M: Tetsuo Handa @@ -29879,9 +29921,11 @@ S: Maintained W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: include/linux/vmalloc.h +F: include/trace/events/vmalloc.h +F: lib/test_vmalloc.c F: mm/vmalloc.c F: mm/vmalloc.h -F: lib/test_vmalloc.c +F: scripts/gdb/linux/vmalloc.py VME SUBSYSTEM L: linux-kernel@vger.kernel.org diff --git a/fs/proc/internal.h b/fs/proc/internal.h index df873df9afc5..9af1f63258b7 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -393,6 +393,7 @@ struct proc_maps_private { struct inode *inode; struct task_struct *task; struct vma_iterator iter; + struct vma_rcu_iter rvmi; loff_t last_pos; struct proc_maps_locking_ctx lock_ctx; #ifdef CONFIG_NUMA diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 7452b695fdae..b87d3e86442d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -160,7 +160,7 @@ static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, return vma_next(&priv->iter); unlock_ctx_vma(lock_ctx); - vma = lock_next_vma(lock_ctx->mm, &priv->iter, last_pos); + vma = lock_next_vma(&priv->rvmi, last_pos); if (!IS_ERR_OR_NULL(vma)) lock_ctx->locked_vma = vma; @@ -265,7 +265,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos) */ if (last_addr > 0) *ppos = last_addr = priv->last_pos; - vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); + vma_rcu_iter_init(&priv->rvmi, mm, &priv->iter, (unsigned long)last_addr); hold_task_mempolicy(priv); /* * If seq_file had to flush its collected data right after m_next() set @@ -515,6 +515,7 @@ static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ct struct mm_struct *mm = lock_ctx->mm; struct vm_area_struct *vma; struct vma_iterator vmi; + struct vma_rcu_iter rvmi; if (lock_ctx->mmap_locked) return find_vma(mm, addr); @@ -522,8 +523,8 @@ static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ct /* Unlock previously locked VMA and find the next one under RCU */ unlock_ctx_vma(lock_ctx); rcu_read_lock(); - vma_iter_init(&vmi, mm, addr); - vma = lock_next_vma(mm, &vmi, addr); + vma_rcu_iter_init(&rvmi, mm, &vmi, addr); + vma = lock_next_vma(&rvmi, addr); rcu_read_unlock(); if (!vma) @@ -1416,7 +1417,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) rcu_read_lock(); reset_lock_ctx(lock_ctx); - vma_iter_init(&priv->iter, mm, 0); + vma_rcu_iter_init(&priv->rvmi, mm, &priv->iter, 0); vma = proc_get_vma(m, &pos); if (unlikely(!vma)) goto empty_set; @@ -1465,7 +1466,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) reset_lock_ctx(lock_ctx); /* Resume from the last position. */ pos = last_vma_end; - vma_iter_init(&priv->iter, mm, pos); + vma_rcu_iter_init(&priv->rvmi, mm, &priv->iter, pos); } vma = proc_get_vma(m, &pos); } @@ -3201,7 +3202,7 @@ static long do_pagemap_cmd(struct file *file, unsigned int cmd, return do_pagemap_scan(mm, arg); default: - return -EINVAL; + return -ENOIOCTLCMD; } } diff --git a/include/linux/damon.h b/include/linux/damon.h index 63050eb2206a..380f9cf6ddd8 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -958,6 +958,16 @@ struct damon_ctx { /* @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct rnd_state rnd_state; + /* + * number of regions after last merge. Read kdamond_split_regions() + * for more details. + */ + unsigned long last_nr_regions; + /* + * number of damon_split_some_regions() invocations for this context. + * Read damon_split_some_regions() for more details. + */ + unsigned long region_split_rotation; }; /* Get a random number in [@l, @r) using @ctx's lockless PRNG. */ diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 5029c7241863..58f755e70e96 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -121,10 +121,9 @@ void hugepage_put_subpool(struct hugepage_subpool *spool); void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); -int move_hugetlb_page_tables(struct vm_area_struct *vma, - struct vm_area_struct *new_vma, - unsigned long old_addr, unsigned long new_addr, - unsigned long len); +unsigned long move_hugetlb_page_tables(struct vm_area_struct *vma, + struct vm_area_struct *new_vma, unsigned long old_addr, + unsigned long new_addr, unsigned long len); int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *, struct vm_area_struct *); void unmap_hugepage_range(struct vm_area_struct *, @@ -338,11 +337,9 @@ static inline int copy_hugetlb_page_range(struct mm_struct *dst, return 0; } -static inline int move_hugetlb_page_tables(struct vm_area_struct *vma, - struct vm_area_struct *new_vma, - unsigned long old_addr, - unsigned long new_addr, - unsigned long len) +static inline unsigned long move_hugetlb_page_tables(struct vm_area_struct *vma, + struct vm_area_struct *new_vma, unsigned long old_addr, + unsigned long new_addr, unsigned long len) { BUG(); return 0; diff --git a/include/linux/mm.h b/include/linux/mm.h index 9f167950926a..6af3b7008f1a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -602,7 +602,7 @@ enum { #define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT) /* These flags can be updated atomically via VMA/mmap read lock. */ -#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD +#define VMA_ATOMIC_SET_ALLOWED mk_vma_flags(VMA_MAYBE_GUARD_BIT) /* Arch-specific flags to clear when updating VM flags on protection change */ #ifndef VM_ARCH_CLEAR @@ -987,22 +987,6 @@ static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_init(vma, flags); } -static inline void vma_flags_reset_once(struct vm_area_struct *vma, - vma_flags_t *flags) -{ - const unsigned long word = flags->__vma_flags[0]; - - /* It is assumed only the first system word must be written once. */ - vma_flags_overwrite_word_once(&vma->flags, word); - /* The remainder can be copied normally. */ - if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { - unsigned long *dst = &vma->flags.__vma_flags[1]; - const unsigned long *src = &flags->__vma_flags[1]; - - bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); - } -} - static inline void vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags) { @@ -1039,48 +1023,6 @@ static inline void vm_flags_mod(struct vm_area_struct *vma, __vm_flags_mod(vma, set, clear); } -static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, - vma_flag_t bit) -{ - const vm_flags_t mask = BIT((__force int)bit); - - /* Only specific flags are permitted */ - if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED))) - return false; - - return true; -} - -/* - * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific - * valid flags are allowed to do this. - */ -static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, - vma_flag_t bit) -{ - unsigned long *bitmap = vma->flags.__vma_flags; - - vma_assert_stabilised(vma); - if (__vma_atomic_valid_flag(vma, bit)) - set_bit((__force int)bit, bitmap); -} - -/* - * Test for VMA flag atomically. Requires no locks. Only specific valid flags - * are allowed to do this. - * - * This is necessarily racey, so callers must ensure that serialisation is - * achieved through some other means, or that races are permissible. - */ -static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, - vma_flag_t bit) -{ - if (__vma_atomic_valid_flag(vma, bit)) - return test_bit((__force int)bit, &vma->vm_flags); - - return false; -} - /* Set an individual VMA flag in flags, non-atomically. */ static __always_inline void vma_flags_set_flag(vma_flags_t *flags, vma_flag_t bit) @@ -1302,33 +1244,23 @@ static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, return dst; } -/* Determine if flags and flags_other have precisely the same flags set. */ -static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, - const vma_flags_t *flags_other) -{ - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other->__vma_flags; - - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); -} - -/* Determine if flags and flags_other have precisely the same flags set. */ -static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, - vma_flags_t flags_other) -{ - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other.__vma_flags; - - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); -} - /* - * Helper macro to determine if only the specific flags are set, e.g.: - * - * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... } + * Determine if flags and flags_other have precisely the same flags set, + * excluding atomic VMA flags. */ -#define vma_flags_same(flags, ...) \ - vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) +static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, + const vma_flags_t *flags_other) +{ + vma_flags_t diff = vma_flags_diff_pair(flags, flags_other); + + /* + * Atomic VMA flags may be set concurrently under the VMA read lock, so + * are disregarded for equality testing purposes. + */ + vma_flags_clear_mask(&diff, VMA_ATOMIC_SET_ALLOWED); + + return vma_flags_empty(&diff); +} /* * Test whether a specific flag in the VMA is set, e.g.: @@ -1436,6 +1368,78 @@ static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, #define vma_clear_flags(vma, ...) \ vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) +static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, + vma_flag_t bit) +{ + const vma_flags_t mask = VMA_ATOMIC_SET_ALLOWED; + + /* Is this flag allowed to be set atomically? */ + if (WARN_ON_ONCE(!vma_flags_test(&mask, bit))) + return false; + + return true; +} + +/* + * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific + * valid flags are allowed to do this. + */ +static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, + vma_flag_t bit) +{ + unsigned long *bitmap = vma->flags.__vma_flags; + + vma_assert_stabilised(vma); + if (__vma_atomic_valid_flag(vma, bit)) + set_bit((__force int)bit, bitmap); +} + +/* + * Test for VMA flag atomically. Requires no locks. Only specific valid flags + * are allowed to do this. + * + * This is necessarily racey, so callers must ensure that serialisation is + * achieved through some other means, or that races are permissible. + */ +static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, + vma_flag_t bit) +{ + if (__vma_atomic_valid_flag(vma, bit)) + return test_bit((__force int)bit, &vma->vm_flags); + + return false; +} + +/* + * Overwrite the VMA's flags with new ones, preserving atomic VMA flags in the + * process. The write is done once and once only. + */ +static inline void vma_overwrite_flags(struct vm_area_struct *vma, + vma_flags_t new_flags) +{ + vma_flags_t atomic; + + vma_assert_write_locked(vma); + + /* + * Preserve the VMA's atomic flags, which are kept stable by the VMA + * write lock. + */ + atomic = vma_flags_and_mask(&vma->flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_clear_mask(&new_flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_set_mask(&new_flags, atomic); + + /* It is assumed only the first system word must be written once. */ + vma_flags_overwrite_word_once(&vma->flags, new_flags.__vma_flags[0]); + /* The remainder can be copied normally. */ + if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { + unsigned long *dst = &vma->flags.__vma_flags[1]; + const unsigned long *src = &new_flags.__vma_flags[1]; + + bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); + } +} + /* * Test whether a specific VMA flag is set in a VMA descriptor, e.g.: * diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 11b3ffef3966..9ccbbcbd7aca 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -17,6 +17,13 @@ extern int rcuwait_wake_up(struct rcuwait *w); #define MMAP_LOCK_INITIALIZER(name) \ .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock), +/* Tie a VMA iterator to mm write lock state at iterator initialisation. */ +struct vma_rcu_iter { + struct mm_struct *mm; + struct vma_iterator *vmi; + unsigned int mm_wr_seq; +}; + DECLARE_TRACEPOINT(mmap_lock_start_locking); DECLARE_TRACEPOINT(mmap_lock_acquire_returned); DECLARE_TRACEPOINT(mmap_lock_released); @@ -509,6 +516,16 @@ static inline void vma_mark_detached(struct vm_area_struct *vma) struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, unsigned long address); +static inline void vma_rcu_iter_init(struct vma_rcu_iter *rvmi, + struct mm_struct *mm, struct vma_iterator *vmi, + unsigned long addr) +{ + vma_iter_init(vmi, mm, addr); + rvmi->mm = mm; + rvmi->vmi = vmi; + rvmi->mm_wr_seq = UINT_MAX; /* Start odd so changed by default. */ +} + /* * Locks next vma pointed by the iterator. Confirms the locked vma has not * been modified and will retry under mmap_lock protection if modification @@ -516,9 +533,8 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, * Returns either a valid locked VMA, NULL if no more VMAs or -EINTR if the * process was interrupted. */ -struct vm_area_struct *lock_next_vma(struct mm_struct *mm, - struct vma_iterator *iter, - unsigned long address); +struct vm_area_struct *lock_next_vma(struct vma_rcu_iter *rvmi, + unsigned long from_addr); static inline void vma_assert_can_modify(struct vm_area_struct *vma) { diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index a14b8a9ffb7b..ebd86677b052 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -168,22 +168,12 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, } /* - * Never enable huge pmd sharing on some uffd registered vmas: - * - * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access - * tracking information is per pgtable entry. - * - * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for - * VMAs which share huge pmds. (If you have two mappings to the same - * underlying pages, and fault in the non-UFFD-registered one with a write, - * with huge pmd sharing this would *also* setup the second UFFD-registered - * mapping, and we'd not get minor faults.) + * Never enable huge pmd sharing on userfaultfd registered VMAs, because + * userfaultfd state can be tracked per VMA or per page table entry. */ static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) { - return vma_test_any_mask(vma, - mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, - VMA_UFFD_MINOR)); + return vma->vm_userfaultfd_ctx.ctx; } /* diff --git a/mm/cma.c b/mm/cma.c index a10ea37a261d..f61d52796f4e 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -381,6 +381,8 @@ static int __init cma_fixed_reserve(phys_addr_t base, phys_addr_t size) return -EBUSY; } + accept_memory(base, size); + return 0; } @@ -678,6 +680,8 @@ int __init cma_declare_contiguous_multi(phys_addr_t total_size, break; } + accept_memory(mlp->base, size); + pr_debug("created region %d: %016llx - %016llx\n", nr, (u64)mlp->base, (u64)mlp->base + size); cmrp = &cma->ranges[nr++]; diff --git a/mm/damon/core.c b/mm/damon/core.c index eb1fcdabc007..73b571b429cb 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -3753,12 +3753,13 @@ static void damon_split_regions_of(struct damon_ctx *ctx, static void damon_split_some_regions(struct damon_ctx *ctx, unsigned long split_step) { - static unsigned long rotation; struct damon_target *t; struct damon_region *r, *next; - unsigned long offset = rotation++ % split_step; + unsigned long offset; unsigned long idx = 0; + offset = ctx->region_split_rotation++ % split_step; + damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) { unsigned long sz_region, sz_sub; @@ -3795,7 +3796,6 @@ static noinline_for_stack void kdamond_split_regions(struct damon_ctx *ctx) struct damon_target *t; unsigned long nr_regions = 0; unsigned long max_nr_regions = ctx->attrs.max_nr_regions; - static unsigned long last_nr_regions; int nr_subregions = 2; damon_for_each_target(t, ctx) @@ -3811,7 +3811,7 @@ static noinline_for_stack void kdamond_split_regions(struct damon_ctx *ctx) } /* Maybe the middle of the region has different access frequency */ - if (last_nr_regions == nr_regions && + if (ctx->last_nr_regions == nr_regions && nr_regions < max_nr_regions / 3) nr_subregions = 3; @@ -3820,7 +3820,7 @@ static noinline_for_stack void kdamond_split_regions(struct damon_ctx *ctx) ctx->min_region_sz); done: - last_nr_regions = nr_regions; + ctx->last_nr_regions = nr_regions; } /* diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 8f083611741f..637f1bd0c950 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1310,7 +1310,10 @@ static ssize_t target_metric_store(struct kobject *kobj, metric_name = &damos_sysfs_qgoal_metric_names[i]; if (sysfs_streq(buf, metric_name->name)) { + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; goal->metric = metric_name->metric; + mutex_unlock(&damon_sysfs_lock); return count; } } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index da5d04edf7c1..ec05e8c1d1f8 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -137,6 +137,10 @@ static int __huge_pmd_unshare(struct mmu_gather *tlb, bool check_locks); static void hugetlb_unshare_pmds(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool take_locks); +static void hugetlb_unmap_range(struct mmu_gather *tlb, + struct vm_area_struct *vma, unsigned long start, + unsigned long end, struct folio *folio, + zap_flags_t zap_flags); static struct resv_map *vma_resv_map(struct vm_area_struct *vma); static inline bool subpool_is_free(struct hugepage_subpool *spool) @@ -5195,10 +5199,9 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, spin_unlock(dst_ptl); } -int move_hugetlb_page_tables(struct vm_area_struct *vma, - struct vm_area_struct *new_vma, - unsigned long old_addr, unsigned long new_addr, - unsigned long len) +unsigned long move_hugetlb_page_tables(struct vm_area_struct *vma, + struct vm_area_struct *new_vma, unsigned long old_addr, + unsigned long new_addr, unsigned long len) { struct hstate *h = hstate_vma(vma); struct address_space *mapping = vma->vm_file->f_mapping; @@ -5228,6 +5231,8 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, for (; old_addr < old_end; old_addr += sz, new_addr += sz) { const unsigned long offset_to_last_entry = (old_addr | last_addr_mask) - old_addr; + const unsigned long offset_to_last_dst_entry = + (new_addr | last_addr_mask) - new_addr; src_pte = hugetlb_walk(vma, old_addr, sz); if (!src_pte) { @@ -5244,6 +5249,24 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, continue; } + /* + * Avoid entering the destination PMD sharing path while holding + * i_mmap_rwsem for write. Leave destination page table + * population to the fault path, which can share the PMD lazily. + */ + if (sz == PMD_SIZE && want_pmd_share(new_vma, new_addr)) { + const unsigned long unmap_size = + min(old_end - old_addr, + offset_to_last_dst_entry + sz); + + hugetlb_unmap_range(&tlb, vma, old_addr, + old_addr + unmap_size, NULL, + ZAP_FLAG_DROP_MARKER); + old_addr += unmap_size - sz; + new_addr += unmap_size - sz; + continue; + } + dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz); if (!dst_pte) break; @@ -5263,9 +5286,10 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, return len + old_addr - old_end; } -void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, - unsigned long start, unsigned long end, - struct folio *folio, zap_flags_t zap_flags) +static void hugetlb_unmap_range(struct mmu_gather *tlb, + struct vm_area_struct *vma, unsigned long start, + unsigned long end, struct folio *folio, + zap_flags_t zap_flags) { struct mm_struct *mm = vma->vm_mm; const bool folio_provided = !!folio; @@ -5283,13 +5307,6 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, BUG_ON(start & ~huge_page_mask(h)); BUG_ON(end & ~huge_page_mask(h)); - /* - * This is a hugetlb vma, all the pte entries should point - * to huge page. - */ - tlb_change_page_size(tlb, sz); - tlb_start_vma(tlb, vma); - last_addr_mask = hugetlb_mask_last_page(h); address = start; for (; address < end; address += sz) { @@ -5418,6 +5435,15 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, if (folio_provided) break; } +} + +void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long start, unsigned long end, + struct folio *folio, zap_flags_t zap_flags) +{ + tlb_change_page_size(tlb, huge_page_size(hstate_vma(vma))); + tlb_start_vma(tlb, vma); + hugetlb_unmap_range(tlb, vma, start, end, folio, zap_flags); tlb_end_vma(tlb, vma); huge_pmd_unshare_flush(tlb, vma); @@ -6389,6 +6415,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, return -EEXIST; } + VM_WARN_ON_ONCE(size == PMD_SIZE && hugetlb_pmd_shared(dst_pte)); _dst_pte = make_pte_marker(PTE_MARKER_POISONED); set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size); diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 0057fa2a16a8..9aa8af5c8856 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -222,10 +222,12 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr, VM_WARN_ON_ONCE(!PageTail((const struct page *)addr)); /* - * Remap the tail pages as read-only to catch illegal write - * operation to the tail pages. + * Map the shared tail page writable. A lockless get that read + * a non-zero refcount before the remap can still do its + * cmpxchg after it. The refcount is zero, so the cmpxchg + * fails and leaves the page unchanged. */ - entry = mk_pte(walk->vmemmap_tail, PAGE_KERNEL_RO); + entry = mk_pte(walk->vmemmap_tail, PAGE_KERNEL); } list_add(&page->lru, walk->vmemmap_pages); diff --git a/mm/internal.h b/mm/internal.h index 566cbd2dce48..8b1263e81054 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -589,14 +589,16 @@ void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *desc); void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); /** - * sync_with_folio_pmd_zap - sync with concurrent zapping of a folio PMD + * sync_with_folio_pmd_zap - sync with concurrent zapping of folios * @mm: The mm_struct. * @pmdp: Pointer to the pmd that was found to be pmd_none(). * * When we find a pmd_none() while unmapping a folio without holding the PTL, - * zap_huge_pmd() may have cleared the PMD but not yet modified the folio to - * indicate that it's unmapped. Skipping the PMD without synchronization could - * make folio unmapping code assume that unmapping failed. + * zap_huge_pmd() may have cleared the PMD but not yet updated the folio + * mapcount, and zap_empty_pte_table() clears the PMD only after the rmaps + * for the table have been removed, but nothing orders the pmd_none() read + * against later mapcount reads. Skipping the PMD without synchronization + * could make folio unmapping code assume that unmapping failed. * * Wait for concurrent zapping to complete by grabbing the PTL. */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 4b71d010b2af..da91762f6358 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2467,6 +2467,11 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, index += folio_nr_pages(folio); continue; out_unlock: + /* + * The folio may have been unmapped with TTU_BATCH_FLUSH. + * Flush before releasing the lock and our last reference. + */ + try_to_unmap_flush(); folio_unlock(folio); folio_put(folio); goto xa_unlocked; @@ -2477,9 +2482,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, xa_unlocked: /* - * If collapse is successful, flush must be done now before copying. - * If collapse is unsuccessful, does flush actually need to be done? - * Do it anyway, to clear the state. + * Flush before copying the folios, or releasing them in rollback. + * This is a no-op if out_unlock already flushed the batch. */ try_to_unmap_flush(); diff --git a/mm/ksm.c b/mm/ksm.c index dbfb8f98b189..d6f736a1aa2f 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -1463,6 +1464,15 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, dec_mm_counter(mm, MM_ANONPAGES); } + /* Preserve soft-dirty and uffd protection when replacing the page. */ + if (pte_soft_dirty(orig_pte)) + newpte = pte_mksoft_dirty(newpte); + if (pte_uffd(orig_pte)) { + newpte = pte_mkuffd(newpte); + if (userfaultfd_rwp(vma)) + newpte = pte_modify(newpte, PAGE_NONE); + } + flush_cache_page(vma, addr, pte_pfn(ptep_get(ptep))); /* * No need to notify as we are replacing a read only page with another diff --git a/mm/madvise.c b/mm/madvise.c index 75b409e97dbf..e9fa4f71efea 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -158,7 +158,7 @@ static int madvise_update_vma(vm_flags_t new_flags, bool set_new_anon_name = madv_behavior->behavior == __MADV_SET_ANON_VMA_NAME; VMA_ITERATOR(vmi, madv_behavior->mm, range->start); - if (vma_flags_same_mask(&vma->flags, new_vma_flags) && + if (vma_flags_same_pair(&vma->flags, &new_vma_flags) && (!set_new_anon_name || anon_vma_name_eq(anon_vma_name(vma), anon_name))) return 0; @@ -177,7 +177,10 @@ static int madvise_update_vma(vm_flags_t new_flags, /* vm_flags is protected by the mmap_lock held in write mode. */ vma_start_write(vma); - vma->flags = new_vma_flags; + if (set_new_anon_name) + return replace_anon_vma_name(vma, anon_name); + + vma_overwrite_flags(vma, new_vma_flags); /* * If the vma become good for khugepaged to scan, * register it here without waiting a page fault that @@ -186,9 +189,6 @@ static int madvise_update_vma(vm_flags_t new_flags, if (vma_flags_test(&new_vma_flags, VMA_HUGEPAGE_BIT)) khugepaged_enter_vma(vma, vma_flags_to_legacy(new_vma_flags)); - if (set_new_anon_name) - return replace_anon_vma_name(vma, anon_name); - return 0; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a05d9a8baa1b..84f4a8fd4d16 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2457,7 +2457,10 @@ static void high_work_func(struct work_struct *work) struct mem_cgroup *memcg; memcg = container_of(work, struct mem_cgroup, high_work); + if (!css_tryget_online(&memcg->css)) + return; reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); + css_put(&memcg->css); } static void high_irq_work_func(struct irq_work *work) @@ -4463,6 +4466,8 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) page_counter_set_min(&memcg->memory, 0); page_counter_set_low(&memcg->memory, 0); + page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); + page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); zswap_memcg_offline_cleanup(memcg); diff --git a/mm/memory.c b/mm/memory.c index 0fb676dd2bd6..b65b56fe9714 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2036,17 +2036,6 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, } } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); - /* - * Fast path: try to hold the pmd lock and unmap the PTE page. - * - * If the pte lock was released midway (retry case), or if the attempt - * to hold the pmd lock failed, then we need to recheck all pte entries - * to ensure they are still none, thereby preventing the pte entries - * from being repopulated by another thread. - */ - if (can_reclaim_pt && direct_reclaim && addr == end) - direct_reclaim = zap_empty_pte_table(mm, pmd, ptl, &pmdval); - add_mm_rss_vec(mm, rss); lazy_mmu_mode_disable(); @@ -2055,6 +2044,20 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, tlb_flush_mmu_tlbonly(tlb); tlb_flush_rmaps(tlb, vma); } + + /* + * Fast path: try to hold the pmd lock and unmap the PTE page. + * + * If the pte lock was released midway (retry case), or if the attempt + * to hold the pmd lock failed, then we need to recheck all pte entries + * to ensure they are still none, thereby preventing the pte entries + * from being repopulated by another thread. + * + * Clear the PMD after the rmaps for the table have been removed, in case + * an incorrect mapcount is observed after unmapping the folio. + */ + if (can_reclaim_pt && direct_reclaim && addr == end) + direct_reclaim = zap_empty_pte_table(mm, pmd, ptl, &pmdval); pte_unmap_unlock(start_pte, ptl); /* @@ -2101,13 +2104,15 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, addr = next; continue; } + /* We already synchronized with concurrent zapping. */ + if (pmd_none(*pmd)) { + addr = next; + continue; + } /* fall through */ - } else if (details && details->single_folio && - folio_test_pmd_mappable(details->single_folio) && - next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) { - sync_with_folio_pmd_zap(tlb->mm, pmd); - } - if (pmd_none(*pmd)) { + } else if (pmd_none(*pmd)) { + if (details && details->single_folio) + sync_with_folio_pmd_zap(tlb->mm, pmd); addr = next; continue; } @@ -4120,8 +4125,11 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) if (unlikely(unshare)) { if (pte_soft_dirty(vmf->orig_pte)) entry = pte_mksoft_dirty(entry); - if (pte_uffd(vmf->orig_pte)) + if (pte_uffd(vmf->orig_pte)) { entry = pte_mkuffd(entry); + if (userfaultfd_rwp(vma)) + entry = pte_modify(entry, PAGE_NONE); + } } else { entry = maybe_mkwrite(pte_mkdirty(entry), vma); } @@ -4933,6 +4941,34 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, } while (--nr_pages); } +/* + * Get the address range in which a folio might be mapped, assuming its pages + * are mapped consecutively and the folio page at the given idx is mapped at + * the given address. + */ +static void folio_pte_range(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long addr, + unsigned long idx, unsigned long *start, unsigned long *end) +{ + /* Stay within the page table and VMA boundaries. */ + const unsigned long pmd_start = addr & PMD_MASK; + const unsigned long pmd_end = pmd_start + PMD_SIZE; + const unsigned long lower = max(pmd_start, vma->vm_start); + /* + * We cannot use pmd_addr_end(), as it does not clamp reliably. To + * avoid wraparounds, compute using the inclusive end. + */ + const unsigned long upper = min(pmd_end - 1, vma->vm_end - 1) + 1; + const unsigned long offset = idx << PAGE_SHIFT; + + /* + * Avoid start/end addresses wrapping by not adding/subtracting more + * than the lower/upper limits would allow. + */ + *start = addr - min(addr - lower, offset); + *end = addr + min(upper - addr, folio_size(folio) - offset); +} + /* * We enter with either the VMA lock or the mmap_lock held (see * FAULT_FLAG_VMA_LOCK), and pte mapped but not yet locked. @@ -5121,16 +5157,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) address = vmf->address; ptep = vmf->pte; if (folio_test_large(folio) && folio_test_swapcache(folio)) { - int nr = folio_nr_pages(folio); - unsigned long idx = folio_page_idx(folio, page); - unsigned long folio_start = address - idx * PAGE_SIZE; - unsigned long folio_end = folio_start + nr * PAGE_SIZE; + const int nr = folio_nr_pages(folio); + const unsigned long idx = folio_page_idx(folio, page); + unsigned long folio_start, folio_end; pte_t *folio_ptep; pte_t folio_pte; - if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start))) - goto check_folio; - if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end))) + folio_pte_range(folio, vma, address, idx, &folio_start, + &folio_end); + if (unlikely((folio_end - folio_start) >> PAGE_SHIFT != nr)) goto check_folio; folio_ptep = vmf->pte - idx; @@ -6280,16 +6315,11 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru struct folio *folio, pte_t fault_pte, bool ignore_writable, bool pte_write_upgrade) { - int nr = pte_pfn(fault_pte) - folio_pfn(folio); + const unsigned long idx = pte_pfn(fault_pte) - folio_pfn(folio); unsigned long start, end, addr = vmf->address; - unsigned long addr_start = addr - (nr << PAGE_SHIFT); - unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE); pte_t *start_ptep; - /* Stay within the VMA and within the page table. */ - start = max3(addr_start, pt_start, vma->vm_start); - end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE, - vma->vm_end); + folio_pte_range(folio, vma, addr, idx, &start, &end); start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT); /* Restore all PTEs' mapping of the large folio */ @@ -6635,9 +6665,12 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is * NUMA hinting and must still fall through to do_numa_page(). */ - if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte)) - return do_uffd_rwp(vmf); - return do_numa_page(vmf); + if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte)) { + if (!(vmf->flags & FAULT_FLAG_UNSHARE)) + return do_uffd_rwp(vmf); + } else { + return do_numa_page(vmf); + } } spin_lock(vmf->ptl); diff --git a/mm/migrate.c b/mm/migrate.c index 7bdcdb57652f..875079464930 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2461,7 +2461,7 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, mmap_read_lock(mm); for (i = 0; i < nr_pages; i++) { - unsigned long addr = (unsigned long)(*pages); + unsigned long addr = (unsigned long)untagged_addr_remote(mm, *pages); struct vm_area_struct *vma; struct folio_walk fw; struct folio *folio; @@ -2756,6 +2756,7 @@ int migrate_misplaced_folio(struct folio *folio, int node) pg_data_t *pgdat = NODE_DATA(node); int nr_remaining; unsigned int nr_succeeded; + int src_node = folio_nid(folio); LIST_HEAD(migratepages); struct mem_cgroup *memcg = get_mem_cgroup_from_folio(folio); struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); @@ -2770,7 +2771,7 @@ int migrate_misplaced_folio(struct folio *folio, int node) count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_succeeded); count_memcg_events(memcg, NUMA_PAGE_MIGRATE, nr_succeeded); if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) - && !node_is_toptier(folio_nid(folio)) + && !node_is_toptier(src_node) && node_is_toptier(node)) mod_lruvec_state(lruvec, PGPROMOTE_SUCCESS, nr_succeeded); } diff --git a/mm/mlock.c b/mm/mlock.c index 67ae9b18b923..1c70a7829394 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -432,14 +432,14 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, } vma_start_write(vma); - vma_flags_reset_once(vma, &walk_flags); + vma_overwrite_flags(vma, walk_flags); lru_add_drain(); walk_page_range_vma(vma, start, end, &mlock_walk_ops, (void *)lock); lru_add_drain(); if (lock) - vma_flags_reset_once(vma, new_vma_flags); + vma_overwrite_flags(vma, *new_vma_flags); } /* @@ -494,7 +494,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) { /* No work to do, and mlocking twice would be wrong */ vma_start_write(vma); - vma->flags = *new_vma_flags; + vma_overwrite_flags(vma, *new_vma_flags); } else { mlock_vma_pages_range(vma, start, end, new_vma_flags); } diff --git a/mm/mm_init.c b/mm/mm_init.c index 2a0e782a151d..d99af80ec752 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2478,8 +2478,6 @@ void __init memblock_free_pages(unsigned long pfn, unsigned int order) return; } - /* pages were reserved and not allocated */ - clear_page_tag_ref(page); __free_pages_core(page, order, MEMINIT_EARLY); } diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c index 83b7d3ded0dc..834bbf86e004 100644 --- a/mm/mmap_lock.c +++ b/mm/mmap_lock.c @@ -13,6 +13,8 @@ #include #include +#include "internal.h" + EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking); EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned); EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released); @@ -428,21 +430,53 @@ static struct vm_area_struct *lock_next_vma_under_mmap_lock(struct mm_struct *mm return vma; } -struct vm_area_struct *lock_next_vma(struct mm_struct *mm, - struct vma_iterator *vmi, +/* Is there no VMA in [start, last]? */ +static bool vma_range_is_empty(struct mm_struct *mm, unsigned long start, + unsigned long last) +{ + VMA_ITERATOR(vmi, mm, start); + + return !vma_iter_load(&vmi) && vma_iter_end(&vmi) - 1 >= last; +} + +/* Might the VMA tree have been modified since last mm speculation? */ +static bool vma_rcu_iter_maybe_changed(const struct vma_rcu_iter *rvmi) +{ + return (rvmi->mm_wr_seq & 1) || /* Odd implies write-lock. */ + mmap_lock_speculate_retry(rvmi->mm, rvmi->mm_wr_seq); +} + +/* Could the VMA iterator have skipped a VMA due to the tree changing? */ +static bool vma_rcu_iter_may_have_skipped(const struct vma_rcu_iter *rvmi, + unsigned long from, unsigned long to) +{ + return vma_rcu_iter_maybe_changed(rvmi) && + !vma_range_is_empty(rvmi->mm, from, to); +} + +struct vm_area_struct *lock_next_vma(struct vma_rcu_iter *rvmi, unsigned long from_addr) { + struct mm_struct *mm = rvmi->mm; + struct vma_iterator *vmi = rvmi->vmi; struct vm_area_struct *vma; - unsigned int mm_wr_seq; - bool mmap_unlocked; RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu read lock held"); retry: - /* Start mmap_lock speculation in case we need to verify the vma later */ - mmap_unlocked = mmap_lock_speculate_try_begin(mm, &mm_wr_seq); + /* Only trust the iterator's position if the tree cannot have changed. */ + if (vma_rcu_iter_maybe_changed(rvmi)) { + vma_iter_set(vmi, from_addr); + mmap_lock_speculate_try_begin(mm, &rvmi->mm_wr_seq); + } vma = vma_next(vmi); - if (!vma) - return NULL; + if (!vma) { + /* No reason to fallback if nothing left in range. */ + if (!vma_rcu_iter_may_have_skipped(rvmi, from_addr, ULONG_MAX)) + return NULL; + + rcu_read_unlock(); + goto fallback; + } vma = vma_start_read(mm, vma); if (IS_ERR_OR_NULL(vma)) { @@ -471,14 +505,9 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm, * installed ahead of it. Otherwise we might observe a gap that should * not be there. */ - if (from_addr < vma->vm_start) { - /* Verify only if the address space might have changed since vma lookup. */ - if (!mmap_unlocked || mmap_lock_speculate_retry(mm, mm_wr_seq)) { - vma_iter_set(vmi, from_addr); - if (vma != vma_next(vmi)) - goto fallback_unlock; - } - } + if (from_addr < vma->vm_start && + vma_rcu_iter_may_have_skipped(rvmi, from_addr, vma->vm_start - 1)) + goto fallback_unlock; return vma; diff --git a/mm/mprotect.c b/mm/mprotect.c index 754333698c1e..e18d4e12fc34 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -833,7 +833,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, * held in write mode. */ vma_start_write(vma); - vma_flags_reset_once(vma, &new_vma_flags); + vma_overwrite_flags(vma, new_vma_flags); if (vma_wants_manual_pte_write_upgrade(vma)) mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; vma_set_page_prot(vma); diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 499a35473e4f..0694ec4098f6 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -883,6 +883,7 @@ static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc, { struct wb_domain *dom = dtc_dom(dtc); struct bdi_writeback *wb = dtc->wb; + u64 threshold = thresh; u64 wb_thresh; u64 wb_max_thresh; unsigned long numerator, denominator; @@ -894,13 +895,14 @@ static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc, fprop_fraction_percpu(&dom->completions, dtc->wb_completions, &numerator, &denominator); - wb_thresh = (thresh * (100 * BDI_RATIO_SCALE - bdi_min_ratio)) / (100 * BDI_RATIO_SCALE); + wb_thresh = div_u64(threshold * (100 * BDI_RATIO_SCALE - bdi_min_ratio), + 100 * BDI_RATIO_SCALE); wb_thresh *= numerator; wb_thresh = div64_ul(wb_thresh, denominator); wb_min_max_ratio(wb, &wb_min_ratio, &wb_max_ratio); - wb_thresh += (thresh * wb_min_ratio) / (100 * BDI_RATIO_SCALE); + wb_thresh += div_u64(threshold * wb_min_ratio, 100 * BDI_RATIO_SCALE); /* * It's very possible that wb_thresh is close to 0 not because the @@ -916,7 +918,7 @@ static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc, wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 8); } - wb_max_thresh = thresh * wb_max_ratio / (100 * BDI_RATIO_SCALE); + wb_max_thresh = div_u64(threshold * wb_max_ratio, 100 * BDI_RATIO_SCALE); if (wb_thresh > wb_max_thresh) wb_thresh = wb_max_thresh; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a8cac5e01066..eb4f02617f53 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1616,6 +1616,9 @@ void __meminit __free_pages_core(struct page *page, unsigned int order, atomic_long_add(nr_pages, &page_zone(page)->managed_pages); } + /* pages were never allocated */ + clear_page_tag_ref(page); + if (page_contains_unaccepted(page, order)) { if (order == MAX_PAGE_ORDER && __free_unaccepted(page)) return; @@ -2929,7 +2932,7 @@ static bool free_frozen_page_commit(struct zone *zone, * kswapd work again by resetting kswapd_failures. */ if (kswapd_test_hopeless(pgdat) && - next_memory_node(pgdat->node_id) < MAX_NUMNODES) + num_node_state(N_MEMORY) > 1) kswapd_clear_hopeless(pgdat, KSWAPD_CLEAR_HOPELESS_PCP); } return ret; @@ -3238,7 +3241,9 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, * reserves as failing now is worse than failing a * high-order atomic allocation in the future. */ - if (!page && (alloc_flags & (ALLOC_OOM|ALLOC_NON_BLOCK))) + if (!page && + ((alloc_flags & ALLOC_OOM) || + (alloc_flags & ALLOC_MASK_ATOMIC) == ALLOC_MASK_ATOMIC)) page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); if (!page) { @@ -4768,10 +4773,10 @@ static inline struct page * __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, struct alloc_context *ac) { - bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; - bool can_compact = can_direct_reclaim && gfp_compaction_allowed(gfp_mask); - bool nofail = gfp_mask & __GFP_NOFAIL; const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; + bool can_direct_reclaim; + bool can_compact; + bool nofail; struct page *page = NULL; unsigned int alloc_flags; unsigned long did_some_progress; @@ -4786,6 +4791,18 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, bool can_retry_reserves = true; unsigned long alloc_start_time = jiffies; + /* + * Costly __GFP_NORETRY callers have a cheap fallback, so don't stall + * them in reclaim or compaction. __GFP_THISNODE callers are exempt. + */ + if (costly_order && (gfp_mask & __GFP_NORETRY) && + !(gfp_mask & __GFP_THISNODE)) + gfp_mask &= ~__GFP_DIRECT_RECLAIM; + + can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; + can_compact = can_direct_reclaim && gfp_compaction_allowed(gfp_mask); + nofail = gfp_mask & __GFP_NOFAIL; + if (unlikely(nofail)) { /* * We don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM, @@ -4902,8 +4919,11 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, * Reclaim/compaction cannot run, so defrag_mode's strategy * of enforcing ALLOC_NOFRAGMENT cannot be fulfilled. Allow * fallbacks rather than failing the allocation outright. + * Not for __GFP_NORETRY: those have a cheap lower order + * fallback, so failing beats fragmenting. */ if (defrag_mode && (alloc_flags & ALLOC_NOFRAGMENT) && + !(gfp_mask & __GFP_NORETRY) && (gfp_mask & __GFP_KSWAPD_RECLAIM)) { alloc_flags &= ~ALLOC_NOFRAGMENT; goto retry; diff --git a/mm/page_alloc.h b/mm/page_alloc.h index e238ee91ed24..7097ea1805f6 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -32,12 +32,11 @@ #define ALLOC_OOM ALLOC_NO_WATERMARKS #endif -#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access - * to 25% of the min watermark or - * 62.5% if __GFP_HIGH is set. - */ +#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. */ #define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50% - * of the min watermark. + * of the min watermark, or 62.5% if + * the caller cannot block either + * (ALLOC_NON_BLOCK). */ #define ALLOC_CPUSET 0x40 /* check for correct cpuset */ #define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ @@ -58,7 +57,10 @@ #define ALLOC_NO_CODETAG 0x1000 /* Flags that allow allocations below the min watermark. */ -#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) +#define ALLOC_RESERVES (ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) + +/* Flags that mean GFP_ATOMIC */ +#define ALLOC_MASK_ATOMIC (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE) /* * Structure for holding the mostly immutable allocation parameters passed diff --git a/mm/shmem.c b/mm/shmem.c index af21a89abb0f..3c0c7edd2020 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1742,6 +1742,10 @@ static int shmem_unuse_inode(struct inode *inode, unsigned int type) if (ret < 0) break; + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } start = indices[folio_batch_count(&fbatch) - 1]; } while (true); diff --git a/mm/swapfile.c b/mm/swapfile.c index 254ce86fa923..0f72a5b23a3a 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2689,6 +2689,8 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, pmd = pmd_offset(pud, addr); do { cond_resched(); + if (signal_pending(current)) + return -ERESTARTSYS; next = pmd_addr_end(addr, end); ret = unuse_pte_range(vma, pmd, addr, next, type); if (ret) @@ -2922,7 +2924,7 @@ static int try_to_unuse(unsigned int type) if (swap_usage_in_pages(si)) { if (!signal_pending(current)) goto retry; - return -EINTR; + return -ERESTARTSYS; } success: diff --git a/mm/vma.c b/mm/vma.c index 1be613044b4a..d416ba58eb4c 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -538,6 +538,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, return err; } + vma_start_write(vma); new = vm_area_dup(vma); if (!new) return -ENOMEM; @@ -568,7 +569,6 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new->vm_ops && new->vm_ops->open) new->vm_ops->open(new); - vma_start_write(vma); vma_start_write(new); init_vma_prep(&vp, vma); @@ -783,6 +783,27 @@ static void vmg_adjust_set_range(struct vma_merge_struct *vmg) } } +static void propagate_sticky_flags(struct vma_merge_struct *vmg, + const struct vm_area_struct *vma) +{ + const vma_flags_t sticky = + vma_flags_and_mask(&vma->flags, VMA_STICKY_FLAGS); + + /* Write lock prevents concurrent atomic VMA flag updates. */ + vma_assert_write_locked(vma); + vma_flags_set_mask(&vmg->vma_flags, sticky); +} + +static void commit_sticky_flags(struct vma_merge_struct *vmg) +{ + const vma_flags_t sticky = + vma_flags_and_mask(&vmg->vma_flags, VMA_STICKY_FLAGS); + struct vm_area_struct *target = vmg->target; + + vma_assert_write_locked(target); + vma_set_flags_mask(target, sticky); +} + /* * Actually perform the VMA merge operation. * @@ -818,6 +839,8 @@ static int commit_merge(struct vma_merge_struct *vmg) return -ENOMEM; vma_prepare(&vp); + /* Must be made visible before vma_complete() for rmap walkers. */ + commit_sticky_flags(vmg); /* * THP pages may need to do additional splits if we increase * middle->vm_start. @@ -857,8 +880,8 @@ static bool can_merge_remove_vma(struct vm_area_struct *vma) * merge. * * As part of this operation, if a merge occurs, the @vmg object will have its - * vma, start, end, and pgoff fields modified to execute the merge. Subsequent - * calls to this function should reset these fields. + * vma, start, end, pgoff and vma_flags fields modified to execute the merge. + * Subsequent calls to this function should reset these fields. * * Returns: The merged VMA if merge succeeds, or NULL otherwise. * @@ -872,8 +895,6 @@ static bool can_merge_remove_vma(struct vm_area_struct *vma) static __must_check struct vm_area_struct *vma_merge_existing_range( struct vma_merge_struct *vmg) { - vma_flags_t sticky_flags = vma_flags_and_mask(&vmg->vma_flags, - VMA_STICKY_FLAGS); struct vm_area_struct *middle = vmg->middle; struct vm_area_struct *prev = vmg->prev; struct vm_area_struct *next; @@ -966,24 +987,18 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( /* No matter what happens, we will be adjusting middle. */ vma_start_write(middle); + propagate_sticky_flags(vmg, middle); if (merge_right) { - vma_flags_t next_sticky; - vma_start_write(next); vmg->target = next; - next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS); - vma_flags_set_mask(&sticky_flags, next_sticky); + propagate_sticky_flags(vmg, next); } if (merge_left) { - vma_flags_t prev_sticky; - vma_start_write(prev); vmg->target = prev; - - prev_sticky = vma_flags_and_mask(&prev->flags, VMA_STICKY_FLAGS); - vma_flags_set_mask(&sticky_flags, prev_sticky); + propagate_sticky_flags(vmg, prev); } if (merge_both) { @@ -1054,7 +1069,6 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (err || commit_merge(vmg)) goto abort; - vma_set_flags_mask(vmg->target, sticky_flags); khugepaged_enter_vma(vmg->target, vmg->vm_flags); vmg->state = VMA_MERGE_SUCCESS; return vmg->target; @@ -1127,6 +1141,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) struct vm_area_struct *next = vmg->next; unsigned long end = vmg->end; bool can_merge_left, can_merge_right; + bool move_iter_back = false; mmap_assert_write_locked(vmg->mm); VM_WARN_ON_VMG(vmg->middle, vmg); @@ -1168,23 +1183,30 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) vmg->end = end; /* In expand-only case we are already positioned at prev. */ - if (!vmg->just_expand) { - /* Equivalent to going to the previous range. */ - vma_prev(vmg->vmi); - } + if (!vmg->just_expand) + move_iter_back = true; } + if (!can_merge_left && !can_merge_right) + return NULL; + + if (move_iter_back) + vma_prev(vmg->vmi); + /* * Now try to expand adjacent VMA(s). This takes care of removing the * following VMA if we have VMAs on both sides. */ - if (vmg->target && !vma_expand(vmg)) { - khugepaged_enter_vma(vmg->target, vmg->vm_flags); - vmg->state = VMA_MERGE_SUCCESS; - return vmg->target; + if (vma_expand(vmg)) { + /* Error occurred, restore iterator. */ + if (move_iter_back) + vma_iter_next_range(vmg->vmi); + return NULL; } - return NULL; + khugepaged_enter_vma(vmg->target, vmg->vm_flags); + vmg->state = VMA_MERGE_SUCCESS; + return vmg->target; } /* @@ -1241,16 +1263,11 @@ int vma_expand(struct vma_merge_struct *vmg) struct vm_area_struct *target = vmg->target; struct vm_area_struct *next = vmg->next; bool remove_next = false; - vma_flags_t sticky_flags = - vma_flags_and_mask(&vmg->vma_flags, VMA_STICKY_FLAGS); - vma_flags_t target_sticky; int ret = 0; mmap_assert_write_locked(vmg->mm); vma_start_write(target); - target_sticky = vma_flags_and_mask(&target->flags, VMA_STICKY_FLAGS); - if (next && target != next && vmg->end == next->vm_end) remove_next = true; @@ -1265,7 +1282,7 @@ int vma_expand(struct vma_merge_struct *vmg) VM_WARN_ON_VMG(target->vm_start < vmg->start || target->vm_end > vmg->end, vmg); - vma_flags_set_mask(&sticky_flags, target_sticky); + propagate_sticky_flags(vmg, target); /* * If we are removing the next VMA or copying from a VMA @@ -1279,21 +1296,17 @@ int vma_expand(struct vma_merge_struct *vmg) if (!ret && vmg->copied_from) ret = dup_anon_vma(target, vmg->copied_from, &anon_dup); if (ret) - return ret; + goto nomem; if (remove_next) { - vma_flags_t next_sticky; - vma_start_write(next); vmg->__remove_next = true; - next_sticky = vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS); - vma_flags_set_mask(&sticky_flags, next_sticky); + propagate_sticky_flags(vmg, next); } if (commit_merge(vmg)) goto nomem; - vma_set_flags_mask(target, sticky_flags); return 0; nomem: @@ -1851,12 +1864,25 @@ struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long delta) { - VMG_VMA_STATE(vmg, vmi, vma, vma, vma->vm_end, vma->vm_end + delta); + const unsigned long start = + vma_can_merge(vma) ? vma->vm_end : vma->vm_start; + VMG_VMA_STATE(vmg, vmi, vma, vma, start, vma->vm_end + delta); - vmg.next = vma_iter_next_rewind(vmi, NULL); vmg.middle = NULL; /* We use the VMA to populate VMG fields only. */ - return vma_merge_new_range(&vmg); + if (vma_can_merge(vma)) { + vmg.next = vma_iter_next_rewind(vmi, NULL); + return vma_merge_new_range(&vmg); + } + + /* Mixed mappings are permitted to be expanded, not merged. */ + VM_WARN_ON_ONCE(vma_is_fixed_mapping(vma)); + VM_WARN_ON_ONCE(!vma_test(vma, VMA_MIXEDMAP_BIT)); + vma_prev(vmi); /* Reposition to start of VMA. */ + vmg.target = vma; + if (vma_expand(&vmg)) + return NULL; + return vma; } void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb) diff --git a/mm/vma_init.c b/mm/vma_init.c index baa7e82f47e3..d661f229a734 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -127,6 +127,9 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) { struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); + /* Must hold a write lock to stabilise atomic VMA flags. */ + vma_assert_write_locked(orig); + if (!new) return NULL; diff --git a/tools/testing/selftests/mm/khugepaged_race.c b/tools/testing/selftests/mm/khugepaged_race.c index abb0678522cd..1a27cb39745b 100644 --- a/tools/testing/selftests/mm/khugepaged_race.c +++ b/tools/testing/selftests/mm/khugepaged_race.c @@ -8,6 +8,7 @@ #define _GNU_SOURCE #include #include +#include #include #include #include diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index 7dd2933417c1..6def490186a9 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -22,9 +22,12 @@ FIXTURE(merge) struct procmap_fd procmap; }; +#define FAR_PAGES 30 +#define CARVEOUT_PAGES (3 * FAR_PAGES) + static char *map_carveout(unsigned int page_size) { - return mmap(NULL, 30 * page_size, PROT_NONE, + return mmap(NULL, CARVEOUT_PAGES * page_size, PROT_NONE, MAP_ANON | MAP_PRIVATE, -1, 0); } @@ -73,7 +76,7 @@ FIXTURE_SETUP(merge) FIXTURE_TEARDOWN(merge) { - ASSERT_EQ(munmap(self->carveout, 30 * self->page_size), 0); + ASSERT_EQ(munmap(self->carveout, CARVEOUT_PAGES * self->page_size), 0); /* May fail for parent of forked process. */ close_procmap(&self->procmap); /* @@ -115,7 +118,7 @@ FIXTURE_SETUP(merge_with_fork) FIXTURE_TEARDOWN(merge_with_fork) { - ASSERT_EQ(munmap(self->carveout, 30 * self->page_size), 0); + ASSERT_EQ(munmap(self->carveout, CARVEOUT_PAGES * self->page_size), 0); ASSERT_EQ(close_procmap(&self->procmap), 0); /* See above. */ prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); @@ -660,7 +663,7 @@ TEST_F(merge, mremap_unfaulted_to_faulted) /* Offset ptr2 further away. */ ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + FAR_PAGES * page_size); ASSERT_NE(ptr2, MAP_FAILED); /* @@ -721,7 +724,7 @@ TEST_F(merge, mremap_unfaulted_behind_faulted) /* Offset ptr2 further away. */ ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + FAR_PAGES * page_size); ASSERT_NE(ptr2, MAP_FAILED); /* @@ -785,12 +788,12 @@ TEST_F(merge, mremap_unfaulted_between_faulted) /* Offset ptr3 further away. */ ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 2000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + 2 * FAR_PAGES * page_size); ASSERT_NE(ptr3, MAP_FAILED); /* Offset ptr2 further away. */ ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + FAR_PAGES * page_size); ASSERT_NE(ptr2, MAP_FAILED); /* @@ -870,13 +873,13 @@ TEST_F(merge, mremap_unfaulted_between_faulted_unfaulted) /* Offset ptr3 further away. */ ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 2000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + 2 * FAR_PAGES * page_size); ASSERT_NE(ptr3, MAP_FAILED); /* Offset ptr2 further away. */ ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + FAR_PAGES * page_size); ASSERT_NE(ptr2, MAP_FAILED); /* @@ -1035,11 +1038,11 @@ TEST_F(merge, mremap_correct_placed_faulted) */ ptr3 = &ptr[10 * page_size]; ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 2000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + 2 * FAR_PAGES * page_size); ASSERT_NE(ptr3, MAP_FAILED); ptr2 = &ptr[5 * page_size]; ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + FAR_PAGES * page_size); ASSERT_NE(ptr2, MAP_FAILED); /* @@ -1075,7 +1078,7 @@ TEST_F(merge, mremap_correct_placed_faulted) * ptr2 \ ptr \ ptr3 */ ptr = sys_mremap(ptr, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr + FAR_PAGES * page_size); ASSERT_NE(ptr, MAP_FAILED); /* @@ -1110,7 +1113,7 @@ TEST_F(merge, mremap_correct_placed_faulted) * ptr2 \ ptr \ ptr3 */ ptr = sys_mremap(ptr, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr + FAR_PAGES * page_size); ASSERT_NE(ptr, MAP_FAILED); /* @@ -1168,7 +1171,7 @@ TEST_F(merge, mremap_correct_placed_faulted) * ptr ptr3 \ ptr2 */ ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr2 + FAR_PAGES * page_size); ASSERT_NE(ptr2, MAP_FAILED); /* @@ -1203,7 +1206,7 @@ TEST_F(merge, mremap_correct_placed_faulted) * ptr \ ptr3 */ ptr3 = sys_mremap(ptr3, 5 * page_size, 5 * page_size, - MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + page_size * 1000); + MREMAP_MAYMOVE | MREMAP_FIXED, ptr3 + FAR_PAGES * page_size); ASSERT_NE(ptr3, MAP_FAILED); /* diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index f449129ea97a..dc4927d26296 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -366,6 +366,9 @@ enum { #define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT) +/* These flags can be updated atomically via VMA/mmap read lock. */ +#define VMA_ATOMIC_SET_ALLOWED mk_vma_flags(VMA_MAYBE_GUARD_BIT) + #define RLIMIT_STACK 3 /* max stack size */ #define RLIMIT_MEMLOCK 8 /* max locked-in-memory address space */ @@ -895,22 +898,6 @@ static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_init(vma, flags); } -static inline void vma_flags_reset_once(struct vm_area_struct *vma, - vma_flags_t *flags) -{ - const unsigned long word = flags->__vma_flags[0]; - - /* It is assumed only the first system word must be written once. */ - vma_flags_overwrite_word_once(&vma->flags, word); - /* The remainder can be copied normally. */ - if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { - unsigned long *dst = &vma->flags.__vma_flags[1]; - const unsigned long *src = &flags->__vma_flags[1]; - - bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); - } -} - static inline void vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags) { @@ -1037,27 +1024,24 @@ static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, return dst; } +/* + * Determine if flags and flags_other have precisely the same flags set, + * excluding atomic VMA flags. + */ static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, - const vma_flags_t *flags_other) + const vma_flags_t *flags_other) { - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other->__vma_flags; + vma_flags_t diff = vma_flags_diff_pair(flags, flags_other); - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); + /* + * Atomic VMA flags may be set concurrently under the VMA read lock, so + * are disregarded for equality testing purposes. + */ + vma_flags_clear_mask(&diff, VMA_ATOMIC_SET_ALLOWED); + + return vma_flags_empty(&diff); } -static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, - vma_flags_t flags_other) -{ - const unsigned long *bitmap = flags->__vma_flags; - const unsigned long *bitmap_other = flags_other.__vma_flags; - - return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); -} - -#define vma_flags_same(flags, ...) \ - vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) - static __always_inline bool vma_test(const struct vm_area_struct *vma, vma_flag_t bit) { @@ -1106,6 +1090,36 @@ static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, #define vma_clear_flags(vma, ...) \ vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) +/* + * Overwrite the VMA's flags with new ones, preserving atomic VMA flags in the + * process. The write is done once and once only. + */ +static inline void vma_overwrite_flags(struct vm_area_struct *vma, + vma_flags_t new_flags) +{ + vma_flags_t atomic; + + vma_assert_write_locked(vma); + + /* + * Preserve the VMA's atomic flags, which are kept stable by the VMA + * write lock. + */ + atomic = vma_flags_and_mask(&vma->flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_clear_mask(&new_flags, VMA_ATOMIC_SET_ALLOWED); + vma_flags_set_mask(&new_flags, atomic); + + /* It is assumed only the first system word must be written once. */ + vma_flags_overwrite_word_once(&vma->flags, new_flags.__vma_flags[0]); + /* The remainder can be copied normally. */ + if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { + unsigned long *dst = &vma->flags.__vma_flags[1]; + const unsigned long *src = &new_flags.__vma_flags[1]; + + bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); + } +} + static __always_inline bool vma_desc_test(const struct vm_area_desc *desc, vma_flag_t bit) { @@ -1198,7 +1212,7 @@ static inline void vma_assert_detached(struct vm_area_struct *vma) WARN_ON_ONCE(vma_is_attached(vma)); } -static inline void vma_assert_write_locked(struct vm_area_struct *); +static inline void vma_assert_write_locked(const struct vm_area_struct *); static inline void vma_mark_attached(struct vm_area_struct *vma) { vma_assert_write_locked(vma); @@ -1727,3 +1741,8 @@ static inline bool vma_flags_can_merge(const vma_flags_t *flags) */ return !vma_flags_is_hugetlb(flags); } + +static inline bool vma_can_merge(const struct vm_area_struct *vma) +{ + return vma_flags_can_merge(&vma->flags); +} diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index f786a585caf0..8d030a3ace84 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -320,7 +320,7 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma) { } -static inline void vma_assert_write_locked(struct vm_area_struct *vma) +static inline void vma_assert_write_locked(const struct vm_area_struct *vma) { } diff --git a/tools/testing/vma/shared.h b/tools/testing/vma/shared.h index 97cd7a679dc1..646edf367d7c 100644 --- a/tools/testing/vma/shared.h +++ b/tools/testing/vma/shared.h @@ -51,17 +51,21 @@ __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ } while (0) -#define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) \ - ASSERT_TRUE(vma_flags_same_mask((_flags), (_flags_other))) +#define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) do { \ + const vma_flags_t __flags_other = (_flags_other); \ + ASSERT_TRUE(vma_flags_same_pair((_flags), &__flags_other)); \ + } while (0) -#define ASSERT_FLAGS_NOT_SAME_MASK(_flags, _flags_other) \ - ASSERT_FALSE(vma_flags_same_mask((_flags), (_flags_other))) +#define ASSERT_FLAGS_NOT_SAME_MASK(_flags, _flags_other) do { \ + const vma_flags_t __flags_other = (_flags_other); \ + ASSERT_FALSE(vma_flags_same_pair((_flags), &__flags_other)); \ + } while (0) #define ASSERT_FLAGS_SAME(_flags, ...) \ - ASSERT_TRUE(vma_flags_same(_flags, __VA_ARGS__)) + ASSERT_FLAGS_SAME_MASK(_flags, mk_vma_flags(__VA_ARGS__)) #define ASSERT_FLAGS_NOT_SAME(_flags, ...) \ - ASSERT_FALSE(vma_flags_same(_flags, __VA_ARGS__)) + ASSERT_FLAGS_NOT_SAME_MASK(_flags, mk_vma_flags(__VA_ARGS__)) #define ASSERT_FLAGS_EMPTY(_flags) \ ASSERT_TRUE(vma_flags_empty(_flags))