mirror of
https://github.com/torvalds/linux.git
synced 2026-08-01 12:11:59 +02:00
Merge 8be976a093 ("mm: multi-gen LRU: design doc") into android-mainline
Steps on the way to 6.1-rc1 Resolves conflicts in: fs/exec.c Signed-off-by: Greg Kroah-Hartman <gregkh@google.com> Change-Id: Ia649390f891780ed03530adcf0fac9560635e58f
This commit is contained in:
Greg Kroah-Hartman
commit
15f46fd6cb
|
|
@ -32,6 +32,7 @@ the Linux memory management.
|
|||
idle_page_tracking
|
||||
ksm
|
||||
memory-hotplug
|
||||
multigen_lru
|
||||
nommu-mmap
|
||||
numa_memory_policy
|
||||
numaperf
|
||||
|
|
|
|||
162
Documentation/admin-guide/mm/multigen_lru.rst
Normal file
162
Documentation/admin-guide/mm/multigen_lru.rst
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
.. SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
=============
|
||||
Multi-Gen LRU
|
||||
=============
|
||||
The multi-gen LRU is an alternative LRU implementation that optimizes
|
||||
page reclaim and improves performance under memory pressure. Page
|
||||
reclaim decides the kernel's caching policy and ability to overcommit
|
||||
memory. It directly impacts the kswapd CPU usage and RAM efficiency.
|
||||
|
||||
Quick start
|
||||
===========
|
||||
Build the kernel with the following configurations.
|
||||
|
||||
* ``CONFIG_LRU_GEN=y``
|
||||
* ``CONFIG_LRU_GEN_ENABLED=y``
|
||||
|
||||
All set!
|
||||
|
||||
Runtime options
|
||||
===============
|
||||
``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the
|
||||
following subsections.
|
||||
|
||||
Kill switch
|
||||
-----------
|
||||
``enabled`` accepts different values to enable or disable the
|
||||
following components. Its default value depends on
|
||||
``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled
|
||||
unless some of them have unforeseen side effects. Writing to
|
||||
``enabled`` has no effect when a component is not supported by the
|
||||
hardware, and valid values will be accepted even when the main switch
|
||||
is off.
|
||||
|
||||
====== ===============================================================
|
||||
Values Components
|
||||
====== ===============================================================
|
||||
0x0001 The main switch for the multi-gen LRU.
|
||||
0x0002 Clearing the accessed bit in leaf page table entries in large
|
||||
batches, when MMU sets it (e.g., on x86). This behavior can
|
||||
theoretically worsen lock contention (mmap_lock). If it is
|
||||
disabled, the multi-gen LRU will suffer a minor performance
|
||||
degradation for workloads that contiguously map hot pages,
|
||||
whose accessed bits can be otherwise cleared by fewer larger
|
||||
batches.
|
||||
0x0004 Clearing the accessed bit in non-leaf page table entries as
|
||||
well, when MMU sets it (e.g., on x86). This behavior was not
|
||||
verified on x86 varieties other than Intel and AMD. If it is
|
||||
disabled, the multi-gen LRU will suffer a negligible
|
||||
performance degradation.
|
||||
[yYnN] Apply to all the components above.
|
||||
====== ===============================================================
|
||||
|
||||
E.g.,
|
||||
::
|
||||
|
||||
echo y >/sys/kernel/mm/lru_gen/enabled
|
||||
cat /sys/kernel/mm/lru_gen/enabled
|
||||
0x0007
|
||||
echo 5 >/sys/kernel/mm/lru_gen/enabled
|
||||
cat /sys/kernel/mm/lru_gen/enabled
|
||||
0x0005
|
||||
|
||||
Thrashing prevention
|
||||
--------------------
|
||||
Personal computers are more sensitive to thrashing because it can
|
||||
cause janks (lags when rendering UI) and negatively impact user
|
||||
experience. The multi-gen LRU offers thrashing prevention to the
|
||||
majority of laptop and desktop users who do not have ``oomd``.
|
||||
|
||||
Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of
|
||||
``N`` milliseconds from getting evicted. The OOM killer is triggered
|
||||
if this working set cannot be kept in memory. In other words, this
|
||||
option works as an adjustable pressure relief valve, and when open, it
|
||||
terminates applications that are hopefully not being used.
|
||||
|
||||
Based on the average human detectable lag (~100ms), ``N=1000`` usually
|
||||
eliminates intolerable janks due to thrashing. Larger values like
|
||||
``N=3000`` make janks less noticeable at the risk of premature OOM
|
||||
kills.
|
||||
|
||||
The default value ``0`` means disabled.
|
||||
|
||||
Experimental features
|
||||
=====================
|
||||
``/sys/kernel/debug/lru_gen`` accepts commands described in the
|
||||
following subsections. Multiple command lines are supported, so does
|
||||
concatenation with delimiters ``,`` and ``;``.
|
||||
|
||||
``/sys/kernel/debug/lru_gen_full`` provides additional stats for
|
||||
debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from
|
||||
evicted generations in this file.
|
||||
|
||||
Working set estimation
|
||||
----------------------
|
||||
Working set estimation measures how much memory an application needs
|
||||
in a given time interval, and it is usually done with little impact on
|
||||
the performance of the application. E.g., data centers want to
|
||||
optimize job scheduling (bin packing) to improve memory utilizations.
|
||||
When a new job comes in, the job scheduler needs to find out whether
|
||||
each server it manages can allocate a certain amount of memory for
|
||||
this new job before it can pick a candidate. To do so, the job
|
||||
scheduler needs to estimate the working sets of the existing jobs.
|
||||
|
||||
When it is read, ``lru_gen`` returns a histogram of numbers of pages
|
||||
accessed over different time intervals for each memcg and node.
|
||||
``MAX_NR_GENS`` decides the number of bins for each histogram. The
|
||||
histograms are noncumulative.
|
||||
::
|
||||
|
||||
memcg memcg_id memcg_path
|
||||
node node_id
|
||||
min_gen_nr age_in_ms nr_anon_pages nr_file_pages
|
||||
...
|
||||
max_gen_nr age_in_ms nr_anon_pages nr_file_pages
|
||||
|
||||
Each bin contains an estimated number of pages that have been accessed
|
||||
within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages
|
||||
and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of
|
||||
the former is the largest and that of the latter is the smallest.
|
||||
|
||||
Users can write the following command to ``lru_gen`` to create a new
|
||||
generation ``max_gen_nr+1``:
|
||||
|
||||
``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]``
|
||||
|
||||
``can_swap`` defaults to the swap setting and, if it is set to ``1``,
|
||||
it forces the scan of anon pages when swap is off, and vice versa.
|
||||
``force_scan`` defaults to ``1`` and, if it is set to ``0``, it
|
||||
employs heuristics to reduce the overhead, which is likely to reduce
|
||||
the coverage as well.
|
||||
|
||||
A typical use case is that a job scheduler runs this command at a
|
||||
certain time interval to create new generations, and it ranks the
|
||||
servers it manages based on the sizes of their cold pages defined by
|
||||
this time interval.
|
||||
|
||||
Proactive reclaim
|
||||
-----------------
|
||||
Proactive reclaim induces page reclaim when there is no memory
|
||||
pressure. It usually targets cold pages only. E.g., when a new job
|
||||
comes in, the job scheduler wants to proactively reclaim cold pages on
|
||||
the server it selected, to improve the chance of successfully landing
|
||||
this new job.
|
||||
|
||||
Users can write the following command to ``lru_gen`` to evict
|
||||
generations less than or equal to ``min_gen_nr``.
|
||||
|
||||
``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]``
|
||||
|
||||
``min_gen_nr`` should be less than ``max_gen_nr-1``, since
|
||||
``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to
|
||||
the active list) and therefore cannot be evicted. ``swappiness``
|
||||
overrides the default value in ``/proc/sys/vm/swappiness``.
|
||||
``nr_to_reclaim`` limits the number of pages to evict.
|
||||
|
||||
A typical use case is that a job scheduler runs this command before it
|
||||
tries to land a new job on a server. If it fails to materialize enough
|
||||
cold pages because of the overestimation, it retries on the next
|
||||
server according to the ranking result obtained from the working set
|
||||
estimation step. This less forceful approach limits the impacts on the
|
||||
existing jobs.
|
||||
|
|
@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose.
|
|||
ksm
|
||||
memory-model
|
||||
mmu_notifier
|
||||
multigen_lru
|
||||
numa
|
||||
overcommit-accounting
|
||||
page_migration
|
||||
|
|
|
|||
159
Documentation/mm/multigen_lru.rst
Normal file
159
Documentation/mm/multigen_lru.rst
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
.. SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
=============
|
||||
Multi-Gen LRU
|
||||
=============
|
||||
The multi-gen LRU is an alternative LRU implementation that optimizes
|
||||
page reclaim and improves performance under memory pressure. Page
|
||||
reclaim decides the kernel's caching policy and ability to overcommit
|
||||
memory. It directly impacts the kswapd CPU usage and RAM efficiency.
|
||||
|
||||
Design overview
|
||||
===============
|
||||
Objectives
|
||||
----------
|
||||
The design objectives are:
|
||||
|
||||
* Good representation of access recency
|
||||
* Try to profit from spatial locality
|
||||
* Fast paths to make obvious choices
|
||||
* Simple self-correcting heuristics
|
||||
|
||||
The representation of access recency is at the core of all LRU
|
||||
implementations. In the multi-gen LRU, each generation represents a
|
||||
group of pages with similar access recency. Generations establish a
|
||||
(time-based) common frame of reference and therefore help make better
|
||||
choices, e.g., between different memcgs on a computer or different
|
||||
computers in a data center (for job scheduling).
|
||||
|
||||
Exploiting spatial locality improves efficiency when gathering the
|
||||
accessed bit. A rmap walk targets a single page and does not try to
|
||||
profit from discovering a young PTE. A page table walk can sweep all
|
||||
the young PTEs in an address space, but the address space can be too
|
||||
sparse to make a profit. The key is to optimize both methods and use
|
||||
them in combination.
|
||||
|
||||
Fast paths reduce code complexity and runtime overhead. Unmapped pages
|
||||
do not require TLB flushes; clean pages do not require writeback.
|
||||
These facts are only helpful when other conditions, e.g., access
|
||||
recency, are similar. With generations as a common frame of reference,
|
||||
additional factors stand out. But obvious choices might not be good
|
||||
choices; thus self-correction is necessary.
|
||||
|
||||
The benefits of simple self-correcting heuristics are self-evident.
|
||||
Again, with generations as a common frame of reference, this becomes
|
||||
attainable. Specifically, pages in the same generation can be
|
||||
categorized based on additional factors, and a feedback loop can
|
||||
statistically compare the refault percentages across those categories
|
||||
and infer which of them are better choices.
|
||||
|
||||
Assumptions
|
||||
-----------
|
||||
The protection of hot pages and the selection of cold pages are based
|
||||
on page access channels and patterns. There are two access channels:
|
||||
|
||||
* Accesses through page tables
|
||||
* Accesses through file descriptors
|
||||
|
||||
The protection of the former channel is by design stronger because:
|
||||
|
||||
1. The uncertainty in determining the access patterns of the former
|
||||
channel is higher due to the approximation of the accessed bit.
|
||||
2. The cost of evicting the former channel is higher due to the TLB
|
||||
flushes required and the likelihood of encountering the dirty bit.
|
||||
3. The penalty of underprotecting the former channel is higher because
|
||||
applications usually do not prepare themselves for major page
|
||||
faults like they do for blocked I/O. E.g., GUI applications
|
||||
commonly use dedicated I/O threads to avoid blocking rendering
|
||||
threads.
|
||||
|
||||
There are also two access patterns:
|
||||
|
||||
* Accesses exhibiting temporal locality
|
||||
* Accesses not exhibiting temporal locality
|
||||
|
||||
For the reasons listed above, the former channel is assumed to follow
|
||||
the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is
|
||||
present, and the latter channel is assumed to follow the latter
|
||||
pattern unless outlying refaults have been observed.
|
||||
|
||||
Workflow overview
|
||||
=================
|
||||
Evictable pages are divided into multiple generations for each
|
||||
``lruvec``. The youngest generation number is stored in
|
||||
``lrugen->max_seq`` for both anon and file types as they are aged on
|
||||
an equal footing. The oldest generation numbers are stored in
|
||||
``lrugen->min_seq[]`` separately for anon and file types as clean file
|
||||
pages can be evicted regardless of swap constraints. These three
|
||||
variables are monotonically increasing.
|
||||
|
||||
Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)``
|
||||
bits in order to fit into the gen counter in ``folio->flags``. Each
|
||||
truncated generation number is an index to ``lrugen->lists[]``. The
|
||||
sliding window technique is used to track at least ``MIN_NR_GENS`` and
|
||||
at most ``MAX_NR_GENS`` generations. The gen counter stores a value
|
||||
within ``[1, MAX_NR_GENS]`` while a page is on one of
|
||||
``lrugen->lists[]``; otherwise it stores zero.
|
||||
|
||||
Each generation is divided into multiple tiers. A page accessed ``N``
|
||||
times through file descriptors is in tier ``order_base_2(N)``. Unlike
|
||||
generations, tiers do not have dedicated ``lrugen->lists[]``. In
|
||||
contrast to moving across generations, which requires the LRU lock,
|
||||
moving across tiers only involves atomic operations on
|
||||
``folio->flags`` and therefore has a negligible cost. A feedback loop
|
||||
modeled after the PID controller monitors refaults over all the tiers
|
||||
from anon and file types and decides which tiers from which types to
|
||||
evict or protect.
|
||||
|
||||
There are two conceptually independent procedures: the aging and the
|
||||
eviction. They form a closed-loop system, i.e., the page reclaim.
|
||||
|
||||
Aging
|
||||
-----
|
||||
The aging produces young generations. Given an ``lruvec``, it
|
||||
increments ``max_seq`` when ``max_seq-min_seq+1`` approaches
|
||||
``MIN_NR_GENS``. The aging promotes hot pages to the youngest
|
||||
generation when it finds them accessed through page tables; the
|
||||
demotion of cold pages happens consequently when it increments
|
||||
``max_seq``. The aging uses page table walks and rmap walks to find
|
||||
young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list``
|
||||
and calls ``walk_page_range()`` with each ``mm_struct`` on this list
|
||||
to scan PTEs, and after each iteration, it increments ``max_seq``. For
|
||||
the latter, when the eviction walks the rmap and finds a young PTE,
|
||||
the aging scans the adjacent PTEs. For both, on finding a young PTE,
|
||||
the aging clears the accessed bit and updates the gen counter of the
|
||||
page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``.
|
||||
|
||||
Eviction
|
||||
--------
|
||||
The eviction consumes old generations. Given an ``lruvec``, it
|
||||
increments ``min_seq`` when ``lrugen->lists[]`` indexed by
|
||||
``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
|
||||
evict from, it first compares ``min_seq[]`` to select the older type.
|
||||
If both types are equally old, it selects the one whose first tier has
|
||||
a lower refault percentage. The first tier contains single-use
|
||||
unmapped clean pages, which are the best bet. The eviction sorts a
|
||||
page according to its gen counter if the aging has found this page
|
||||
accessed through page tables and updated its gen counter. It also
|
||||
moves a page to the next generation, i.e., ``min_seq+1``, if this page
|
||||
was accessed multiple times through file descriptors and the feedback
|
||||
loop has detected outlying refaults from the tier this page is in. To
|
||||
this end, the feedback loop uses the first tier as the baseline, for
|
||||
the reason stated earlier.
|
||||
|
||||
Summary
|
||||
-------
|
||||
The multi-gen LRU can be disassembled into the following parts:
|
||||
|
||||
* Generations
|
||||
* Rmap walks
|
||||
* Page table walks
|
||||
* Bloom filters
|
||||
* PID controller
|
||||
|
||||
The aging and the eviction form a producer-consumer model;
|
||||
specifically, the latter drives the former by the sliding window over
|
||||
generations. Within the aging, rmap walks drive page table walks by
|
||||
inserting hot densely populated page tables to the Bloom filters.
|
||||
Within the eviction, the PID controller uses refaults as the feedback
|
||||
to select types to evict and tiers to protect.
|
||||
|
|
@ -1010,6 +1010,7 @@ static int exec_mmap(struct mm_struct *mm)
|
|||
active_mm = tsk->active_mm;
|
||||
tsk->active_mm = mm;
|
||||
tsk->mm = mm;
|
||||
lru_gen_add_mm(mm);
|
||||
/*
|
||||
* This prevents preemption while active_mm is being loaded and
|
||||
* it and mm are being updated, which could cause problems for
|
||||
|
|
@ -1025,6 +1026,7 @@ static int exec_mmap(struct mm_struct *mm)
|
|||
tsk->mm->vmacache_seqnum = 0;
|
||||
vmacache_flush(tsk);
|
||||
task_unlock(tsk);
|
||||
lru_gen_use_mm(mm);
|
||||
if (old_mm) {
|
||||
mmap_read_unlock(old_mm);
|
||||
BUG_ON(active_mm != old_mm);
|
||||
|
|
|
|||
|
|
@ -780,7 +780,8 @@ static int fuse_check_page(struct page *page)
|
|||
1 << PG_active |
|
||||
1 << PG_workingset |
|
||||
1 << PG_reclaim |
|
||||
1 << PG_waiters))) {
|
||||
1 << PG_waiters |
|
||||
LRU_GEN_MASK | LRU_REFS_MASK))) {
|
||||
dump_page(page, "fuse: trying to steal weird page");
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -433,6 +433,18 @@ static inline void cgroup_put(struct cgroup *cgrp)
|
|||
css_put(&cgrp->self);
|
||||
}
|
||||
|
||||
extern struct mutex cgroup_mutex;
|
||||
|
||||
static inline void cgroup_lock(void)
|
||||
{
|
||||
mutex_lock(&cgroup_mutex);
|
||||
}
|
||||
|
||||
static inline void cgroup_unlock(void)
|
||||
{
|
||||
mutex_unlock(&cgroup_mutex);
|
||||
}
|
||||
|
||||
/**
|
||||
* task_css_set_check - obtain a task's css_set with extra access conditions
|
||||
* @task: the task to obtain css_set for
|
||||
|
|
@ -447,7 +459,6 @@ static inline void cgroup_put(struct cgroup *cgrp)
|
|||
* as locks used during the cgroup_subsys::attach() methods.
|
||||
*/
|
||||
#ifdef CONFIG_PROVE_RCU
|
||||
extern struct mutex cgroup_mutex;
|
||||
extern spinlock_t css_set_lock;
|
||||
#define task_css_set_check(task, __c) \
|
||||
rcu_dereference_check((task)->cgroups, \
|
||||
|
|
@ -707,6 +718,8 @@ struct cgroup;
|
|||
static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; }
|
||||
static inline void css_get(struct cgroup_subsys_state *css) {}
|
||||
static inline void css_put(struct cgroup_subsys_state *css) {}
|
||||
static inline void cgroup_lock(void) {}
|
||||
static inline void cgroup_unlock(void) {}
|
||||
static inline int cgroup_attach_task_all(struct task_struct *from,
|
||||
struct task_struct *t) { return 0; }
|
||||
static inline int cgroupstats_build(struct cgroupstats *stats,
|
||||
|
|
|
|||
|
|
@ -350,6 +350,11 @@ struct mem_cgroup {
|
|||
struct deferred_split deferred_split_queue;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
/* per-memcg mm_struct list */
|
||||
struct lru_gen_mm_list mm_list;
|
||||
#endif
|
||||
|
||||
struct mem_cgroup_per_node *nodeinfo[];
|
||||
};
|
||||
|
||||
|
|
@ -445,6 +450,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
|
|||
* - LRU isolation
|
||||
* - lock_page_memcg()
|
||||
* - exclusive reference
|
||||
* - mem_cgroup_trylock_pages()
|
||||
*
|
||||
* For a kmem folio a caller should hold an rcu read lock to protect memcg
|
||||
* associated with a kmem folio from being released.
|
||||
|
|
@ -506,6 +512,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
|
|||
* - LRU isolation
|
||||
* - lock_page_memcg()
|
||||
* - exclusive reference
|
||||
* - mem_cgroup_trylock_pages()
|
||||
*
|
||||
* For a kmem page a caller should hold an rcu read lock to protect memcg
|
||||
* associated with a kmem page from being released.
|
||||
|
|
@ -960,6 +967,23 @@ void unlock_page_memcg(struct page *page);
|
|||
|
||||
void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
|
||||
|
||||
/* try to stablize folio_memcg() for all the pages in a memcg */
|
||||
static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
|
||||
{
|
||||
rcu_read_lock();
|
||||
|
||||
if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account))
|
||||
return true;
|
||||
|
||||
rcu_read_unlock();
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline void mem_cgroup_unlock_pages(void)
|
||||
{
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
/* idx can be of type enum memcg_stat_item or node_stat_item */
|
||||
static inline void mod_memcg_state(struct mem_cgroup *memcg,
|
||||
int idx, int val)
|
||||
|
|
@ -1434,6 +1458,18 @@ static inline void folio_memcg_unlock(struct folio *folio)
|
|||
{
|
||||
}
|
||||
|
||||
static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
|
||||
{
|
||||
/* to match folio_memcg_rcu() */
|
||||
rcu_read_lock();
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline void mem_cgroup_unlock_pages(void)
|
||||
{
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
static inline void mem_cgroup_handle_over_high(void)
|
||||
{
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1490,6 +1490,11 @@ static inline unsigned long folio_pfn(struct folio *folio)
|
|||
return page_to_pfn(&folio->page);
|
||||
}
|
||||
|
||||
static inline struct folio *pfn_folio(unsigned long pfn)
|
||||
{
|
||||
return page_folio(pfn_to_page(pfn));
|
||||
}
|
||||
|
||||
static inline atomic_t *folio_pincount_ptr(struct folio *folio)
|
||||
{
|
||||
return &folio_page(folio, 1)->compound_pincount;
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ static __always_inline void __update_lru_size(struct lruvec *lruvec,
|
|||
{
|
||||
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
|
||||
|
||||
lockdep_assert_held(&lruvec->lru_lock);
|
||||
WARN_ON_ONCE(nr_pages != (int)nr_pages);
|
||||
|
||||
__mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
|
||||
__mod_zone_page_state(&pgdat->node_zones[zid],
|
||||
NR_ZONE_LRU_BASE + lru, nr_pages);
|
||||
|
|
@ -101,11 +104,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio)
|
|||
return lru;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
|
||||
#ifdef CONFIG_LRU_GEN_ENABLED
|
||||
static inline bool lru_gen_enabled(void)
|
||||
{
|
||||
DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]);
|
||||
|
||||
return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]);
|
||||
}
|
||||
#else
|
||||
static inline bool lru_gen_enabled(void)
|
||||
{
|
||||
DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]);
|
||||
|
||||
return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]);
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline bool lru_gen_in_fault(void)
|
||||
{
|
||||
return current->in_lru_fault;
|
||||
}
|
||||
|
||||
static inline int lru_gen_from_seq(unsigned long seq)
|
||||
{
|
||||
return seq % MAX_NR_GENS;
|
||||
}
|
||||
|
||||
static inline int lru_hist_from_seq(unsigned long seq)
|
||||
{
|
||||
return seq % NR_HIST_GENS;
|
||||
}
|
||||
|
||||
static inline int lru_tier_from_refs(int refs)
|
||||
{
|
||||
VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH));
|
||||
|
||||
/* see the comment in folio_lru_refs() */
|
||||
return order_base_2(refs + 1);
|
||||
}
|
||||
|
||||
static inline int folio_lru_refs(struct folio *folio)
|
||||
{
|
||||
unsigned long flags = READ_ONCE(folio->flags);
|
||||
bool workingset = flags & BIT(PG_workingset);
|
||||
|
||||
/*
|
||||
* Return the number of accesses beyond PG_referenced, i.e., N-1 if the
|
||||
* total number of accesses is N>1, since N=0,1 both map to the first
|
||||
* tier. lru_tier_from_refs() will account for this off-by-one. Also see
|
||||
* the comment on MAX_NR_TIERS.
|
||||
*/
|
||||
return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset;
|
||||
}
|
||||
|
||||
static inline int folio_lru_gen(struct folio *folio)
|
||||
{
|
||||
unsigned long flags = READ_ONCE(folio->flags);
|
||||
|
||||
return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)
|
||||
{
|
||||
unsigned long max_seq = lruvec->lrugen.max_seq;
|
||||
|
||||
VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
|
||||
|
||||
/* see the comment on MIN_NR_GENS */
|
||||
return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
|
||||
}
|
||||
|
||||
static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
|
||||
int old_gen, int new_gen)
|
||||
{
|
||||
int type = folio_is_file_lru(folio);
|
||||
int zone = folio_zonenum(folio);
|
||||
int delta = folio_nr_pages(folio);
|
||||
enum lru_list lru = type * LRU_INACTIVE_FILE;
|
||||
struct lru_gen_struct *lrugen = &lruvec->lrugen;
|
||||
|
||||
VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
|
||||
VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
|
||||
VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);
|
||||
|
||||
if (old_gen >= 0)
|
||||
WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
|
||||
lrugen->nr_pages[old_gen][type][zone] - delta);
|
||||
if (new_gen >= 0)
|
||||
WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
|
||||
lrugen->nr_pages[new_gen][type][zone] + delta);
|
||||
|
||||
/* addition */
|
||||
if (old_gen < 0) {
|
||||
if (lru_gen_is_active(lruvec, new_gen))
|
||||
lru += LRU_ACTIVE;
|
||||
__update_lru_size(lruvec, lru, zone, delta);
|
||||
return;
|
||||
}
|
||||
|
||||
/* deletion */
|
||||
if (new_gen < 0) {
|
||||
if (lru_gen_is_active(lruvec, old_gen))
|
||||
lru += LRU_ACTIVE;
|
||||
__update_lru_size(lruvec, lru, zone, -delta);
|
||||
return;
|
||||
}
|
||||
|
||||
/* promotion */
|
||||
if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
|
||||
__update_lru_size(lruvec, lru, zone, -delta);
|
||||
__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
|
||||
}
|
||||
|
||||
/* demotion requires isolation, e.g., lru_deactivate_fn() */
|
||||
VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
|
||||
}
|
||||
|
||||
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
|
||||
{
|
||||
unsigned long seq;
|
||||
unsigned long flags;
|
||||
int gen = folio_lru_gen(folio);
|
||||
int type = folio_is_file_lru(folio);
|
||||
int zone = folio_zonenum(folio);
|
||||
struct lru_gen_struct *lrugen = &lruvec->lrugen;
|
||||
|
||||
VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
|
||||
|
||||
if (folio_test_unevictable(folio) || !lrugen->enabled)
|
||||
return false;
|
||||
/*
|
||||
* There are three common cases for this page:
|
||||
* 1. If it's hot, e.g., freshly faulted in or previously hot and
|
||||
* migrated, add it to the youngest generation.
|
||||
* 2. If it's cold but can't be evicted immediately, i.e., an anon page
|
||||
* not in swapcache or a dirty page pending writeback, add it to the
|
||||
* second oldest generation.
|
||||
* 3. Everything else (clean, cold) is added to the oldest generation.
|
||||
*/
|
||||
if (folio_test_active(folio))
|
||||
seq = lrugen->max_seq;
|
||||
else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) ||
|
||||
(folio_test_reclaim(folio) &&
|
||||
(folio_test_dirty(folio) || folio_test_writeback(folio))))
|
||||
seq = lrugen->min_seq[type] + 1;
|
||||
else
|
||||
seq = lrugen->min_seq[type];
|
||||
|
||||
gen = lru_gen_from_seq(seq);
|
||||
flags = (gen + 1UL) << LRU_GEN_PGOFF;
|
||||
/* see the comment on MIN_NR_GENS about PG_active */
|
||||
set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);
|
||||
|
||||
lru_gen_update_size(lruvec, folio, -1, gen);
|
||||
/* for folio_rotate_reclaimable() */
|
||||
if (reclaiming)
|
||||
list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]);
|
||||
else
|
||||
list_add(&folio->lru, &lrugen->lists[gen][type][zone]);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
|
||||
{
|
||||
unsigned long flags;
|
||||
int gen = folio_lru_gen(folio);
|
||||
|
||||
if (gen < 0)
|
||||
return false;
|
||||
|
||||
VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
|
||||
VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
|
||||
|
||||
/* for folio_migrate_flags() */
|
||||
flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
|
||||
flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);
|
||||
gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
|
||||
|
||||
lru_gen_update_size(lruvec, folio, gen, -1);
|
||||
list_del(&folio->lru);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#else /* !CONFIG_LRU_GEN */
|
||||
|
||||
static inline bool lru_gen_enabled(void)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_in_fault(void)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
static __always_inline
|
||||
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
|
||||
{
|
||||
enum lru_list lru = folio_lru_list(folio);
|
||||
|
||||
if (lru_gen_add_folio(lruvec, folio, false))
|
||||
return;
|
||||
|
||||
update_lru_size(lruvec, lru, folio_zonenum(folio),
|
||||
folio_nr_pages(folio));
|
||||
if (lru != LRU_UNEVICTABLE)
|
||||
|
|
@ -123,6 +339,9 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
|
|||
{
|
||||
enum lru_list lru = folio_lru_list(folio);
|
||||
|
||||
if (lru_gen_add_folio(lruvec, folio, true))
|
||||
return;
|
||||
|
||||
update_lru_size(lruvec, lru, folio_zonenum(folio),
|
||||
folio_nr_pages(folio));
|
||||
/* This is not expected to be used on LRU_UNEVICTABLE */
|
||||
|
|
@ -140,6 +359,9 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
|
|||
{
|
||||
enum lru_list lru = folio_lru_list(folio);
|
||||
|
||||
if (lru_gen_del_folio(lruvec, folio, false))
|
||||
return;
|
||||
|
||||
if (lru != LRU_UNEVICTABLE)
|
||||
list_del(&folio->lru);
|
||||
update_lru_size(lruvec, lru, folio_zonenum(folio),
|
||||
|
|
|
|||
|
|
@ -672,6 +672,22 @@ struct mm_struct {
|
|||
*/
|
||||
unsigned long ksm_merging_pages;
|
||||
#endif
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
struct {
|
||||
/* this mm_struct is on lru_gen_mm_list */
|
||||
struct list_head list;
|
||||
/*
|
||||
* Set when switching to this mm_struct, as a hint of
|
||||
* whether it has been used since the last time per-node
|
||||
* page table walkers cleared the corresponding bits.
|
||||
*/
|
||||
unsigned long bitmap;
|
||||
#ifdef CONFIG_MEMCG
|
||||
/* points to the memcg of "owner" above */
|
||||
struct mem_cgroup *memcg;
|
||||
#endif
|
||||
} lru_gen;
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
} __randomize_layout;
|
||||
|
||||
/*
|
||||
|
|
@ -698,6 +714,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
|
|||
return (struct cpumask *)&mm->cpu_bitmap;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
|
||||
struct lru_gen_mm_list {
|
||||
/* mm_struct list for page table walkers */
|
||||
struct list_head fifo;
|
||||
/* protects the list above */
|
||||
spinlock_t lock;
|
||||
};
|
||||
|
||||
void lru_gen_add_mm(struct mm_struct *mm);
|
||||
void lru_gen_del_mm(struct mm_struct *mm);
|
||||
#ifdef CONFIG_MEMCG
|
||||
void lru_gen_migrate_mm(struct mm_struct *mm);
|
||||
#endif
|
||||
|
||||
static inline void lru_gen_init_mm(struct mm_struct *mm)
|
||||
{
|
||||
INIT_LIST_HEAD(&mm->lru_gen.list);
|
||||
mm->lru_gen.bitmap = 0;
|
||||
#ifdef CONFIG_MEMCG
|
||||
mm->lru_gen.memcg = NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void lru_gen_use_mm(struct mm_struct *mm)
|
||||
{
|
||||
/*
|
||||
* When the bitmap is set, page reclaim knows this mm_struct has been
|
||||
* used since the last time it cleared the bitmap. So it might be worth
|
||||
* walking the page tables of this mm_struct to clear the accessed bit.
|
||||
*/
|
||||
WRITE_ONCE(mm->lru_gen.bitmap, -1);
|
||||
}
|
||||
|
||||
#else /* !CONFIG_LRU_GEN */
|
||||
|
||||
static inline void lru_gen_add_mm(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void lru_gen_del_mm(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
static inline void lru_gen_migrate_mm(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline void lru_gen_init_mm(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void lru_gen_use_mm(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
struct mmu_gather;
|
||||
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
|
||||
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
|
||||
|
|
|
|||
|
|
@ -316,6 +316,207 @@ enum lruvec_flags {
|
|||
*/
|
||||
};
|
||||
|
||||
#endif /* !__GENERATING_BOUNDS_H */
|
||||
|
||||
/*
|
||||
* Evictable pages are divided into multiple generations. The youngest and the
|
||||
* oldest generation numbers, max_seq and min_seq, are monotonically increasing.
|
||||
* They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An
|
||||
* offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the
|
||||
* corresponding generation. The gen counter in folio->flags stores gen+1 while
|
||||
* a page is on one of lrugen->lists[]. Otherwise it stores 0.
|
||||
*
|
||||
* A page is added to the youngest generation on faulting. The aging needs to
|
||||
* check the accessed bit at least twice before handing this page over to the
|
||||
* eviction. The first check takes care of the accessed bit set on the initial
|
||||
* fault; the second check makes sure this page hasn't been used since then.
|
||||
* This process, AKA second chance, requires a minimum of two generations,
|
||||
* hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive
|
||||
* LRU, e.g., /proc/vmstat, these two generations are considered active; the
|
||||
* rest of generations, if they exist, are considered inactive. See
|
||||
* lru_gen_is_active().
|
||||
*
|
||||
* PG_active is always cleared while a page is on one of lrugen->lists[] so that
|
||||
* the aging needs not to worry about it. And it's set again when a page
|
||||
* considered active is isolated for non-reclaiming purposes, e.g., migration.
|
||||
* See lru_gen_add_folio() and lru_gen_del_folio().
|
||||
*
|
||||
* MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the
|
||||
* number of categories of the active/inactive LRU when keeping track of
|
||||
* accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits
|
||||
* in folio->flags.
|
||||
*/
|
||||
#define MIN_NR_GENS 2U
|
||||
#define MAX_NR_GENS 4U
|
||||
|
||||
/*
|
||||
* Each generation is divided into multiple tiers. A page accessed N times
|
||||
* through file descriptors is in tier order_base_2(N). A page in the first tier
|
||||
* (N=0,1) is marked by PG_referenced unless it was faulted in through page
|
||||
* tables or read ahead. A page in any other tier (N>1) is marked by
|
||||
* PG_referenced and PG_workingset. This implies a minimum of two tiers is
|
||||
* supported without using additional bits in folio->flags.
|
||||
*
|
||||
* In contrast to moving across generations which requires the LRU lock, moving
|
||||
* across tiers only involves atomic operations on folio->flags and therefore
|
||||
* has a negligible cost in the buffered access path. In the eviction path,
|
||||
* comparisons of refaulted/(evicted+protected) from the first tier and the
|
||||
* rest infer whether pages accessed multiple times through file descriptors
|
||||
* are statistically hot and thus worth protecting.
|
||||
*
|
||||
* MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the
|
||||
* number of categories of the active/inactive LRU when keeping track of
|
||||
* accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in
|
||||
* folio->flags.
|
||||
*/
|
||||
#define MAX_NR_TIERS 4U
|
||||
|
||||
#ifndef __GENERATING_BOUNDS_H
|
||||
|
||||
struct lruvec;
|
||||
struct page_vma_mapped_walk;
|
||||
|
||||
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
|
||||
#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
|
||||
enum {
|
||||
LRU_GEN_ANON,
|
||||
LRU_GEN_FILE,
|
||||
};
|
||||
|
||||
enum {
|
||||
LRU_GEN_CORE,
|
||||
LRU_GEN_MM_WALK,
|
||||
LRU_GEN_NONLEAF_YOUNG,
|
||||
NR_LRU_GEN_CAPS
|
||||
};
|
||||
|
||||
#define MIN_LRU_BATCH BITS_PER_LONG
|
||||
#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64)
|
||||
|
||||
/* whether to keep historical stats from evicted generations */
|
||||
#ifdef CONFIG_LRU_GEN_STATS
|
||||
#define NR_HIST_GENS MAX_NR_GENS
|
||||
#else
|
||||
#define NR_HIST_GENS 1U
|
||||
#endif
|
||||
|
||||
/*
|
||||
* The youngest generation number is stored in max_seq for both anon and file
|
||||
* types as they are aged on an equal footing. The oldest generation numbers are
|
||||
* stored in min_seq[] separately for anon and file types as clean file pages
|
||||
* can be evicted regardless of swap constraints.
|
||||
*
|
||||
* Normally anon and file min_seq are in sync. But if swapping is constrained,
|
||||
* e.g., out of swap space, file min_seq is allowed to advance and leave anon
|
||||
* min_seq behind.
|
||||
*
|
||||
* The number of pages in each generation is eventually consistent and therefore
|
||||
* can be transiently negative when reset_batch_size() is pending.
|
||||
*/
|
||||
struct lru_gen_struct {
|
||||
/* the aging increments the youngest generation number */
|
||||
unsigned long max_seq;
|
||||
/* the eviction increments the oldest generation numbers */
|
||||
unsigned long min_seq[ANON_AND_FILE];
|
||||
/* the birth time of each generation in jiffies */
|
||||
unsigned long timestamps[MAX_NR_GENS];
|
||||
/* the multi-gen LRU lists, lazily sorted on eviction */
|
||||
struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
|
||||
/* the multi-gen LRU sizes, eventually consistent */
|
||||
long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
|
||||
/* the exponential moving average of refaulted */
|
||||
unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
|
||||
/* the exponential moving average of evicted+protected */
|
||||
unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS];
|
||||
/* the first tier doesn't need protection, hence the minus one */
|
||||
unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1];
|
||||
/* can be modified without holding the LRU lock */
|
||||
atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
|
||||
atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
|
||||
/* whether the multi-gen LRU is enabled */
|
||||
bool enabled;
|
||||
};
|
||||
|
||||
enum {
|
||||
MM_LEAF_TOTAL, /* total leaf entries */
|
||||
MM_LEAF_OLD, /* old leaf entries */
|
||||
MM_LEAF_YOUNG, /* young leaf entries */
|
||||
MM_NONLEAF_TOTAL, /* total non-leaf entries */
|
||||
MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */
|
||||
MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */
|
||||
NR_MM_STATS
|
||||
};
|
||||
|
||||
/* double-buffering Bloom filters */
|
||||
#define NR_BLOOM_FILTERS 2
|
||||
|
||||
struct lru_gen_mm_state {
|
||||
/* set to max_seq after each iteration */
|
||||
unsigned long seq;
|
||||
/* where the current iteration continues (inclusive) */
|
||||
struct list_head *head;
|
||||
/* where the last iteration ended (exclusive) */
|
||||
struct list_head *tail;
|
||||
/* to wait for the last page table walker to finish */
|
||||
struct wait_queue_head wait;
|
||||
/* Bloom filters flip after each iteration */
|
||||
unsigned long *filters[NR_BLOOM_FILTERS];
|
||||
/* the mm stats for debugging */
|
||||
unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
|
||||
/* the number of concurrent page table walkers */
|
||||
int nr_walkers;
|
||||
};
|
||||
|
||||
struct lru_gen_mm_walk {
|
||||
/* the lruvec under reclaim */
|
||||
struct lruvec *lruvec;
|
||||
/* unstable max_seq from lru_gen_struct */
|
||||
unsigned long max_seq;
|
||||
/* the next address within an mm to scan */
|
||||
unsigned long next_addr;
|
||||
/* to batch promoted pages */
|
||||
int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
|
||||
/* to batch the mm stats */
|
||||
int mm_stats[NR_MM_STATS];
|
||||
/* total batched items */
|
||||
int batched;
|
||||
bool can_swap;
|
||||
bool force_scan;
|
||||
};
|
||||
|
||||
void lru_gen_init_lruvec(struct lruvec *lruvec);
|
||||
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
void lru_gen_init_memcg(struct mem_cgroup *memcg);
|
||||
void lru_gen_exit_memcg(struct mem_cgroup *memcg);
|
||||
#endif
|
||||
|
||||
#else /* !CONFIG_LRU_GEN */
|
||||
|
||||
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
|
||||
{
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
struct lruvec {
|
||||
struct list_head lists[NR_LRU_LISTS];
|
||||
/* per lruvec lru_lock for memcg */
|
||||
|
|
@ -333,6 +534,12 @@ struct lruvec {
|
|||
unsigned long refaults[ANON_AND_FILE];
|
||||
/* Various lruvec state flags (enum lruvec_flags) */
|
||||
unsigned long flags;
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
/* evictable pages divided into generations */
|
||||
struct lru_gen_struct lrugen;
|
||||
/* to concurrently iterate lru_gen_mm_list */
|
||||
struct lru_gen_mm_state mm_state;
|
||||
#endif
|
||||
#ifdef CONFIG_MEMCG
|
||||
struct pglist_data *pgdat;
|
||||
#endif
|
||||
|
|
@ -748,6 +955,8 @@ static inline bool zone_is_empty(struct zone *zone)
|
|||
#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
|
||||
#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
|
||||
#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)
|
||||
#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH)
|
||||
#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH)
|
||||
|
||||
/*
|
||||
* Define the bit shifts to access each section. For non-existent
|
||||
|
|
@ -1026,6 +1235,11 @@ typedef struct pglist_data {
|
|||
|
||||
unsigned long flags;
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
/* kswap mm walk data */
|
||||
struct lru_gen_mm_walk mm_walk;
|
||||
#endif
|
||||
|
||||
ZONE_PADDING(_pad2_)
|
||||
|
||||
/* Per-node vmstats */
|
||||
|
|
|
|||
|
|
@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state)
|
|||
#define first_online_node 0
|
||||
#define first_memory_node 0
|
||||
#define next_online_node(nid) (MAX_NUMNODES)
|
||||
#define next_memory_node(nid) (MAX_NUMNODES)
|
||||
#define nr_node_ids 1U
|
||||
#define nr_online_nodes 1U
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,8 @@
|
|||
#define SECTIONS_WIDTH 0
|
||||
#endif
|
||||
|
||||
#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \
|
||||
<= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#define NODES_WIDTH NODES_SHIFT
|
||||
#elif defined(CONFIG_SPARSEMEM_VMEMMAP)
|
||||
#error "Vmemmap: No space for nodes field in page flags"
|
||||
|
|
@ -89,8 +90,8 @@
|
|||
#define LAST_CPUPID_SHIFT 0
|
||||
#endif
|
||||
|
||||
#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \
|
||||
<= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
|
||||
KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
|
||||
#else
|
||||
#define LAST_CPUPID_WIDTH 0
|
||||
|
|
@ -100,10 +101,15 @@
|
|||
#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
|
||||
#endif
|
||||
|
||||
#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \
|
||||
> BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
|
||||
KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#error "Not enough bits in page flags"
|
||||
#endif
|
||||
|
||||
/* see the comment on MAX_NR_TIERS */
|
||||
#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \
|
||||
ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \
|
||||
NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH)
|
||||
|
||||
#endif
|
||||
#endif /* _LINUX_PAGE_FLAGS_LAYOUT */
|
||||
|
|
|
|||
|
|
@ -1058,7 +1058,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
|
|||
1UL << PG_private | 1UL << PG_private_2 | \
|
||||
1UL << PG_writeback | 1UL << PG_reserved | \
|
||||
1UL << PG_slab | 1UL << PG_active | \
|
||||
1UL << PG_unevictable | __PG_MLOCKED)
|
||||
1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK)
|
||||
|
||||
/*
|
||||
* Flags checked when a page is prepped for return by the page allocator.
|
||||
|
|
@ -1069,7 +1069,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
|
|||
* alloc-free cycle to prevent from reusing the page.
|
||||
*/
|
||||
#define PAGE_FLAGS_CHECK_AT_PREP \
|
||||
(PAGEFLAGS_MASK & ~__PG_HWPOISON)
|
||||
((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK)
|
||||
|
||||
#define PAGE_FLAGS_PRIVATE \
|
||||
(1UL << PG_private | 1UL << PG_private_2)
|
||||
|
|
|
|||
|
|
@ -926,6 +926,10 @@ struct task_struct {
|
|||
#ifdef CONFIG_MEMCG
|
||||
unsigned in_user_fault:1;
|
||||
#endif
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
/* whether the LRU algorithm may apply to this access */
|
||||
unsigned in_lru_fault:1;
|
||||
#endif
|
||||
#ifdef CONFIG_COMPAT_BRK
|
||||
unsigned brk_randomized:1;
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -162,6 +162,10 @@ union swap_header {
|
|||
*/
|
||||
struct reclaim_state {
|
||||
unsigned long reclaimed_slab;
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
/* per-thread mm walk data */
|
||||
struct lru_gen_mm_walk *mm_walk;
|
||||
#endif
|
||||
};
|
||||
|
||||
#ifdef __KERNEL__
|
||||
|
|
|
|||
|
|
@ -22,6 +22,13 @@ int main(void)
|
|||
DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
|
||||
#endif
|
||||
DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1));
|
||||
DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2);
|
||||
#else
|
||||
DEFINE(LRU_GEN_WIDTH, 0);
|
||||
DEFINE(__LRU_REFS_WIDTH, 0);
|
||||
#endif
|
||||
/* End of constants */
|
||||
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -164,7 +164,6 @@ struct cgroup_mgctx {
|
|||
#define DEFINE_CGROUP_MGCTX(name) \
|
||||
struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name)
|
||||
|
||||
extern struct mutex cgroup_mutex;
|
||||
extern spinlock_t css_set_lock;
|
||||
extern struct cgroup_subsys *cgroup_subsys[];
|
||||
extern struct list_head cgroup_roots;
|
||||
|
|
|
|||
|
|
@ -466,6 +466,7 @@ void mm_update_next_owner(struct mm_struct *mm)
|
|||
goto retry;
|
||||
}
|
||||
WRITE_ONCE(mm->owner, c);
|
||||
lru_gen_migrate_mm(mm);
|
||||
task_unlock(c);
|
||||
put_task_struct(c);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1163,6 +1163,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
|
|||
goto fail_nocontext;
|
||||
|
||||
mm->user_ns = get_user_ns(user_ns);
|
||||
lru_gen_init_mm(mm);
|
||||
return mm;
|
||||
|
||||
fail_nocontext:
|
||||
|
|
@ -1205,6 +1206,7 @@ static inline void __mmput(struct mm_struct *mm)
|
|||
}
|
||||
if (mm->binfmt)
|
||||
module_put(mm->binfmt->module);
|
||||
lru_gen_del_mm(mm);
|
||||
mmdrop(mm);
|
||||
}
|
||||
|
||||
|
|
@ -2708,6 +2710,13 @@ pid_t kernel_clone(struct kernel_clone_args *args)
|
|||
get_task_struct(p);
|
||||
}
|
||||
|
||||
if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) {
|
||||
/* lock the task to synchronize with memcg migration */
|
||||
task_lock(p);
|
||||
lru_gen_add_mm(p->mm);
|
||||
task_unlock(p);
|
||||
}
|
||||
|
||||
wake_up_new_task(p);
|
||||
|
||||
/* forking complete and child started to run, tell ptracer */
|
||||
|
|
|
|||
|
|
@ -5257,6 +5257,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
|
|||
* finish_task_switch()'s mmdrop().
|
||||
*/
|
||||
switch_mm_irqs_off(prev->active_mm, next->mm, next);
|
||||
lru_gen_use_mm(next->mm);
|
||||
|
||||
if (!prev->mm) { // from kernel
|
||||
/* will mmdrop() in finish_task_switch(). */
|
||||
|
|
|
|||
26
mm/Kconfig
26
mm/Kconfig
|
|
@ -1124,6 +1124,32 @@ config PTE_MARKER_UFFD_WP
|
|||
purposes. It is required to enable userfaultfd write protection on
|
||||
file-backed memory types like shmem and hugetlbfs.
|
||||
|
||||
# multi-gen LRU {
|
||||
config LRU_GEN
|
||||
bool "Multi-Gen LRU"
|
||||
depends on MMU
|
||||
# make sure folio->flags has enough spare bits
|
||||
depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP
|
||||
help
|
||||
A high performance LRU implementation to overcommit memory. See
|
||||
Documentation/admin-guide/mm/multigen_lru.rst for details.
|
||||
|
||||
config LRU_GEN_ENABLED
|
||||
bool "Enable by default"
|
||||
depends on LRU_GEN
|
||||
help
|
||||
This option enables the multi-gen LRU by default.
|
||||
|
||||
config LRU_GEN_STATS
|
||||
bool "Full stats for debugging"
|
||||
depends on LRU_GEN
|
||||
help
|
||||
Do not enable this option unless you plan to look at historical stats
|
||||
from evicted generations for debugging purpose.
|
||||
|
||||
This option has a per-memcg and per-node memory overhead.
|
||||
# }
|
||||
|
||||
source "mm/damon/Kconfig"
|
||||
|
||||
endmenu
|
||||
|
|
|
|||
|
|
@ -2444,7 +2444,8 @@ static void __split_huge_page_tail(struct page *head, int tail,
|
|||
#ifdef CONFIG_64BIT
|
||||
(1L << PG_arch_2) |
|
||||
#endif
|
||||
(1L << PG_dirty)));
|
||||
(1L << PG_dirty) |
|
||||
LRU_GEN_MASK | LRU_REFS_MASK));
|
||||
|
||||
/* ->mapping in first tail page is compound_mapcount */
|
||||
VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf);
|
|||
void folio_rotate_reclaimable(struct folio *folio);
|
||||
bool __folio_end_writeback(struct folio *folio);
|
||||
void deactivate_file_folio(struct folio *folio);
|
||||
void folio_activate(struct folio *folio);
|
||||
|
||||
void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
|
||||
unsigned long floor, unsigned long ceiling);
|
||||
|
|
|
|||
|
|
@ -2783,6 +2783,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
|
|||
* - LRU isolation
|
||||
* - lock_page_memcg()
|
||||
* - exclusive reference
|
||||
* - mem_cgroup_trylock_pages()
|
||||
*/
|
||||
folio->memcg_data = (unsigned long)memcg;
|
||||
}
|
||||
|
|
@ -5169,6 +5170,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
|
|||
|
||||
static void mem_cgroup_free(struct mem_cgroup *memcg)
|
||||
{
|
||||
lru_gen_exit_memcg(memcg);
|
||||
memcg_wb_domain_exit(memcg);
|
||||
__mem_cgroup_free(memcg);
|
||||
}
|
||||
|
|
@ -5227,6 +5229,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void)
|
|||
memcg->deferred_split_queue.split_queue_len = 0;
|
||||
#endif
|
||||
idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
|
||||
lru_gen_init_memcg(memcg);
|
||||
return memcg;
|
||||
fail:
|
||||
mem_cgroup_id_remove(memcg);
|
||||
|
|
@ -6195,6 +6198,30 @@ static void mem_cgroup_move_task(void)
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
static void mem_cgroup_attach(struct cgroup_taskset *tset)
|
||||
{
|
||||
struct task_struct *task;
|
||||
struct cgroup_subsys_state *css;
|
||||
|
||||
/* find the first leader if there is any */
|
||||
cgroup_taskset_for_each_leader(task, css, tset)
|
||||
break;
|
||||
|
||||
if (!task)
|
||||
return;
|
||||
|
||||
task_lock(task);
|
||||
if (task->mm && READ_ONCE(task->mm->owner) == task)
|
||||
lru_gen_migrate_mm(task->mm);
|
||||
task_unlock(task);
|
||||
}
|
||||
#else
|
||||
static void mem_cgroup_attach(struct cgroup_taskset *tset)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
|
||||
{
|
||||
if (value == PAGE_COUNTER_MAX)
|
||||
|
|
@ -6600,6 +6627,7 @@ struct cgroup_subsys memory_cgrp_subsys = {
|
|||
.css_reset = mem_cgroup_css_reset,
|
||||
.css_rstat_flush = mem_cgroup_css_rstat_flush,
|
||||
.can_attach = mem_cgroup_can_attach,
|
||||
.attach = mem_cgroup_attach,
|
||||
.cancel_attach = mem_cgroup_cancel_attach,
|
||||
.post_attach = mem_cgroup_move_task,
|
||||
.dfl_cftypes = memory_files,
|
||||
|
|
|
|||
25
mm/memory.c
25
mm/memory.c
|
|
@ -5118,6 +5118,27 @@ static inline void mm_account_fault(struct pt_regs *regs,
|
|||
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
static void lru_gen_enter_fault(struct vm_area_struct *vma)
|
||||
{
|
||||
/* the LRU algorithm doesn't apply to sequential or random reads */
|
||||
current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ));
|
||||
}
|
||||
|
||||
static void lru_gen_exit_fault(void)
|
||||
{
|
||||
current->in_lru_fault = false;
|
||||
}
|
||||
#else
|
||||
static void lru_gen_enter_fault(struct vm_area_struct *vma)
|
||||
{
|
||||
}
|
||||
|
||||
static void lru_gen_exit_fault(void)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
/*
|
||||
* By the time we get here, we already hold the mm semaphore
|
||||
*
|
||||
|
|
@ -5149,11 +5170,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
|
|||
if (flags & FAULT_FLAG_USER)
|
||||
mem_cgroup_enter_user_fault();
|
||||
|
||||
lru_gen_enter_fault(vma);
|
||||
|
||||
if (unlikely(is_vm_hugetlb_page(vma)))
|
||||
ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
|
||||
else
|
||||
ret = __handle_mm_fault(vma, address, flags);
|
||||
|
||||
lru_gen_exit_fault();
|
||||
|
||||
if (flags & FAULT_FLAG_USER) {
|
||||
mem_cgroup_exit_user_fault();
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void)
|
|||
|
||||
shift = 8 * sizeof(unsigned long);
|
||||
width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH
|
||||
- LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH;
|
||||
- LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;
|
||||
mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
|
||||
"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n",
|
||||
"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
|
||||
SECTIONS_WIDTH,
|
||||
NODES_WIDTH,
|
||||
ZONES_WIDTH,
|
||||
LAST_CPUPID_WIDTH,
|
||||
KASAN_TAG_WIDTH,
|
||||
LRU_GEN_WIDTH,
|
||||
LRU_REFS_WIDTH,
|
||||
NR_PAGEFLAGS);
|
||||
mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
|
||||
"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",
|
||||
|
|
|
|||
|
|
@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec)
|
|||
* Poison its list head, so that any operations on it would crash.
|
||||
*/
|
||||
list_del(&lruvec->lists[LRU_UNEVICTABLE]);
|
||||
|
||||
lru_gen_init_lruvec(lruvec);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)
|
||||
|
|
|
|||
|
|
@ -828,6 +828,12 @@ static bool folio_referenced_one(struct folio *folio,
|
|||
}
|
||||
|
||||
if (pvmw.pte) {
|
||||
if (lru_gen_enabled() && pte_young(*pvmw.pte) &&
|
||||
!(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) {
|
||||
lru_gen_look_around(&pvmw);
|
||||
referenced++;
|
||||
}
|
||||
|
||||
if (ptep_clear_flush_young_notify(vma, address,
|
||||
pvmw.pte)) {
|
||||
/*
|
||||
|
|
|
|||
54
mm/swap.c
54
mm/swap.c
|
|
@ -366,7 +366,7 @@ static void folio_activate_drain(int cpu)
|
|||
folio_batch_move_lru(fbatch, folio_activate_fn);
|
||||
}
|
||||
|
||||
static void folio_activate(struct folio *folio)
|
||||
void folio_activate(struct folio *folio)
|
||||
{
|
||||
if (folio_test_lru(folio) && !folio_test_active(folio) &&
|
||||
!folio_test_unevictable(folio)) {
|
||||
|
|
@ -385,7 +385,7 @@ static inline void folio_activate_drain(int cpu)
|
|||
{
|
||||
}
|
||||
|
||||
static void folio_activate(struct folio *folio)
|
||||
void folio_activate(struct folio *folio)
|
||||
{
|
||||
struct lruvec *lruvec;
|
||||
|
||||
|
|
@ -428,6 +428,40 @@ static void __lru_cache_activate_folio(struct folio *folio)
|
|||
local_unlock(&cpu_fbatches.lock);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
static void folio_inc_refs(struct folio *folio)
|
||||
{
|
||||
unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
|
||||
|
||||
if (folio_test_unevictable(folio))
|
||||
return;
|
||||
|
||||
if (!folio_test_referenced(folio)) {
|
||||
folio_set_referenced(folio);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!folio_test_workingset(folio)) {
|
||||
folio_set_workingset(folio);
|
||||
return;
|
||||
}
|
||||
|
||||
/* see the comment on MAX_NR_TIERS */
|
||||
do {
|
||||
new_flags = old_flags & LRU_REFS_MASK;
|
||||
if (new_flags == LRU_REFS_MASK)
|
||||
break;
|
||||
|
||||
new_flags += BIT(LRU_REFS_PGOFF);
|
||||
new_flags |= old_flags & ~LRU_REFS_MASK;
|
||||
} while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
|
||||
}
|
||||
#else
|
||||
static void folio_inc_refs(struct folio *folio)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
/*
|
||||
* Mark a page as having seen activity.
|
||||
*
|
||||
|
|
@ -440,6 +474,11 @@ static void __lru_cache_activate_folio(struct folio *folio)
|
|||
*/
|
||||
void folio_mark_accessed(struct folio *folio)
|
||||
{
|
||||
if (lru_gen_enabled()) {
|
||||
folio_inc_refs(folio);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!folio_test_referenced(folio)) {
|
||||
folio_set_referenced(folio);
|
||||
} else if (folio_test_unevictable(folio)) {
|
||||
|
|
@ -484,6 +523,11 @@ void folio_add_lru(struct folio *folio)
|
|||
folio_test_unevictable(folio), folio);
|
||||
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
|
||||
|
||||
/* see the comment in lru_gen_add_folio() */
|
||||
if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
|
||||
lru_gen_in_fault() && !(current->flags & PF_MEMALLOC))
|
||||
folio_set_active(folio);
|
||||
|
||||
folio_get(folio);
|
||||
local_lock(&cpu_fbatches.lock);
|
||||
fbatch = this_cpu_ptr(&cpu_fbatches.lru_add);
|
||||
|
|
@ -575,7 +619,7 @@ static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio)
|
|||
|
||||
static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio)
|
||||
{
|
||||
if (folio_test_active(folio) && !folio_test_unevictable(folio)) {
|
||||
if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) {
|
||||
long nr_pages = folio_nr_pages(folio);
|
||||
|
||||
lruvec_del_folio(lruvec, folio);
|
||||
|
|
@ -688,8 +732,8 @@ void deactivate_page(struct page *page)
|
|||
{
|
||||
struct folio *folio = page_folio(page);
|
||||
|
||||
if (folio_test_lru(folio) && folio_test_active(folio) &&
|
||||
!folio_test_unevictable(folio)) {
|
||||
if (folio_test_lru(folio) && !folio_test_unevictable(folio) &&
|
||||
(folio_test_active(folio) || lru_gen_enabled())) {
|
||||
struct folio_batch *fbatch;
|
||||
|
||||
folio_get(folio);
|
||||
|
|
|
|||
2797
mm/vmscan.c
2797
mm/vmscan.c
File diff suppressed because it is too large
Load Diff
110
mm/workingset.c
110
mm/workingset.c
|
|
@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly;
|
|||
static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction,
|
||||
bool workingset)
|
||||
{
|
||||
eviction >>= bucket_order;
|
||||
eviction &= EVICTION_MASK;
|
||||
eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
|
||||
eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
|
||||
|
|
@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
|
|||
|
||||
*memcgidp = memcgid;
|
||||
*pgdat = NODE_DATA(nid);
|
||||
*evictionp = entry << bucket_order;
|
||||
*evictionp = entry;
|
||||
*workingsetp = workingset;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
|
||||
static void *lru_gen_eviction(struct folio *folio)
|
||||
{
|
||||
int hist;
|
||||
unsigned long token;
|
||||
unsigned long min_seq;
|
||||
struct lruvec *lruvec;
|
||||
struct lru_gen_struct *lrugen;
|
||||
int type = folio_is_file_lru(folio);
|
||||
int delta = folio_nr_pages(folio);
|
||||
int refs = folio_lru_refs(folio);
|
||||
int tier = lru_tier_from_refs(refs);
|
||||
struct mem_cgroup *memcg = folio_memcg(folio);
|
||||
struct pglist_data *pgdat = folio_pgdat(folio);
|
||||
|
||||
BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT);
|
||||
|
||||
lruvec = mem_cgroup_lruvec(memcg, pgdat);
|
||||
lrugen = &lruvec->lrugen;
|
||||
min_seq = READ_ONCE(lrugen->min_seq[type]);
|
||||
token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
|
||||
|
||||
hist = lru_hist_from_seq(min_seq);
|
||||
atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
|
||||
|
||||
return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs);
|
||||
}
|
||||
|
||||
static void lru_gen_refault(struct folio *folio, void *shadow)
|
||||
{
|
||||
int hist, tier, refs;
|
||||
int memcg_id;
|
||||
bool workingset;
|
||||
unsigned long token;
|
||||
unsigned long min_seq;
|
||||
struct lruvec *lruvec;
|
||||
struct lru_gen_struct *lrugen;
|
||||
struct mem_cgroup *memcg;
|
||||
struct pglist_data *pgdat;
|
||||
int type = folio_is_file_lru(folio);
|
||||
int delta = folio_nr_pages(folio);
|
||||
|
||||
unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset);
|
||||
|
||||
if (pgdat != folio_pgdat(folio))
|
||||
return;
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
memcg = folio_memcg_rcu(folio);
|
||||
if (memcg_id != mem_cgroup_id(memcg))
|
||||
goto unlock;
|
||||
|
||||
lruvec = mem_cgroup_lruvec(memcg, pgdat);
|
||||
lrugen = &lruvec->lrugen;
|
||||
|
||||
min_seq = READ_ONCE(lrugen->min_seq[type]);
|
||||
if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH)))
|
||||
goto unlock;
|
||||
|
||||
hist = lru_hist_from_seq(min_seq);
|
||||
/* see the comment in folio_lru_refs() */
|
||||
refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset;
|
||||
tier = lru_tier_from_refs(refs);
|
||||
|
||||
atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
|
||||
mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
|
||||
|
||||
/*
|
||||
* Count the following two cases as stalls:
|
||||
* 1. For pages accessed through page tables, hotter pages pushed out
|
||||
* hot pages which refaulted immediately.
|
||||
* 2. For pages accessed multiple times through file descriptors,
|
||||
* numbers of accesses might have been out of the range.
|
||||
*/
|
||||
if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) {
|
||||
folio_set_workingset(folio);
|
||||
mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
|
||||
}
|
||||
unlock:
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
#else /* !CONFIG_LRU_GEN */
|
||||
|
||||
static void *lru_gen_eviction(struct folio *folio)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void lru_gen_refault(struct folio *folio, void *shadow)
|
||||
{
|
||||
}
|
||||
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
/**
|
||||
* workingset_age_nonresident - age non-resident entries as LRU ages
|
||||
* @lruvec: the lruvec that was aged
|
||||
|
|
@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg)
|
|||
VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
|
||||
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
|
||||
|
||||
if (lru_gen_enabled())
|
||||
return lru_gen_eviction(folio);
|
||||
|
||||
lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
|
||||
/* XXX: target_memcg can be NULL, go through lruvec */
|
||||
memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
|
||||
eviction = atomic_long_read(&lruvec->nonresident_age);
|
||||
eviction >>= bucket_order;
|
||||
workingset_age_nonresident(lruvec, folio_nr_pages(folio));
|
||||
return pack_shadow(memcgid, pgdat, eviction,
|
||||
folio_test_workingset(folio));
|
||||
|
|
@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow)
|
|||
int memcgid;
|
||||
long nr;
|
||||
|
||||
if (lru_gen_enabled()) {
|
||||
lru_gen_refault(folio, shadow);
|
||||
return;
|
||||
}
|
||||
|
||||
unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset);
|
||||
eviction <<= bucket_order;
|
||||
|
||||
rcu_read_lock();
|
||||
/*
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user