summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKalesh Singh <kaleshsingh@google.com>2023-05-23 23:59:21 +0300
committerAndrew Morton <akpm@linux-foundation.org>2023-06-10 02:25:46 +0300
commit3af0191a594d5ca5d6d2e3602b5d4284c6835e77 (patch)
tree53b368f7763895ae8914982efa4ec9796180bc6b
parent06b27ce36a1a3dc5ea6f8314d0c7d1baa9f8ece7 (diff)
downloadlinux-3af0191a594d5ca5d6d2e3602b5d4284c6835e77.tar.xz
Multi-gen LRU: fix workingset accounting
On Android app cycle workloads, MGLRU showed a significant reduction in workingset refaults although pgpgin/pswpin remained relatively unchanged. This indicated MGLRU may be undercounting workingset refaults. This has impact on userspace programs, like Android's LMKD, that monitor workingset refault statistics to detect thrashing. It was found that refaults were only accounted if the MGLRU shadow entry was for a recently evicted folio. However, recently evicted folios should be accounted as workingset activation, and refaults should be accounted regardless of recency. Fix MGLRU's workingset refault and activation accounting to more closely match that of the conventional active/inactive LRU. Link: https://lkml.kernel.org/r/20230523205922.3852731-1-kaleshsingh@google.com Fixes: ac35a4902374 ("mm: multi-gen LRU: minimal implementation") Signed-off-by: Kalesh Singh <kaleshsingh@google.com> Reported-by: Charan Teja Kalla <quic_charante@quicinc.com> Acked-by: Yu Zhao <yuzhao@google.com> Cc: Brian Geffon <bgeffon@google.com> Cc: Jan Alexander Steffens (heftig) <heftig@archlinux.org> Cc: Oleksandr Natalenko <oleksandr@natalenko.name> Cc: Suren Baghdasaryan <surenb@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
-rw-r--r--mm/vmscan.c1
-rw-r--r--mm/workingset.c10
2 files changed, 7 insertions, 4 deletions
diff --git a/mm/vmscan.c b/mm/vmscan.c
index dbbfcc631f5c..3024b4619b65 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4925,7 +4925,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
lrugen->protected[hist][type][tier - 1] + delta);
- __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
return true;
}
diff --git a/mm/workingset.c b/mm/workingset.c
index 5796e927e6d7..4686ae363000 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -278,6 +278,7 @@ static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec,
static void lru_gen_refault(struct folio *folio, void *shadow)
{
+ bool recent;
int hist, tier, refs;
bool workingset;
unsigned long token;
@@ -288,10 +289,13 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
rcu_read_lock();
- if (!lru_gen_test_recent(shadow, type, &lruvec, &token, &workingset))
+ recent = lru_gen_test_recent(shadow, type, &lruvec, &token, &workingset);
+ if (lruvec != folio_lruvec(folio))
goto unlock;
- if (lruvec != folio_lruvec(folio))
+ mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
+
+ if (!recent)
goto unlock;
lrugen = &lruvec->lrugen;
@@ -302,7 +306,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
tier = lru_tier_from_refs(refs);
atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
- mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
+ mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
/*
* Count the following two cases as stalls: