summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/mmzone.h5
-rw-r--r--mm/internal.h6
-rw-r--r--mm/rmap.c28
-rw-r--r--mm/vmscan.c43
4 files changed, 49 insertions, 33 deletions
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 5c3ae0348754..3f651baf7e2b 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -684,7 +684,7 @@ struct lru_gen_memcg {
void lru_gen_init_pgdat(struct pglist_data *pgdat);
void lru_gen_init_lruvec(struct lruvec *lruvec);
-bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
+bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr);
void lru_gen_init_memcg(struct mem_cgroup *memcg);
void lru_gen_exit_memcg(struct mem_cgroup *memcg);
@@ -703,7 +703,8 @@ static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
{
}
-static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw,
+ unsigned int nr)
{
return false;
}
diff --git a/mm/internal.h b/mm/internal.h
index 1357dc04f065..4ab833b8bcdf 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1848,10 +1848,4 @@ static inline int pmdp_test_and_clear_young_notify(struct vm_area_struct *vma,
#endif /* CONFIG_MMU_NOTIFIER */
-static inline int ptep_test_and_clear_young_notify(struct vm_area_struct *vma,
- unsigned long addr, pte_t *ptep)
-{
- return test_and_clear_young_ptes_notify(vma, addr, ptep, 1);
-}
-
#endif /* __MM_INTERNAL_H */
diff --git a/mm/rmap.c b/mm/rmap.c
index cd48f34f11b5..abe4712a220c 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -965,25 +965,20 @@ static bool folio_referenced_one(struct folio *folio,
return false;
}
+ if (pvmw.pte && folio_test_large(folio)) {
+ const unsigned long end_addr = pmd_addr_end(address, vma->vm_end);
+ const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
+ pte_t pteval = ptep_get(pvmw.pte);
+
+ nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr);
+ }
+
if (lru_gen_enabled() && pvmw.pte) {
- if (lru_gen_look_around(&pvmw))
+ if (lru_gen_look_around(&pvmw, nr))
referenced++;
} else if (pvmw.pte) {
- if (folio_test_large(folio)) {
- unsigned long end_addr = pmd_addr_end(address, vma->vm_end);
- unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
- pte_t pteval = ptep_get(pvmw.pte);
-
- nr = folio_pte_batch(folio, pvmw.pte,
- pteval, max_nr);
- }
-
- ptes += nr;
if (clear_flush_young_ptes_notify(vma, address, pvmw.pte, nr))
referenced++;
- /* Skip the batched PTEs */
- pvmw.pte += nr - 1;
- pvmw.address += (nr - 1) * PAGE_SIZE;
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,
pvmw.pmd))
@@ -993,6 +988,7 @@ static bool folio_referenced_one(struct folio *folio,
WARN_ON_ONCE(1);
}
+ ptes += nr;
pra->mapcount -= nr;
/*
* If we are sure that we batched the entire folio,
@@ -1002,6 +998,10 @@ static bool folio_referenced_one(struct folio *folio,
page_vma_mapped_walk_done(&pvmw);
break;
}
+
+ /* Skip the batched PTEs */
+ pvmw.pte += nr - 1;
+ pvmw.address += (nr - 1) * PAGE_SIZE;
}
if (referenced)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 7ab9e1cdccd2..3a4a0a81c871 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3499,6 +3499,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
DEFINE_MAX_SEQ(walk->lruvec);
int gen = lru_gen_from_seq(max_seq);
+ unsigned int nr;
pmd_t pmdval;
pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl);
@@ -3517,11 +3518,13 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
lazy_mmu_mode_enable();
restart:
- for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) {
+ for (i = pte_index(start), addr = start; addr != end; i += nr, addr += nr * PAGE_SIZE) {
unsigned long pfn;
struct folio *folio;
- pte_t ptent = ptep_get(pte + i);
+ pte_t *cur_pte = pte + i;
+ pte_t ptent = ptep_get(cur_pte);
+ nr = 1;
total++;
walk->mm_stats[MM_LEAF_TOTAL]++;
@@ -3533,7 +3536,16 @@ restart:
if (!folio)
continue;
- if (!ptep_test_and_clear_young_notify(args->vma, addr, pte + i))
+ if (folio_test_large(folio)) {
+ const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
+
+ nr = folio_pte_batch_flags(folio, NULL, cur_pte, &ptent,
+ max_nr, FPB_MERGE_YOUNG_DIRTY);
+ total += nr - 1;
+ walk->mm_stats[MM_LEAF_TOTAL] += nr - 1;
+ }
+
+ if (!test_and_clear_young_ptes_notify(args->vma, addr, cur_pte, nr))
continue;
if (last != folio) {
@@ -3546,8 +3558,8 @@ restart:
if (pte_dirty(ptent))
dirty = true;
- young++;
- walk->mm_stats[MM_LEAF_YOUNG]++;
+ young += nr;
+ walk->mm_stats[MM_LEAF_YOUNG] += nr;
}
walk_update_folio(walk, last, gen, dirty);
@@ -4191,7 +4203,7 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
* the PTE table to the Bloom filter. This forms a feedback loop between the
* eviction and the aging.
*/
-bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
{
int i;
bool dirty;
@@ -4214,7 +4226,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
lockdep_assert_held(pvmw->ptl);
VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
- if (!ptep_test_and_clear_young_notify(vma, addr, pte))
+ if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr))
return false;
if (spin_is_contended(pvmw->ptl))
@@ -4248,10 +4260,12 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
pte -= (addr - start) / PAGE_SIZE;
- for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
+ for (i = 0, addr = start; addr != end;
+ i += nr, pte += nr, addr += nr * PAGE_SIZE) {
unsigned long pfn;
- pte_t ptent = ptep_get(pte + i);
+ pte_t ptent = ptep_get(pte);
+ nr = 1;
pfn = get_pte_pfn(ptent, vma, addr, pgdat);
if (pfn == -1)
continue;
@@ -4260,7 +4274,14 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
if (!folio)
continue;
- if (!ptep_test_and_clear_young_notify(vma, addr, pte + i))
+ if (folio_test_large(folio)) {
+ const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
+
+ nr = folio_pte_batch_flags(folio, NULL, pte, &ptent,
+ max_nr, FPB_MERGE_YOUNG_DIRTY);
+ }
+
+ if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr))
continue;
if (last != folio) {
@@ -4273,7 +4294,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
if (pte_dirty(ptent))
dirty = true;
- young++;
+ young += nr;
}
walk_update_folio(walk, last, gen, dirty);
generated by cgit v1.2.3 (git 2.39.0) at 2026-05-14 21:56:09 +0300