136 lines
4.5 KiB
Diff
136 lines
4.5 KiB
Diff
From fc0e3b06e0f19917b7ecad7967a72f61d4743644 Mon Sep 17 00:00:00 2001
|
|
From: "T.J. Alumbaugh" <talumbau@google.com>
|
|
Date: Wed, 18 Jan 2023 00:18:26 +0000
|
|
Subject: [PATCH 17/19] UPSTREAM: mm: multi-gen LRU: improve walk_pmd_range()
|
|
|
|
Improve readability of walk_pmd_range() and walk_pmd_range_locked().
|
|
|
|
Link: https://lkml.kernel.org/r/20230118001827.1040870-7-talumbau@google.com
|
|
Change-Id: Ia084fbf53fe989673b7804ca8ca520af12d7d52a
|
|
Signed-off-by: T.J. Alumbaugh <talumbau@google.com>
|
|
Cc: Yu Zhao <yuzhao@google.com>
|
|
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
|
|
(cherry picked from commit b5ff4133617d0eced35b685da0bd0929dd9fabb7)
|
|
Bug: 274865848
|
|
Signed-off-by: T.J. Mercier <tjmercier@google.com>
|
|
---
|
|
mm/vmscan.c | 40 ++++++++++++++++++++--------------------
|
|
1 file changed, 20 insertions(+), 20 deletions(-)
|
|
|
|
--- a/mm/vmscan.c
|
|
+++ b/mm/vmscan.c
|
|
@@ -3980,8 +3980,8 @@ restart:
|
|
}
|
|
|
|
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
|
|
-static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
|
|
- struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
|
|
+static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
|
|
+ struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
|
|
{
|
|
int i;
|
|
pmd_t *pmd;
|
|
@@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t
|
|
VM_WARN_ON_ONCE(pud_leaf(*pud));
|
|
|
|
/* try to batch at most 1+MIN_LRU_BATCH+1 entries */
|
|
- if (*start == -1) {
|
|
- *start = next;
|
|
+ if (*first == -1) {
|
|
+ *first = addr;
|
|
+ bitmap_zero(bitmap, MIN_LRU_BATCH);
|
|
return;
|
|
}
|
|
|
|
- i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start);
|
|
+ i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
|
|
if (i && i <= MIN_LRU_BATCH) {
|
|
__set_bit(i - 1, bitmap);
|
|
return;
|
|
}
|
|
|
|
- pmd = pmd_offset(pud, *start);
|
|
+ pmd = pmd_offset(pud, *first);
|
|
|
|
ptl = pmd_lockptr(args->mm, pmd);
|
|
if (!spin_trylock(ptl))
|
|
@@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t
|
|
do {
|
|
unsigned long pfn;
|
|
struct folio *folio;
|
|
- unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start;
|
|
+
|
|
+ /* don't round down the first address */
|
|
+ addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
|
|
|
|
pfn = get_pmd_pfn(pmd[i], vma, addr);
|
|
if (pfn == -1)
|
|
goto next;
|
|
|
|
if (!pmd_trans_huge(pmd[i])) {
|
|
- if (arch_has_hw_nonleaf_pmd_young() &&
|
|
- get_cap(LRU_GEN_NONLEAF_YOUNG))
|
|
+ if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
|
|
pmdp_test_and_clear_young(vma, addr, pmd + i);
|
|
goto next;
|
|
}
|
|
@@ -4053,12 +4055,11 @@ next:
|
|
arch_leave_lazy_mmu_mode();
|
|
spin_unlock(ptl);
|
|
done:
|
|
- *start = -1;
|
|
- bitmap_zero(bitmap, MIN_LRU_BATCH);
|
|
+ *first = -1;
|
|
}
|
|
#else
|
|
-static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
|
|
- struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
|
|
+static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
|
|
+ struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
|
|
{
|
|
}
|
|
#endif
|
|
@@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, u
|
|
unsigned long next;
|
|
unsigned long addr;
|
|
struct vm_area_struct *vma;
|
|
- unsigned long pos = -1;
|
|
+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];
|
|
+ unsigned long first = -1;
|
|
struct lru_gen_mm_walk *walk = args->private;
|
|
- unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
|
|
|
|
VM_WARN_ON_ONCE(pud_leaf(*pud));
|
|
|
|
@@ -4115,18 +4116,17 @@ restart:
|
|
if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
|
|
continue;
|
|
|
|
- walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
|
|
+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
|
|
continue;
|
|
}
|
|
#endif
|
|
walk->mm_stats[MM_NONLEAF_TOTAL]++;
|
|
|
|
- if (arch_has_hw_nonleaf_pmd_young() &&
|
|
- get_cap(LRU_GEN_NONLEAF_YOUNG)) {
|
|
+ if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {
|
|
if (!pmd_young(val))
|
|
continue;
|
|
|
|
- walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
|
|
+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
|
|
}
|
|
|
|
if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
|
|
@@ -4143,7 +4143,7 @@ restart:
|
|
update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
|
|
}
|
|
|
|
- walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos);
|
|
+ walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
|
|
|
|
if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
|
|
goto restart;
|