154 lines
5.6 KiB
Diff
154 lines
5.6 KiB
Diff
From 493de1c4b0f2cd909169401da8c445f6c8a7e29d Mon Sep 17 00:00:00 2001
|
|
From: Yu Zhao <yuzhao@google.com>
|
|
Date: Sun, 18 Sep 2022 01:59:59 -0600
|
|
Subject: [PATCH 02/29] mm: x86: add CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain; charset=UTF-8
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
Some architectures support the accessed bit in non-leaf PMD entries, e.g.,
|
|
x86 sets the accessed bit in a non-leaf PMD entry when using it as part of
|
|
linear address translation [1]. Page table walkers that clear the
|
|
accessed bit may use this capability to reduce their search space.
|
|
|
|
Note that:
|
|
1. Although an inline function is preferable, this capability is added
|
|
as a configuration option for consistency with the existing macros.
|
|
2. Due to the little interest in other varieties, this capability was
|
|
only tested on Intel and AMD CPUs.
|
|
|
|
Thanks to the following developers for their efforts [2][3].
|
|
Randy Dunlap <rdunlap@infradead.org>
|
|
Stephen Rothwell <sfr@canb.auug.org.au>
|
|
|
|
[1]: Intel 64 and IA-32 Architectures Software Developer's Manual
|
|
Volume 3 (June 2021), section 4.8
|
|
[2] https://lore.kernel.org/r/bfdcc7c8-922f-61a9-aa15-7e7250f04af7@infradead.org/
|
|
[3] https://lore.kernel.org/r/20220413151513.5a0d7a7e@canb.auug.org.au/
|
|
|
|
Link: https://lkml.kernel.org/r/20220918080010.2920238-3-yuzhao@google.com
|
|
Signed-off-by: Yu Zhao <yuzhao@google.com>
|
|
Reviewed-by: Barry Song <baohua@kernel.org>
|
|
Acked-by: Brian Geffon <bgeffon@google.com>
|
|
Acked-by: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
|
|
Acked-by: Oleksandr Natalenko <oleksandr@natalenko.name>
|
|
Acked-by: Steven Barrett <steven@liquorix.net>
|
|
Acked-by: Suleiman Souhlal <suleiman@google.com>
|
|
Tested-by: Daniel Byrne <djbyrne@mtu.edu>
|
|
Tested-by: Donald Carr <d@chaos-reins.com>
|
|
Tested-by: Holger Hoffstätte <holger@applied-asynchrony.com>
|
|
Tested-by: Konstantin Kharlamov <Hi-Angel@yandex.ru>
|
|
Tested-by: Shuang Zhai <szhai2@cs.rochester.edu>
|
|
Tested-by: Sofia Trinh <sofia.trinh@edi.works>
|
|
Tested-by: Vaibhav Jain <vaibhav@linux.ibm.com>
|
|
Cc: Andi Kleen <ak@linux.intel.com>
|
|
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
|
|
Cc: Catalin Marinas <catalin.marinas@arm.com>
|
|
Cc: Dave Hansen <dave.hansen@linux.intel.com>
|
|
Cc: Hillf Danton <hdanton@sina.com>
|
|
Cc: Jens Axboe <axboe@kernel.dk>
|
|
Cc: Johannes Weiner <hannes@cmpxchg.org>
|
|
Cc: Jonathan Corbet <corbet@lwn.net>
|
|
Cc: Linus Torvalds <torvalds@linux-foundation.org>
|
|
Cc: Matthew Wilcox <willy@infradead.org>
|
|
Cc: Mel Gorman <mgorman@suse.de>
|
|
Cc: Miaohe Lin <linmiaohe@huawei.com>
|
|
Cc: Michael Larabel <Michael@MichaelLarabel.com>
|
|
Cc: Michal Hocko <mhocko@kernel.org>
|
|
Cc: Mike Rapoport <rppt@kernel.org>
|
|
Cc: Mike Rapoport <rppt@linux.ibm.com>
|
|
Cc: Peter Zijlstra <peterz@infradead.org>
|
|
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
|
|
Cc: Tejun Heo <tj@kernel.org>
|
|
Cc: Vlastimil Babka <vbabka@suse.cz>
|
|
Cc: Will Deacon <will@kernel.org>
|
|
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
|
|
---
|
|
arch/Kconfig | 8 ++++++++
|
|
arch/x86/Kconfig | 1 +
|
|
arch/x86/include/asm/pgtable.h | 3 ++-
|
|
arch/x86/mm/pgtable.c | 5 ++++-
|
|
include/linux/pgtable.h | 4 ++--
|
|
5 files changed, 17 insertions(+), 4 deletions(-)
|
|
|
|
--- a/arch/Kconfig
|
|
+++ b/arch/Kconfig
|
|
@@ -1299,6 +1299,14 @@ config ARCH_HAS_ELFCORE_COMPAT
|
|
config ARCH_HAS_PARANOID_L1D_FLUSH
|
|
bool
|
|
|
|
+config ARCH_HAS_NONLEAF_PMD_YOUNG
|
|
+ bool
|
|
+ help
|
|
+ Architectures that select this option are capable of setting the
|
|
+ accessed bit in non-leaf PMD entries when using them as part of linear
|
|
+ address translations. Page table walkers that clear the accessed bit
|
|
+ may use this capability to reduce their search space.
|
|
+
|
|
source "kernel/gcov/Kconfig"
|
|
|
|
source "scripts/gcc-plugins/Kconfig"
|
|
--- a/arch/x86/Kconfig
|
|
+++ b/arch/x86/Kconfig
|
|
@@ -85,6 +85,7 @@ config X86
|
|
select ARCH_HAS_PMEM_API if X86_64
|
|
select ARCH_HAS_PTE_DEVMAP if X86_64
|
|
select ARCH_HAS_PTE_SPECIAL
|
|
+ select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2
|
|
select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64
|
|
select ARCH_HAS_COPY_MC if X86_64
|
|
select ARCH_HAS_SET_MEMORY
|
|
--- a/arch/x86/include/asm/pgtable.h
|
|
+++ b/arch/x86/include/asm/pgtable.h
|
|
@@ -817,7 +817,8 @@ static inline unsigned long pmd_page_vad
|
|
|
|
static inline int pmd_bad(pmd_t pmd)
|
|
{
|
|
- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
|
|
+ return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) !=
|
|
+ (_KERNPG_TABLE & ~_PAGE_ACCESSED);
|
|
}
|
|
|
|
static inline unsigned long pages_to_mb(unsigned long npg)
|
|
--- a/arch/x86/mm/pgtable.c
|
|
+++ b/arch/x86/mm/pgtable.c
|
|
@@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_
|
|
return ret;
|
|
}
|
|
|
|
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
|
|
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
|
unsigned long addr, pmd_t *pmdp)
|
|
{
|
|
@@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_
|
|
|
|
return ret;
|
|
}
|
|
+#endif
|
|
+
|
|
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
int pudp_test_and_clear_young(struct vm_area_struct *vma,
|
|
unsigned long addr, pud_t *pudp)
|
|
{
|
|
--- a/include/linux/pgtable.h
|
|
+++ b/include/linux/pgtable.h
|
|
@@ -212,7 +212,7 @@ static inline int ptep_test_and_clear_yo
|
|
#endif
|
|
|
|
#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
|
|
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
|
|
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
|
unsigned long address,
|
|
pmd_t *pmdp)
|
|
@@ -233,7 +233,7 @@ static inline int pmdp_test_and_clear_yo
|
|
BUILD_BUG();
|
|
return 0;
|
|
}
|
|
-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
|
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
|
|
#endif
|
|
|
|
#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
|