From: Kefeng Wang wangkefeng.wang@huawei.com
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9R7N8 CVE: NA
-------------------------------------------------
After committing df0ad0579ac9 ('mm: align larger anonymous mappings on THP boundaries'), we found a number of issues, eg, a significant increase in memory access latency on arm64, which was addressed by commit bb713c2a4a43 (arm64: mmap: disable align larger anonymous mappings on THP boundaries), but subsequently found on x86 that the patch also resulted in a significant degradation of the SPECCPU test. Commit 449846913eb6('mm: huge_memory: add thp mapping align control') has implemented the sysfs interface control of 64 KB aligned mapping in arm64, based on this for thp_align_mapping to add a new bit to control pmd size page alignment, where the 64K alignment mapping enable for arm64 will depend on the new bits being turned on. it is disabled by default, and could be enabled by /sys/kernel/mm/transparent_hugepage/thp_mapping_align.
Signed-off-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Ze Zuo zuoze1@huawei.com
Changes since v1: - Fix build error without CONFIG_TRANSPARENT_HUGEPAGE. - delete some unsed define. --- Documentation/admin-guide/mm/transhuge.rst | 19 +++++++++++-------- include/linux/huge_mm.h | 7 +++++++ mm/huge_memory.c | 12 ++++++++++-- mm/mmap.c | 2 +- 4 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 54046f487f15..aeff60881fd0 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -211,14 +211,17 @@ possible to enable/disable it by configurate the corresponding bit:: echo 0x2 >/sys/kernel/mm/transparent_hugepage/thp_exec_enabled echo 0x3 >/sys/kernel/mm/transparent_hugepage/thp_exec_enabled
-The kernel could try to enable other larger size mappings align other -than THP size, eg, 64K on arm64, BIT0 for file mapping, BIT1 for anon -mapping, it is disabled by default, and could enable this feature by -writing the corresponding bit to 1:: - - echo 0x1 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align - echo 0x2 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align - echo 0x3 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align +The kernel could try to enable mappings for different sizes, eg, 64K for +arm64, BIT0 for file mapping, BIT1 for anonymous mapping, and THP size +page, BIT3 for anonymous mapping, where 64K for arm64 is dependent on +BIT3 being turned on, the above feature are disabled by default, and +could enable the above feature by writing the corresponding bit to 1:: + + echo 0x4 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align + echo 0x5 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align + echo 0x6 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align + echo 0x7 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align +
The kernel could enable high-orders(greated than PAGE_ALLOC_COSTLY_ORDER, only support order 4 for now) be stored on PCP lists(except PMD order), which could diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 3e5f7064e2de..3a15b8417199 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -54,6 +54,7 @@ enum transparent_hugepage_flag { TRANSPARENT_HUGEPAGE_FILE_EXEC_MTHP_FLAG, TRANSPARENT_HUGEPAGE_FILE_MAPPING_ALIGN_FLAG, TRANSPARENT_HUGEPAGE_ANON_MAPPING_ALIGN_FLAG, + TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG, };
struct kobject; @@ -288,6 +289,10 @@ static inline void count_mthp_stat(int order, enum mthp_stat_item item) (transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
+#define thp_anon_mapping_pmd_align() \ + (transparent_hugepage_flags & \ + (1<<TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG)) + unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags);
@@ -443,6 +448,8 @@ static inline void folio_prep_large_rmappable(struct folio *folio) {}
#define transparent_hugepage_flags 0UL
+#define thp_anon_mapping_pmd_align() 0UL + #define thp_get_unmapped_area NULL
static inline bool diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 763bb25e4f99..c090e6cbef3b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -518,7 +518,9 @@ static struct kobj_attribute thp_exec_enabled_attr =
#define FILE_MAPPING_ALIGN BIT(0) #define ANON_MAPPING_ALIGN BIT(1) -#define THP_MAPPING_ALIGN_ALL (FILE_MAPPING_ALIGN | ANON_MAPPING_ALIGN) +#define ANON_MAPPING_PMD_ALIGN BIT(2) +#define THP_MAPPING_ALIGN_ALL \ + (FILE_MAPPING_ALIGN | ANON_MAPPING_ALIGN | ANON_MAPPING_PMD_ALIGN)
static ssize_t thp_mapping_align_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -533,6 +535,10 @@ static ssize_t thp_mapping_align_show(struct kobject *kobj, &transparent_hugepage_flags)) val |= ANON_MAPPING_ALIGN;
+ if (test_bit(TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG, + &transparent_hugepage_flags)) + val |= ANON_MAPPING_PMD_ALIGN; + return sysfs_emit(buf, "0x%lx\n", val); } static ssize_t thp_mapping_align_store(struct kobject *kobj, @@ -544,13 +550,15 @@ static ssize_t thp_mapping_align_store(struct kobject *kobj, ret = kstrtoul(buf, 16, &val); if (ret < 0) return ret; - if (val & ~THP_MAPPING_ALIGN_ALL) + if ((val & ~THP_MAPPING_ALIGN_ALL) || (val && !(val & ANON_MAPPING_PMD_ALIGN))) return -EINVAL;
thp_flag_set(TRANSPARENT_HUGEPAGE_FILE_MAPPING_ALIGN_FLAG, val & FILE_MAPPING_ALIGN); thp_flag_set(TRANSPARENT_HUGEPAGE_ANON_MAPPING_ALIGN_FLAG, val & ANON_MAPPING_ALIGN); + thp_flag_set(TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG, + val & ANON_MAPPING_PMD_ALIGN);
return count; } diff --git a/mm/mmap.c b/mm/mmap.c index 1c256f38128e..1d8def3db125 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1857,7 +1857,7 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, */ get_area = shmem_get_unmapped_area; } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && - !IS_ENABLED(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT)) { + thp_anon_mapping_pmd_align()) { /* Ensures that larger anonymous mappings are THP aligned. */ get_area = thp_get_unmapped_area; }