From: Kefeng Wang wangkefeng.wang@huawei.com
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I98AW9 CVE: NA
-------------------------------------------------
Let's only enable large folio order for exec mapping with large folio support, also add a new sysfs interface to make it configurable.
Signed-off-by: Kefeng Wang wangkefeng.wang@huawei.com --- include/linux/huge_mm.h | 1 + mm/filemap.c | 27 ++++++++++++++++-- mm/huge_memory.c | 62 +++++++++++++++++++++++++++++++---------- 3 files changed, 74 insertions(+), 16 deletions(-)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index abf2340a2d18..896d7870ecd9 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -51,6 +51,7 @@ enum transparent_hugepage_flag { TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG, TRANSPARENT_HUGEPAGE_FILE_EXEC_THP_FLAG, + TRANSPARENT_HUGEPAGE_FILE_EXEC_MTHP_FLAG, };
struct kobject; diff --git a/mm/filemap.c b/mm/filemap.c index b7881f1db472..d3c813429bf2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -46,6 +46,7 @@ #include <linux/pipe_fs_i.h> #include <linux/splice.h> #include <linux/huge_mm.h> +#include <linux/pgtable.h> #include <asm/pgalloc.h> #include <asm/tlbflush.h> #include "internal.h" @@ -3141,6 +3142,10 @@ static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, (transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_FILE_EXEC_THP_FLAG))
+#define file_exec_mthp_enabled() \ + (transparent_hugepage_flags & \ + (1<<TRANSPARENT_HUGEPAGE_FILE_EXEC_MTHP_FLAG)) + static inline void try_enable_file_exec_thp(struct vm_area_struct *vma, unsigned long *vm_flags, struct file *file) @@ -3157,6 +3162,24 @@ static inline void try_enable_file_exec_thp(struct vm_area_struct *vma, if (file_exec_thp_enabled()) hugepage_madvise(vma, vm_flags, MADV_HUGEPAGE); } + +static inline bool file_exec_can_enable_mthp(struct address_space *mapping, + unsigned long vm_flags) +{ +#ifndef arch_wants_exec_folio_order + return false; +#endif + if (!is_exec_mapping(vm_flags)) + return false; + + if (!mapping_large_folio_support(mapping)) + return false; + + if (!file_exec_mthp_enabled()) + return false; + + return true; +} #endif
/* @@ -3195,7 +3218,6 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER); return fpin; } -#endif
/* * Allow arch to request a preferred minimum folio order for executable @@ -3203,7 +3225,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) * can contpte-map the folio. Executable memory rarely benefits from * read-ahead anyway, due to its random access nature. */ - if (vm_flags & VM_EXEC) { + if (file_exec_can_enable_mthp(mapping, vm_flags)) { int order = arch_wants_exec_folio_order();
if (order >= 0) { @@ -3215,6 +3237,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) return fpin; } } +#endif
/* If we don't want any read-ahead, don't bother */ if (vm_flags & VM_RAND_READ) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0c61e7c7c2c1..4301e0fb6f3f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -426,31 +426,67 @@ static struct kobj_attribute hpage_pmd_size_attr = __ATTR_RO(hpage_pmd_size);
#ifdef CONFIG_READ_ONLY_THP_FOR_FS +#define FILE_EXEC_THP_ENABLE BIT(0) +#else +#define FILE_EXEC_THP_ENABLE 0 +#endif + +#define FILE_EXEC_MTHP_ENABLE BIT(1) +#define FILE_EXEC_THP_ALL (FILE_EXEC_THP_ENABLE | FILE_EXEC_MTHP_ENABLE) + +static void thp_exec_enabled_set(enum transparent_hugepage_flag flag, + bool enable) +{ + if (enable) + set_bit(flag, &transparent_hugepage_flags); + else + clear_bit(flag, &transparent_hugepage_flags); +} + static ssize_t thp_exec_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return single_hugepage_flag_show(kobj, attr, buf, - TRANSPARENT_HUGEPAGE_FILE_EXEC_THP_FLAG); + unsigned long val = 0; + +#ifdef CONFIG_READ_ONLY_THP_FOR_FS + if (test_bit(TRANSPARENT_HUGEPAGE_FILE_EXEC_THP_FLAG, + &transparent_hugepage_flags)) + val |= FILE_EXEC_THP_ENABLE; +#endif + + if (test_bit(TRANSPARENT_HUGEPAGE_FILE_EXEC_MTHP_FLAG, + &transparent_hugepage_flags)) + val |= FILE_EXEC_MTHP_ENABLE; + + return sysfs_emit(buf, "0x%lx\n", val); } static ssize_t thp_exec_enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - size_t ret = single_hugepage_flag_store(kobj, attr, buf, count, - TRANSPARENT_HUGEPAGE_FILE_EXEC_THP_FLAG); - if (ret > 0) { - int err = start_stop_khugepaged(); + unsigned long val; + int ret;
- if (err) - ret = err; - } + ret = kstrtoul(buf, 16, &val); + if (ret < 0) + return ret; + if (val & ~FILE_EXEC_THP_ALL) + return -EINVAL;
- return ret; +#ifdef CONFIG_READ_ONLY_THP_FOR_FS + thp_exec_enabled_set(TRANSPARENT_HUGEPAGE_FILE_EXEC_THP_FLAG, + val & FILE_EXEC_THP_ENABLE); + ret = start_stop_khugepaged(); + if (ret) + return ret; +#endif + thp_exec_enabled_set(TRANSPARENT_HUGEPAGE_FILE_EXEC_MTHP_FLAG, + val & FILE_EXEC_MTHP_ENABLE); + + return count; } static struct kobj_attribute thp_exec_enabled_attr = __ATTR_RW(thp_exec_enabled);
-#endif - static struct attribute *hugepage_attr[] = { &enabled_attr.attr, &defrag_attr.attr, @@ -459,9 +495,7 @@ static struct attribute *hugepage_attr[] = { #ifdef CONFIG_SHMEM &shmem_enabled_attr.attr, #endif -#ifdef CONFIG_READ_ONLY_THP_FOR_FS &thp_exec_enabled_attr.attr, -#endif NULL, };