hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IAHJKC CVE: NA

--------------------------------

Sometimes migrate THP is not beneficial, for example, when 64K page size is set on ARM64, THP will be 512M, migration may result in performance regression. This featrue add a interface to contrl THP migration when do numa balancing: /sys/kernel/mm/transparent_hugepage/numa_control

Default value is 0 which means keep default policy(will migrate). Write 1 to disable migrate THP while taskes still have chance to collect numa group info and may migrate.

The current control logic is applied for both autonuma and SPE based numa affinity.

Spark benchmark show 5% performance improvement after set 1 to the numa_control.

Fixes: 34387bcad1cd ("mm: numa-affinity: support THP migration") Signed-off-by: Nanyong Sun sunnanyong@huawei.com --- Documentation/admin-guide/mm/transhuge.rst | 8 ++++++ arch/arm64/Kconfig | 1 + include/linux/huge_mm.h | 13 +++++++++ mm/Kconfig | 10 +++++++ mm/huge_memory.c | 33 ++++++++++++++++++++++ mm/migrate.c | 3 ++ 6 files changed, 68 insertions(+)

diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 2bfb380e8380..fdff6c4247db 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -160,6 +160,14 @@ library) may want to know the size (in bytes) of a transparent hugepage::

cat /sys/kernel/mm/transparent_hugepage/hpage_pmd_size

+If CONFIG_THP_NUMA_CONTROL is on, user can control THP migration when +do numa balancing, 0 is default which means keep the default behavior, +writing 1 will disable thp migrate while tasks still have chance to +migrate:: + + echo 0 > /sys/kernel/mm/transparent_hugepage/numa_control + echo 1 > /sys/kernel/mm/transparent_hugepage/numa_control + khugepaged will be automatically started when transparent_hugepage/enabled is set to "always" or "madvise, and it'll be automatically shutdown if it's set to "never". diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index cae54a9bf65d..bde9ec4af773 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -216,6 +216,7 @@ config ARM64 select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK select HAVE_LIVEPATCH_WO_FTRACE + select THP_NUMA_CONTROL if ARM64_64K_PAGES help ARM 64-bit (AArch64) Linux support.

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index efb370e79ac3..d9dde313d267 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -498,6 +498,19 @@ static inline unsigned long thp_size(struct page *page) return PAGE_SIZE << thp_order(page); }

+#ifdef CONFIG_THP_NUMA_CONTROL +#define THP_DISABLE_NUMA_MIGRATE 1 +extern unsigned long thp_numa_control; +static inline bool thp_numa_migrate_disabled(void) +{ + return thp_numa_control == THP_DISABLE_NUMA_MIGRATE; +} +#else +static inline bool thp_numa_migrate_disabled(void) +{ + return false; +} +#endif /* * archs that select ARCH_WANTS_THP_SWAP but don't support THP_SWP due to * limitations in the implementation like arm64 MTE can override this to diff --git a/mm/Kconfig b/mm/Kconfig index ccbad233f2b1..cc43f5124cb3 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1038,6 +1038,16 @@ config NUMABALANCING_MEM_SAMPLING

if unsure, say N to disable the NUMABALANCING_MEM_SAMPLING.

+config THP_NUMA_CONTROL + bool "Control THP migration when numa balancing" + depends on NUMA_BALANCING && TRANSPARENT_HUGEPAGE + default n + help + Sometimes migrate THP is not beneficial, for example, when 64K page + size is set on ARM64, THP will be 512M, migration will be expensive. + This featrue add a switch to control the behavior of THP migration + when do numa balancing. + source "mm/damon/Kconfig"

endmenu diff --git a/mm/huge_memory.c b/mm/huge_memory.c index eb293d17a104..f286261f5525 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -316,6 +316,36 @@ static ssize_t hpage_pmd_size_show(struct kobject *kobj, static struct kobj_attribute hpage_pmd_size_attr = __ATTR_RO(hpage_pmd_size);

+#ifdef CONFIG_THP_NUMA_CONTROL +unsigned long thp_numa_control __read_mostly; + +static ssize_t numa_control_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%lu\n", READ_ONCE(thp_numa_control)); +} + +static ssize_t numa_control_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + unsigned long value; + int ret; + + ret = kstrtoul(buf, 10, &value); + if (ret < 0) + return ret; + if (value > THP_DISABLE_NUMA_MIGRATE) + return -EINVAL; + + WRITE_ONCE(thp_numa_control, value); + + return count; +} + +static struct kobj_attribute numa_control_attr = + __ATTR(numa_control, 0644, numa_control_show, numa_control_store); +#endif + static struct attribute *hugepage_attr[] = { &enabled_attr.attr, &defrag_attr.attr, @@ -323,6 +353,9 @@ static struct attribute *hugepage_attr[] = { &hpage_pmd_size_attr.attr, #ifdef CONFIG_SHMEM &shmem_enabled_attr.attr, +#endif +#ifdef CONFIG_THP_NUMA_CONTROL + &numa_control_attr.attr, #endif NULL, }; diff --git a/mm/migrate.c b/mm/migrate.c index 857c15e43497..cff5e11437d9 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2161,6 +2161,9 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, */ compound = PageTransHuge(page);

+ if (compound && thp_numa_migrate_disabled()) + return 0; + if (compound) new = alloc_misplaced_dst_page_thp; else

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IAHJKC CVE: NA

--------------------------------

The numa_scan_seq update depends on the numa scanning work done, which is skipped when sampling based numa affinity is on, so the numa_scan_seq will always be 0, skip the judge to avoid false migrate here.

Spark benchmark show 1%~2% performance improvement after applying this.

Fixes: bdc4701337d7 ("mm/mem_sampling.c: Drive NUMA balancing via mem_sampling access data") Signed-off-by: Nanyong Sun sunnanyong@huawei.com --- kernel/sched/fair.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2139edac2cb1..d22936de5714 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1423,6 +1423,20 @@ static inline unsigned long group_weight(struct task_struct *p, int nid, return 1000 * faults / total_faults; }

+static inline bool in_early_stage(struct task_struct *p, int early_seq) +{ + /* + * For sampling based autonuma, numa_scan_seq never update. Currently, + * just skip here to avoid false migrate. In the future, the real + * lifetime judgment can be implemented if the workloads are very + * sensitive to the starting stage of the process. + */ + if (numa_affinity_sampling_enabled()) + return false; + + return p->numa_scan_seq <= early_seq; +} + bool should_numa_migrate_memory(struct task_struct *p, struct page * page, int src_nid, int dst_cpu) { @@ -1439,7 +1453,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, * two full passes of the "multi-stage node selection" test that is * executed below. */ - if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && + if ((p->numa_preferred_nid == NUMA_NO_NODE || in_early_stage(p, 4)) && (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) return true;

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IAHJKC CVE: NA

--------------------------------

Fix follow build error when CONFIG_PROC_SYSCTL is not set: mm/mem_sampling.c: error: ‘sysctl_mem_sampling_enable’ undeclared here mm/mem_sampling.c: error: ‘sysctl_numabalancing_mem_sampling’ undeclared here

Fixes: 9878268b0b9f ("mm/mem_sampling.c: Add controlling interface for mem_sampling") Signed-off-by: Nanyong Sun sunnanyong@huawei.com --- mm/mem_sampling.c | 4 ---- 1 file changed, 4 deletions(-)

diff --git a/mm/mem_sampling.c b/mm/mem_sampling.c index 1d8a831be531..5bff12212471 100644 --- a/mm/mem_sampling.c +++ b/mm/mem_sampling.c @@ -369,7 +369,6 @@ static void set_numabalancing_mem_sampling_state(bool enabled) } }

-#ifdef CONFIG_PROC_SYSCTL int sysctl_numabalancing_mem_sampling(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -391,7 +390,6 @@ int sysctl_numabalancing_mem_sampling(struct ctl_table *table, int write,

return err; } -#endif #else static inline void set_numabalancing_mem_sampling_state(bool enabled) { @@ -423,7 +421,6 @@ static void set_mem_sampling_state(bool enabled) set_numabalancing_mem_sampling_state(enabled); }

-#ifdef CONFIG_PROC_SYSCTL static int sysctl_mem_sampling_enable(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -443,7 +440,6 @@ static int sysctl_mem_sampling_enable(struct ctl_table *table, int write, set_mem_sampling_state(state); return err; } -#endif

static struct ctl_table ctl_table[] = { {