hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC8KS8 CVE: NA -------------------------------- Introduce two control interfaces for enabling or disabling mem_sampling: - A /proc interface: /proc/sys/kernel/mem_sampling_enable - A kernel command-line parameter: mem_sampling_enable=enable/disable The proc interface allows runtime control of memory access sampling, enabling dynamic enable/disable during system operation. The command-line parameter provides early boot-time control, useful for environments where sampling should be enabled automatically at startup. These interfaces improve usability and flexibility for users and developers working with memory sampling features. Signed-off-by: Ze Zuo <zuoze1@huawei.com> Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> Signed-off-by: Shuang Yan <yanshuang7@huawei.com> --- drivers/arm/mm_monitor/mm_spe.c | 1 + drivers/perf/arm_spe_pmu.c | 4 + include/linux/mem_sampling.h | 9 ++ mm/mem_sampling.c | 168 +++++++++++++++++++++++++++++++- 4 files changed, 180 insertions(+), 2 deletions(-) diff --git a/drivers/arm/mm_monitor/mm_spe.c b/drivers/arm/mm_monitor/mm_spe.c index 38e236fed36c..f5c3668bb656 100644 --- a/drivers/arm/mm_monitor/mm_spe.c +++ b/drivers/arm/mm_monitor/mm_spe.c @@ -114,6 +114,7 @@ void mm_spe_buffer_free(void) mm_spe_percpu_buffer_free(cpu); } spe_probe_status -= 1; + set_mem_sampling_state(false); } EXPORT_SYMBOL_GPL(mm_spe_buffer_free); diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index d1b3b9783681..a59d70ea44c9 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -1364,6 +1364,10 @@ static int arm_spe_pmu_device_probe(struct platform_device *pdev) void arm_spe_set_user(enum arm_spe_user_e user) { __this_cpu_write(arm_spe_user, user); + if (user == SPE_USER_PERF) + mem_sampling_user_switch_process(USER_SWITCH_AWAY_FROM_MEM_SAMPLING); + else + mem_sampling_user_switch_process(USER_SWITCH_BACK_TO_MEM_SAMPLING); __arm_spe_pmu_reset_local(); } #endif diff --git a/include/linux/mem_sampling.h b/include/linux/mem_sampling.h index 573c45cc929d..9ec16a218756 100644 --- a/include/linux/mem_sampling.h +++ b/include/linux/mem_sampling.h @@ -66,6 +66,12 @@ enum mem_sampling_type_enum { MEM_SAMPLING_UNSUPPORTED }; +enum user_switch_type { + USER_SWITCH_AWAY_FROM_MEM_SAMPLING, + USER_SWITCH_BACK_TO_MEM_SAMPLING, + USER_SWITCH_TYPE_MAX, +}; + #ifdef CONFIG_ARM_SPE_MEM_SAMPLING int mm_spe_start(void); void mm_spe_stop(void); @@ -89,6 +95,8 @@ static inline int mm_spe_enabled(void) { return 0; } #if IS_ENABLED(CONFIG_MEM_SAMPLING) void mem_sampling_process(void); void arm_spe_set_user(enum arm_spe_user_e user); +void set_mem_sampling_state(bool enabled); +void mem_sampling_user_switch_process(enum user_switch_type type); static inline bool spe_user_is_perf(void) { return __this_cpu_read(arm_spe_user) == SPE_USER_PERF; @@ -99,6 +107,7 @@ static inline bool spe_user_is_mem_sampling(void) } void mem_sampling_sched_in(struct task_struct *prev, struct task_struct *curr); #else +static inline void set_mem_sampling_state(bool enabled) { } static inline void mem_sampling_sched_in(struct task_struct *prev, struct task_struct *curr) { } #endif /* CONFIG_MEM_SAMPLING */ #endif /* __MEM_SAMPLING_H */ diff --git a/mm/mem_sampling.c b/mm/mem_sampling.c index 9ebc14dee570..5b18747ee8fd 100644 --- a/mm/mem_sampling.c +++ b/mm/mem_sampling.c @@ -19,12 +19,24 @@ #include <linux/mm.h> #include <linux/mem_sampling.h> +#define MEM_SAMPLING_DISABLED 0x0 +#define MEM_SAMPLING_NORMAL 0x1 + struct mem_sampling_ops_struct mem_sampling_ops; +static int mem_sampling_override __initdata; +static int sysctl_mem_sampling_mode; /* keep track of who use the SPE */ DEFINE_PER_CPU(enum arm_spe_user_e, arm_spe_user); EXPORT_PER_CPU_SYMBOL_GPL(arm_spe_user); +enum mem_sampling_saved_state_e { + MEM_SAMPLING_STATE_ENABLE, + MEM_SAMPLING_STATE_DISABLE, + MEM_SAMPLING_STATE_EMPTY, +}; +enum mem_sampling_saved_state_e mem_sampling_saved_state = MEM_SAMPLING_STATE_EMPTY; + /* * Callbacks should be registered using mem_sampling_record_cb_register() * by NUMA, DAMON and etc during their initialisation. @@ -68,8 +80,11 @@ void mem_sampling_record_cb_unregister(mem_sampling_record_cb_type cb) } } +DEFINE_STATIC_KEY_FALSE(mem_sampling_access_hints); void mem_sampling_sched_in(struct task_struct *prev, struct task_struct *curr) { + if (!static_branch_unlikely(&mem_sampling_access_hints)) + return; if (!mem_sampling_ops.sampling_start) return; @@ -102,8 +117,11 @@ void mem_sampling_process(void) } } out: - mem_sampling_ops.sampling_continue(); - + /* if mem_sampling_access_hints is set to false, stop sampling */ + if (static_branch_unlikely(&mem_sampling_access_hints)) + mem_sampling_ops.sampling_continue(); + else + mem_sampling_ops.sampling_stop(); } EXPORT_SYMBOL_GPL(mem_sampling_process); @@ -116,6 +134,149 @@ static inline enum mem_sampling_type_enum mem_sampling_get_type(void) #endif } +static void __set_mem_sampling_state(bool enabled) +{ + if (enabled) + static_branch_enable(&mem_sampling_access_hints); + else + static_branch_disable(&mem_sampling_access_hints); +} + +void set_mem_sampling_state(bool enabled) +{ + if (mem_sampling_saved_state != MEM_SAMPLING_STATE_EMPTY) { + mem_sampling_saved_state = enabled ? MEM_SAMPLING_STATE_ENABLE : + MEM_SAMPLING_STATE_DISABLE; + return; + } + + if (!mem_sampling_ops.sampling_start || !mm_spe_enabled()) + return; + if (enabled) + sysctl_mem_sampling_mode = MEM_SAMPLING_NORMAL; + else + sysctl_mem_sampling_mode = MEM_SAMPLING_DISABLED; + __set_mem_sampling_state(enabled); +} + +void mem_sampling_user_switch_process(enum user_switch_type type) +{ + bool state, mm_spe_is_perf_user = false; + int cpu; + + if (type >= USER_SWITCH_TYPE_MAX) { + pr_err("user switch type error.\n"); + return; + } + + for_each_possible_cpu(cpu) { + if (per_cpu(arm_spe_user, cpu) == SPE_USER_PERF) { + mm_spe_is_perf_user = true; + break; + } + } + + if (type == USER_SWITCH_AWAY_FROM_MEM_SAMPLING) { + /* save state only the status when leave mem_sampling for the first time */ + if (mem_sampling_saved_state != MEM_SAMPLING_STATE_EMPTY) + return; + + if (static_branch_unlikely(&mem_sampling_access_hints)) + mem_sampling_saved_state = MEM_SAMPLING_STATE_ENABLE; + else + mem_sampling_saved_state = MEM_SAMPLING_STATE_DISABLE; + + pr_debug("user switch away from mem_sampling, %s is saved, set to disable.\n", + mem_sampling_saved_state ? "disabled" : "enabled"); + + set_mem_sampling_state(false); + } else { + /* If the state is not backed up, do not restore it */ + if (mem_sampling_saved_state == MEM_SAMPLING_STATE_EMPTY || mm_spe_is_perf_user) + return; + + state = (mem_sampling_saved_state == MEM_SAMPLING_STATE_ENABLE) ? true : false; + set_mem_sampling_state(state); + mem_sampling_saved_state = MEM_SAMPLING_STATE_EMPTY; + + pr_debug("user switch back to mem_sampling, set to saved %s.\n", + state ? "enalbe" : "disable"); + } +} +EXPORT_SYMBOL_GPL(mem_sampling_user_switch_process); + +#ifdef CONFIG_PROC_SYSCTL +static int proc_mem_sampling_enable(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table t; + int err; + int state = sysctl_mem_sampling_mode; + + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + t = *table; + t.data = &state; + err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (err < 0) + return err; + if (write) + set_mem_sampling_state(state); + return err; +} + +static struct ctl_table mem_sampling_sysctls[] = { + { + .procname = "mem_sampling_enable", + .data = NULL, /* filled in by handler */ + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_mem_sampling_enable, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + {} +}; + +static void __init mem_sampling_sysctl_init(void) +{ + register_sysctl_init("kernel", mem_sampling_sysctls); +} +#else +#define mem_sampling_sysctl_init() do { } while (0) +#endif + +static void __init check_mem_sampling_enable(void) +{ + bool mem_sampling_default = false; + + /* Parsed by setup_mem_sampling. override == 1 enables, -1 disables */ + if (mem_sampling_override) + set_mem_sampling_state(mem_sampling_override == 1); + else + set_mem_sampling_state(mem_sampling_default); +} + +static int __init setup_mem_sampling_enable(char *str) +{ + int ret = 0; + + if (!str) + goto out; + + if (!strcmp(str, "enable")) { + mem_sampling_override = 1; + ret = 1; + } +out: + if (!ret) + pr_warn("Unable to parse mem_sampling=\n"); + + return ret; +} +__setup("mem_sampling=", setup_mem_sampling_enable); + static int __init mem_sampling_init(void) { enum mem_sampling_type_enum mem_sampling_type = mem_sampling_get_type(); @@ -135,8 +296,11 @@ static int __init mem_sampling_init(void) default: pr_info("unsupport hardware pmu type(%d), disable access hint!\n", mem_sampling_type); + set_mem_sampling_state(false); return -ENODEV; } + check_mem_sampling_enable(); + mem_sampling_sysctl_init(); for_each_possible_cpu(cpu) per_cpu(arm_spe_user, cpu) = SPE_USER_MEM_SAMPLING; -- 2.25.1