From: Hui Tang tanghui20@huawei.com

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I7BQZ0 CVE: NA

----------------------------------------

We want to dynamically expand or shrink the affinity range of tasks based on the CPU topology level while meeting the minimum resource requirements of tasks.

We divide several level of affinity domains according to sched domains:

level4 * SOCKET [ ] level3 * DIE [ ] level2 * MC [ ] [ ] level1 * SMT [ ] [ ] [ ] [ ] level0 * CPU 0 1 2 3 4 5 6 7

Whether users tend to choose power saving or performance will affect strategy of adjusting affinity, when selecting the power saving mode, we will choose a more appropriate affinity based on the energy model to reduce power consumption, while considering the QOS of resources such as CPU and memory consumption, for instance, if the current task CPU load is less than required, smart grid will judge whether to aggregate tasks together into a smaller range or not according to energy model.

The main difference from EAS is that we pay more attention to the impact of power consumption brought by such as cpuidle and DVFS, and classify tasks to reduce interference and ensure resource QOS in each divided unit, which are more suitable for general-purpose on non-heterogeneous CPUs.

-------- -------- -------- | group0 | | group1 | | group2 | -------- -------- -------- | | | v | v ---------------------+----- ----------------- | ---v-- | | | DIE0 | MC1 | | | DIE1 | ------ | | --------------------------- -----------------

We regularly count the resource satisfaction of groups, and adjust the affinity, scheduling balance and migrating memory will be considered based on memory location for better meetting resource requirements.

Signed-off-by: Hui Tang tanghui20@huawei.com Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Reviewed-by: Chen Hui judy.chenhui@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- fs/proc/array.c | 13 ++ include/linux/sched.h | 13 ++ include/linux/sched/sysctl.h | 4 + init/Kconfig | 13 ++ kernel/sched/core.c | 146 ++++++++++++ kernel/sched/fair.c | 426 ++++++++++++++++++++++++++++++++++- kernel/sched/sched.h | 47 ++++ kernel/sysctl.c | 9 + 8 files changed, 669 insertions(+), 2 deletions(-)

diff --git a/fs/proc/array.c b/fs/proc/array.c index 18a4588c35be..989f7602035c 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -389,6 +389,16 @@ static void task_cpus_allowed(struct seq_file *m, struct task_struct *task) cpumask_pr_args(task->cpus_ptr)); }

+#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY +static void task_cpus_preferred(struct seq_file *m, struct task_struct *task) +{ + seq_printf(m, "Cpus_preferred:\t%*pb\n", + cpumask_pr_args(task->prefer_cpus)); + seq_printf(m, "Cpus_preferred_list:\t%*pbl\n", + cpumask_pr_args(task->prefer_cpus)); +} +#endif + static inline void task_core_dumping(struct seq_file *m, struct mm_struct *mm) { seq_put_decimal_ull(m, "CoreDumping:\t", !!mm->core_state); @@ -427,6 +437,9 @@ int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, task_cpus_allowed(m, task); cpuset_task_status_allowed(m, task); task_context_switch_counts(m, task); +#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY + task_cpus_preferred(m, task); +#endif return 0; }

diff --git a/include/linux/sched.h b/include/linux/sched.h index d39427f8044d..44db39317cef 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2255,6 +2255,19 @@ void sched_prefer_cpus_free(struct task_struct *p); void dynamic_affinity_enable(void); #endif

+#ifdef CONFIG_QOS_SCHED_SMART_GRID +extern struct static_key __smart_grid_used; +static inline bool smart_grid_used(void) +{ + return static_key_false(&__smart_grid_used); +} +#else +static inline bool smart_grid_used(void) +{ + return false; +} +#endif + #ifdef CONFIG_BPF_SCHED extern void sched_settag(struct task_struct *tsk, s64 tag);

diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 4d6bbc0934c9..31c4a84ce3df 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -35,6 +35,10 @@ extern unsigned int sysctl_sched_child_runs_first; extern int sysctl_sched_util_low_pct; #endif

+#ifdef CONFIG_QOS_SCHED_SMART_GRID +extern int sysctl_affinity_adjust_delay_ms; +#endif + enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, SCHED_TUNABLESCALING_LOG, diff --git a/init/Kconfig b/init/Kconfig index 83714edd7bf9..bb9063807556 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1078,6 +1078,19 @@ config UCLAMP_TASK_GROUP

If in doubt, say N.

+config QOS_SCHED_SMART_GRID + bool "qos smart grid scheduler" + depends on FAIR_GROUP_SCHED && QOS_SCHED_DYNAMIC_AFFINITY + default n + help + This feature is used for power consumption tuning in server scenario. + This can be divided into the following aspects: + 1. User interface, manage user needs. + 2. Collect tasks' features to ensure key tasks' QOS. + 3. Weaken the influence the impact of CPU frequency and cpuidle + adjustment on tasks. + 4. Docking EAS (Energy Aware Scheduling) model. + config CGROUP_PIDS bool "PIDs controller" help diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8c8c946ee1de..9855cb5458c0 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8009,6 +8009,7 @@ int sched_cpu_activate(unsigned int cpu) static_branch_inc_cpuslocked(&sched_smt_present); #endif set_cpu_active(cpu, true); + tg_update_affinity_domains(cpu, 1);

if (sched_smp_initialized) { sched_domains_numa_masks_set(cpu); @@ -8071,6 +8072,7 @@ int sched_cpu_deactivate(unsigned int cpu) return ret; } sched_domains_numa_masks_clear(cpu); + tg_update_affinity_domains(cpu, 0); return 0; }

@@ -8140,6 +8142,8 @@ void __init sched_init_smp(void) init_sched_dl_class();

sched_smp_initialized = true; + + init_auto_affinity(&root_task_group); }

static int __init migration_init(void) @@ -8786,6 +8790,9 @@ void sched_move_task(struct task_struct *tsk) DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; struct rq_flags rf; struct rq *rq; +#ifdef CONFIG_QOS_SCHED_SMART_GRID + struct affinity_domain *ad; +#endif

rq = task_rq_lock(tsk, &rf); update_rq_clock(rq); @@ -8813,6 +8820,13 @@ void sched_move_task(struct task_struct *tsk) }

task_rq_unlock(rq, tsk, &rf); + +#ifdef CONFIG_QOS_SCHED_SMART_GRID + if (smart_grid_used()) { + ad = &task_group(tsk)->auto_affinity->ad; + set_prefer_cpus_ptr(tsk, ad->domains[ad->curr_level]); + } +#endif }

static inline struct task_group *css_tg(struct cgroup_subsys_state *css) @@ -9512,6 +9526,117 @@ static inline s64 cpu_smt_expell_read(struct cgroup_subsys_state *css, } #endif

+#ifdef CONFIG_QOS_SCHED_SMART_GRID +int tg_set_dynamic_affinity_mode(struct task_group *tg, u64 mode) +{ + struct auto_affinity *auto_affi = tg->auto_affinity; + int ret = 0; + + raw_spin_lock_irq(&auto_affi->lock); + + /* auto mode*/ + if (mode == 1) { + start_auto_affinity(auto_affi); + } else if (mode == 0) { + stop_auto_affinity(auto_affi); + } else { + raw_spin_unlock_irq(&auto_affi->lock); + return -EINVAL; + } + + auto_affi->mode = mode; + raw_spin_unlock_irq(&auto_affi->lock); + + return ret; +} + +static u64 cpu_affinity_mode_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return tg->auto_affinity->mode; +} + +static int cpu_affinity_mode_write_u64(struct cgroup_subsys_state *css, + struct cftype *cftype, u64 mode) +{ + return tg_set_dynamic_affinity_mode(css_tg(css), mode); +} + +int tg_set_affinity_period(struct task_group *tg, u64 period_ms) +{ + if (period_ms > U64_MAX / NSEC_PER_MSEC) + return -EINVAL; + + raw_spin_lock_irq(&tg->auto_affinity->lock); + tg->auto_affinity->period = ms_to_ktime(period_ms); + raw_spin_unlock_irq(&tg->auto_affinity->lock); + return 0; +} + +u64 tg_get_affinity_period(struct task_group *tg) +{ + return ktime_to_ms(tg->auto_affinity->period); +} + +static int cpu_affinity_period_write_uint(struct cgroup_subsys_state *css, + struct cftype *cftype, u64 period) +{ + return tg_set_affinity_period(css_tg(css), period); +} + +static u64 cpu_affinity_period_read_uint(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return tg_get_affinity_period(css_tg(css)); +} + +static int cpu_affinity_domain_mask_write_u64(struct cgroup_subsys_state *css, + struct cftype *cftype, + u64 mask) +{ + struct task_group *tg = css_tg(css); + struct affinity_domain *ad = &tg->auto_affinity->ad; + u16 full = (1 << ad->dcount) - 1; + + if (mask > full) + return -EINVAL; + + raw_spin_lock_irq(&tg->auto_affinity->lock); + ad->domain_mask = mask; + raw_spin_unlock_irq(&tg->auto_affinity->lock); + return 0; +} + +static u64 cpu_affinity_domain_mask_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return tg->auto_affinity->ad.domain_mask; +} + +static int cpu_affinity_stat_show(struct seq_file *sf, void *v) +{ + struct task_group *tg = css_tg(seq_css(sf)); + struct auto_affinity *auto_affi = tg->auto_affinity; + struct affinity_domain *ad = &auto_affi->ad; + int i; + + seq_printf(sf, "period_active %d\n", auto_affi->period_active); + seq_printf(sf, "dcount %d\n", ad->dcount); + seq_printf(sf, "domain_mask 0x%x\n", ad->domain_mask); + seq_printf(sf, "curr_level %d\n", ad->curr_level); + for (i = 0; i < ad->dcount; i++) + seq_printf(sf, "sd_level %d, cpu list %*pbl, stay_cnt %llu\n", + i, cpumask_pr_args(ad->domains[i]), + schedstat_val(ad->stay_cnt[i])); + + return 0; +} +#endif /* CONFIG_QOS_SCHED_SMART_GRID */ + #ifdef CONFIG_QOS_SCHED static int tg_change_scheduler(struct task_group *tg, void *data) { @@ -9673,6 +9798,27 @@ static struct cftype cpu_legacy_files[] = { .write_u64 = cpu_shares_write_u64, }, #endif +#ifdef CONFIG_QOS_SCHED_SMART_GRID + { + .name = "dynamic_affinity_mode", + .read_u64 = cpu_affinity_mode_read_u64, + .write_u64 = cpu_affinity_mode_write_u64, + }, + { + .name = "affinity_period_ms", + .read_u64 = cpu_affinity_period_read_uint, + .write_u64 = cpu_affinity_period_write_uint, + }, + { + .name = "affinity_domain_mask", + .read_u64 = cpu_affinity_domain_mask_read_u64, + .write_u64 = cpu_affinity_domain_mask_write_u64, + }, + { + .name = "affinity_stat", + .seq_show = cpu_affinity_stat_show, + }, +#endif #ifdef CONFIG_CFS_BANDWIDTH { .name = "cfs_quota_us", diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6ab0296e7be2..db18ebdd313a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5810,6 +5810,416 @@ static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}

#endif /* CONFIG_CFS_BANDWIDTH */

+#ifdef CONFIG_QOS_SCHED_SMART_GRID +#define AUTO_AFFINITY_DEFAULT_PERIOD_MS 2000 +#define IS_DOMAIN_SET(level, mask) ((1 << (level)) & (mask)) + +static inline unsigned long cpu_util(int cpu); +static unsigned long capacity_of(int cpu); +static int sched_idle_cpu(int cpu); +static unsigned long cpu_runnable(struct rq *rq); + +int sysctl_affinity_adjust_delay_ms = 5000; + +struct static_key __smart_grid_used; + +static void smart_grid_usage_inc(void) +{ + static_key_slow_inc_cpuslocked(&__smart_grid_used); +} + +static void smart_grid_usage_dec(void) +{ + static_key_slow_dec_cpuslocked(&__smart_grid_used); +} + +static void tg_update_task_prefer_cpus(struct task_group *tg) +{ + struct affinity_domain *ad = &tg->auto_affinity->ad; + struct task_struct *task; + struct css_task_iter it; + + css_task_iter_start(&tg->css, 0, &it); + while ((task = css_task_iter_next(&it))) { + if (tg == &root_task_group && !task->mm) + continue; + + set_prefer_cpus_ptr(task, ad->domains[ad->curr_level]); + } + css_task_iter_end(&it); +} + +static void affinity_domain_up(struct task_group *tg) +{ + struct affinity_domain *ad = &tg->auto_affinity->ad; + u16 level = ad->curr_level; + + if (ad->curr_level >= ad->dcount - 1) + return; + + while (level < ad->dcount) { + if (IS_DOMAIN_SET(level + 1, ad->domain_mask)) { + ad->curr_level = level + 1; + break; + } + level++; + } + + if (level == ad->dcount) + return; + + tg_update_task_prefer_cpus(tg); +} + +static void affinity_domain_down(struct task_group *tg) +{ + struct affinity_domain *ad = &tg->auto_affinity->ad; + u16 level = ad->curr_level; + + if (ad->curr_level <= 0) + return; + + while (level > 0) { + if (IS_DOMAIN_SET(level - 1, ad->domain_mask)) { + ad->curr_level = level - 1; + break; + } + level--; + } + + if (!level) + return; + + tg_update_task_prefer_cpus(tg); +} + +static enum hrtimer_restart sched_auto_affi_period_timer(struct hrtimer *timer) +{ + struct auto_affinity *auto_affi = + container_of(timer, struct auto_affinity, period_timer); + struct task_group *tg = auto_affi->tg; + struct affinity_domain *ad = &auto_affi->ad; + struct cpumask *span = ad->domains[ad->curr_level]; + unsigned long util_avg_sum = 0; + unsigned long tg_capacity = 0; + unsigned long flags; + int cpu; + + for_each_cpu(cpu, span) { + util_avg_sum += cpu_util(cpu); + tg_capacity += capacity_of(cpu); + } + + if (unlikely(!tg_capacity)) + return HRTIMER_RESTART; + + raw_spin_lock_irqsave(&auto_affi->lock, flags); + if (util_avg_sum * 100 > tg_capacity * sysctl_sched_util_low_pct) { + affinity_domain_up(tg); + } else if (util_avg_sum * 100 < tg_capacity * + sysctl_sched_util_low_pct / 2) { + affinity_domain_down(tg); + } + + schedstat_inc(ad->stay_cnt[ad->curr_level]); + hrtimer_forward_now(timer, auto_affi->period); + raw_spin_unlock_irqrestore(&auto_affi->lock, flags); + return HRTIMER_RESTART; +} + +static int tg_update_affinity_domain_down(struct task_group *tg, void *data) +{ + struct auto_affinity *auto_affi = tg->auto_affinity; + struct affinity_domain *ad; + int *cpu_state = data; + unsigned long flags; + int i; + + if (!auto_affi) + return 0; + + ad = &tg->auto_affinity->ad; + raw_spin_lock_irqsave(&auto_affi->lock, flags); + + for (i = 0; i < ad->dcount; i++) { + if (!cpumask_test_cpu(cpu_state[0], ad->domains_orig[i])) + continue; + + /* online */ + if (cpu_state[1]) + cpumask_set_cpu(cpu_state[0], ad->domains[i]); + else + cpumask_clear_cpu(cpu_state[0], ad->domains[i]); + } + raw_spin_unlock_irqrestore(&auto_affi->lock, flags); + + if (!smart_grid_used()) + return 0; + + if (auto_affi->mode) + tg_update_task_prefer_cpus(tg); + return 0; +} + +void tg_update_affinity_domains(int cpu, int online) +{ + int cpu_state[2]; + + cpu_state[0] = cpu; + cpu_state[1] = online; + + rcu_read_lock(); + walk_tg_tree(tg_update_affinity_domain_down, tg_nop, cpu_state); + rcu_read_unlock(); +} + +void start_auto_affinity(struct auto_affinity *auto_affi) +{ + struct task_group *tg = auto_affi->tg; + ktime_t delay_ms; + + if (auto_affi->period_active == 1) + return; + + tg_update_task_prefer_cpus(tg); + + auto_affi->period_active = 1; + delay_ms = ms_to_ktime(sysctl_affinity_adjust_delay_ms); + hrtimer_forward_now(&auto_affi->period_timer, delay_ms); + hrtimer_start_expires(&auto_affi->period_timer, + HRTIMER_MODE_ABS_PINNED); + smart_grid_usage_inc(); +} + +void stop_auto_affinity(struct auto_affinity *auto_affi) +{ + struct task_group *tg = auto_affi->tg; + struct affinity_domain *ad = &auto_affi->ad; + + if (auto_affi->period_active == 0) + return; + + hrtimer_cancel(&auto_affi->period_timer); + auto_affi->period_active = 0; + ad->curr_level = ad->dcount > 0 ? ad->dcount - 1 : 0; + + tg_update_task_prefer_cpus(tg); + smart_grid_usage_dec(); +} + +static struct sched_group *sd_find_idlest_group(struct sched_domain *sd) +{ + struct sched_group *idlest = NULL, *group = sd->groups; + unsigned long min_runnable_load = ULONG_MAX; + unsigned long min_avg_load = ULONG_MAX; + int imbalance_scale = 100 + (sd->imbalance_pct-100)/2; + unsigned long imbalance = scale_load_down(NICE_0_LOAD) * + (sd->imbalance_pct-100) / 100; + + do { + unsigned long load, avg_load, runnable_load; + int i; + + avg_load = 0; + runnable_load = 0; + + for_each_cpu(i, sched_group_span(group)) { + load = cpu_runnable(cpu_rq(i)); + runnable_load += load; + avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs); + } + + avg_load = (avg_load * SCHED_CAPACITY_SCALE) / + group->sgc->capacity; + runnable_load = (runnable_load * SCHED_CAPACITY_SCALE) / + group->sgc->capacity; + + if (min_runnable_load > (runnable_load + imbalance)) { + min_runnable_load = runnable_load; + min_avg_load = avg_load; + idlest = group; + } else if ((runnable_load < (min_runnable_load + imbalance)) && + (100*min_avg_load > imbalance_scale*avg_load)) { + min_avg_load = avg_load; + idlest = group; + } + } while (group = group->next, group != sd->groups); + + return idlest ? idlest : group; +} + +static int group_find_idlest_cpu(struct sched_group *group) +{ + int least_loaded_cpu = cpumask_first(sched_group_span(group)); + unsigned long load, min_load = ULONG_MAX; + unsigned int min_exit_latency = UINT_MAX; + u64 latest_idle_timestamp = 0; + int shallowest_idle_cpu = -1; + int i; + + if (group->group_weight == 1) + return least_loaded_cpu; + + for_each_cpu(i, sched_group_span(group)) { + if (sched_idle_cpu(i)) + return i; + + if (available_idle_cpu(i)) { + struct rq *rq = cpu_rq(i); + struct cpuidle_state *idle = idle_get_state(rq); + + if (idle && idle->exit_latency < min_exit_latency) { + min_exit_latency = idle->exit_latency; + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } else if ((!idle || + idle->exit_latency == min_exit_latency) && + rq->idle_stamp > latest_idle_timestamp) { + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } + } else if (shallowest_idle_cpu == -1) { + load = cpu_runnable(cpu_rq(i)); + if (load < min_load) { + min_load = load; + least_loaded_cpu = i; + } + } + } + + return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : + least_loaded_cpu; +} + +void free_affinity_domains(struct affinity_domain *ad) +{ + int i; + + for (i = 0; i < ad->dcount; i++) { + kfree(ad->domains[i]); + ad->domains[i] = NULL; + } + ad->dcount = 0; +} + +static int init_affinity_domains_orig(struct affinity_domain *ad) +{ + int i, j; + + for (i = 0; i < ad->dcount; i++) { + ad->domains_orig[i] = kmalloc(sizeof(cpumask_t), GFP_KERNEL); + if (!ad->domains_orig[i]) + goto err; + + cpumask_copy(ad->domains_orig[i], ad->domains[i]); + } + + return 0; +err: + for (j = 0; j < i; j++) { + kfree(ad->domains_orig[j]); + ad->domains_orig[i] = NULL; + } + return -ENOMEM; +} + +static int init_affinity_domains(struct affinity_domain *ad) +{ + struct sched_domain *sd = NULL, *tmp; + struct sched_group *idlest = NULL; + int ret = -ENOMEM; + int dcount = 0; + int i = 0; + int cpu; + + rcu_read_lock(); + cpu = cpumask_first_and(cpu_active_mask, + housekeeping_cpumask(HK_FLAG_DOMAIN)); + for_each_domain(cpu, tmp) { + sd = tmp; + dcount++; + } + + if (!sd) { + ad->dcount = 0; + rcu_read_unlock(); + return -EINVAL; + } + rcu_read_unlock(); + + for (i = 0; i < dcount; i++) { + ad->domains[i] = kmalloc(sizeof(cpumask_t), GFP_KERNEL); + if (!ad->domains[i]) + goto err; + } + + rcu_read_lock(); + idlest = sd_find_idlest_group(sd); + cpu = group_find_idlest_cpu(idlest); + i = 0; + for_each_domain(cpu, tmp) { + cpumask_copy(ad->domains[i], sched_domain_span(tmp)); + __schedstat_set(ad->stay_cnt[i], 0); + i++; + } + rcu_read_unlock(); + + ad->dcount = dcount; + ad->curr_level = ad->dcount > 0 ? ad->dcount - 1 : 0; + ad->domain_mask = (1 << ad->dcount) - 1; + + ret = init_affinity_domains_orig(ad); + if (ret) + goto err; + + return 0; +err: + free_affinity_domains(ad); + return ret; +} + +int init_auto_affinity(struct task_group *tg) +{ + struct auto_affinity *auto_affi; + int ret; + + auto_affi = kmalloc(sizeof(*auto_affi), GFP_KERNEL); + if (!auto_affi) + return -ENOMEM; + + raw_spin_lock_init(&auto_affi->lock); + auto_affi->mode = 0; + auto_affi->period_active = 0; + auto_affi->period = ms_to_ktime(AUTO_AFFINITY_DEFAULT_PERIOD_MS); + hrtimer_init(&auto_affi->period_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_ABS_PINNED); + auto_affi->period_timer.function = sched_auto_affi_period_timer; + + ret = init_affinity_domains(&auto_affi->ad); + if (ret) { + kfree(auto_affi); + return ret; + } + + auto_affi->tg = tg; + tg->auto_affinity = auto_affi; + return 0; +} + +static void destroy_auto_affinity(struct task_group *tg) +{ + struct auto_affinity *auto_affi = tg->auto_affinity; + + hrtimer_cancel(&auto_affi->period_timer); + free_affinity_domains(&auto_affi->ad); + + kfree(tg->auto_affinity); + tg->auto_affinity = NULL; +} +#else +static void destroy_auto_affinity(struct task_group *tg) {} +#endif + /************************************************** * CFS operations on tasks: */ @@ -7327,7 +7737,7 @@ static inline unsigned long taskgroup_cpu_util(struct task_group *tg, /* * set_task_select_cpus: select the cpu range for task * @p: the task whose available cpu range will to set - * @idlest_cpu: the cpu which is the idlest in prefer cpus + *uto_affinity_used @idlest_cpu: the cpu which is the idlest in prefer cpus * * If sum of 'util_avg' among 'preferred_cpus' lower than the percentage * 'sysctl_sched_util_low_pct' of 'preferred_cpus' capacity, select @@ -7351,6 +7761,13 @@ static void set_task_select_cpus(struct task_struct *p, int *idlest_cpu, if (!prefer_cpus_valid(p)) return;

+ if (smart_grid_used()) { + p->select_cpus = p->prefer_cpus; + if (idlest_cpu) + *idlest_cpu = cpumask_first(p->select_cpus); + return; + } + rcu_read_lock(); tg = task_group(p); for_each_cpu(cpu, p->prefer_cpus) { @@ -12926,6 +13343,7 @@ void free_fair_sched_group(struct task_group *tg) int i;

destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); + destroy_auto_affinity(tg);

for_each_possible_cpu(i) { #ifdef CONFIG_QOS_SCHED @@ -12949,7 +13367,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) { struct sched_entity *se; struct cfs_rq *cfs_rq; - int i; + int i, ret;

tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); if (!tg->cfs_rq) @@ -12961,6 +13379,9 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) tg->shares = NICE_0_LOAD;

init_cfs_bandwidth(tg_cfs_bandwidth(tg)); + ret = init_auto_affinity(tg); + if (ret) + goto err;

for_each_possible_cpu(i) { cfs_rq = kzalloc_node(sizeof(struct cfs_rq), @@ -12983,6 +13404,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) err_free_rq: kfree(cfs_rq); err: + destroy_auto_affinity(tg); return 0; }

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 91ae933c20be..317aea6505a3 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -400,6 +400,34 @@ struct cfs_bandwidth { #endif };

+ +#ifdef CONFIG_QOS_SCHED_SMART_GRID +#define AD_LEVEL_MAX 8 + +struct affinity_domain { + int dcount; + int curr_level; + u32 domain_mask; +#ifdef CONFIG_SCHEDSTATS + u64 stay_cnt[AD_LEVEL_MAX]; +#endif + struct cpumask *domains[AD_LEVEL_MAX]; + struct cpumask *domains_orig[AD_LEVEL_MAX]; +}; +#endif + +struct auto_affinity { +#ifdef CONFIG_QOS_SCHED_SMART_GRID + raw_spinlock_t lock; + u64 mode; + ktime_t period; + struct hrtimer period_timer; + int period_active; + struct affinity_domain ad; + struct task_group *tg; +#endif +}; + /* Task group related information */ struct task_group { struct cgroup_subsys_state css; @@ -471,7 +499,11 @@ struct task_group { #else KABI_RESERVE(3) #endif +#if defined(CONFIG_QOS_SCHED_SMART_GRID) && !defined(__GENKSYMS__) + KABI_USE(4, struct auto_affinity *auto_affinity) +#else KABI_RESERVE(4) +#endif };

#ifdef CONFIG_FAIR_GROUP_SCHED @@ -542,6 +574,21 @@ extern void sched_offline_group(struct task_group *tg);

extern void sched_move_task(struct task_struct *tsk);

+#ifdef CONFIG_QOS_SCHED_SMART_GRID +extern void start_auto_affinity(struct auto_affinity *auto_affi); +extern void stop_auto_affinity(struct auto_affinity *auto_affi); +extern int init_auto_affinity(struct task_group *tg); +extern void tg_update_affinity_domains(int cpu, int online); + +#else +static inline int init_auto_affinity(struct task_group *tg) +{ + return 0; +} + +static inline void tg_update_affinity_domains(int cpu, int online) {} +#endif + #ifdef CONFIG_FAIR_GROUP_SCHED extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);

diff --git a/kernel/sysctl.c b/kernel/sysctl.c index edb80160491f..b7a0e9a34035 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2802,6 +2802,15 @@ static struct ctl_table kern_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &one_hundred, }, +#endif +#ifdef CONFIG_QOS_SCHED_SMART_GRID + { + .procname = "affinity_adjust_delay_ms", + .data = &sysctl_affinity_adjust_delay_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #endif { } };

From: Hui Tang tanghui20@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7D98G CVE: NA

----------------------------------------

smart_grid_usage_dec() should called when free taskgroup if the mode is auto.

Signed-off-by: Hui Tang tanghui20@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- kernel/sched/fair.c | 3 +++ 1 file changed, 3 insertions(+)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2e7d27824c3a..73c0ca42b548 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6215,6 +6215,9 @@ static void destroy_auto_affinity(struct task_group *tg) { struct auto_affinity *auto_affi = tg->auto_affinity;

+ if (auto_affi->mode) + smart_grid_usage_dec(); + hrtimer_cancel(&auto_affi->period_timer); free_affinity_domains(&auto_affi->ad);

From: Hui Tang tanghui20@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7CGD0 CVE: NA

----------------------------------------

Deadlock occurs in two situations as follows:

The first case:

tg_set_dynamic_affinity_mode --- raw_spin_lock_irq(&auto_affi->lock); ->start_auto_affintiy --- trigger timer ->tg_update_task_prefer_cpus >css_task_inter_next ->raw_spin_unlock_irq

hr_timer_run_queues ->sched_auto_affi_period_timer --- try spin lock (&auto_affi->lock)

The second case as follows:

[ 291.470810] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [ 291.472715] rcu: 1-...0: (0 ticks this GP) idle=a6a/1/0x4000000000000002 softirq=78516/78516 fqs=5249 [ 291.475268] rcu: (detected by 6, t=21006 jiffies, g=202169, q=9862) [ 291.477038] Sending NMI from CPU 6 to CPUs 1: [ 291.481268] NMI backtrace for cpu 1 [ 291.481273] CPU: 1 PID: 1923 Comm: sh Kdump: loaded Not tainted 4.19.90+ #150 [ 291.481278] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.15.0-0-g2dd4b9b3f840-prebuilt.qemu.org 04/01/2014 [ 291.481281] RIP: 0010:queued_spin_lock_slowpath+0x136/0x9a0 [ 291.481289] Code: c0 74 3f 49 89 dd 48 89 dd 48 b8 00 00 00 00 00 fc ff df 49 c1 ed 03 83 e5 07 49 01 c5 83 c5 03 48 83 05 c4 66 b9 05 01 f3 90 <41> 0f b6 45 00 40 38 c5 7c 08 84 c0 0f 85 ad 07 00 00 0 [ 291.481292] RSP: 0018:ffff88801de87cd8 EFLAGS: 00000002 [ 291.481297] RAX: 0000000000000101 RBX: ffff888001be0a28 RCX: ffffffffb8090f7d [ 291.481301] RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffff888001be0a28 [ 291.481304] RBP: 0000000000000003 R08: ffffed100037c146 R09: ffffed100037c146 [ 291.481307] R10: 000000001106b143 R11: ffffed100037c145 R12: 1ffff11003bd0f9c [ 291.481311] R13: ffffed100037c145 R14: fffffbfff7a38dee R15: dffffc0000000000 [ 291.481315] FS: 00007fac4f306740(0000) GS:ffff88801de80000(0000) knlGS:0000000000000000 [ 291.481318] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 291.481321] CR2: 00007fac4f4bb650 CR3: 00000000046b6000 CR4: 00000000000006e0 [ 291.481323] Call Trace: [ 291.481324] <IRQ> [ 291.481326] ? osq_unlock+0x2a0/0x2a0 [ 291.481329] ? check_preemption_disabled+0x4c/0x290 [ 291.481331] ? rcu_accelerate_cbs+0x33/0xed0 [ 291.481333] _raw_spin_lock_irqsave+0x83/0xa0 [ 291.481336] sched_auto_affi_period_timer+0x251/0x820 [ 291.481338] ? __remove_hrtimer+0x151/0x200 [ 291.481340] __hrtimer_run_queues+0x39d/0xa50 [ 291.481343] ? tg_update_affinity_domain_down+0x460/0x460 [ 291.481345] ? enqueue_hrtimer+0x2e0/0x2e0 [ 291.481348] ? ktime_get_update_offsets_now+0x1d7/0x2c0 [ 291.481350] hrtimer_run_queues+0x243/0x470 [ 291.481352] run_local_timers+0x5e/0x150 [ 291.481354] update_process_times+0x36/0xb0 [ 291.481357] tick_sched_handle.isra.4+0x7c/0x180 [ 291.481359] tick_nohz_handler+0xd1/0x1d0 [ 291.481365] smp_apic_timer_interrupt+0x12c/0x4e0 [ 291.481368] apic_timer_interrupt+0xf/0x20 [ 291.481370] </IRQ> [ 291.481372] ? smp_call_function_many+0x68c/0x840 [ 291.481375] ? smp_call_function_many+0x6ab/0x840 [ 291.481377] ? arch_unregister_cpu+0x60/0x60 [ 291.481379] ? native_set_fixmap+0x100/0x180 [ 291.481381] ? arch_unregister_cpu+0x60/0x60 [ 291.481384] ? set_task_select_cpus+0x116/0x940 [ 291.481386] ? smp_call_function+0x53/0xc0 [ 291.481388] ? arch_unregister_cpu+0x60/0x60 [ 291.481390] ? on_each_cpu+0x49/0xf0 [ 291.481393] ? set_task_select_cpus+0x115/0x940 [ 291.481395] ? text_poke_bp+0xff/0x180 [ 291.481397] ? poke_int3_handler+0xc0/0xc0 [ 291.481400] ? __set_prefer_cpus_ptr.constprop.4+0x1cd/0x900 [ 291.481402] ? hrtick+0x1b0/0x1b0 [ 291.481404] ? set_task_select_cpus+0x115/0x940 [ 291.481407] ? __jump_label_transform.isra.0+0x3a1/0x470 [ 291.481409] ? kernel_init+0x280/0x280 [ 291.481411] ? kasan_check_read+0x1d/0x30 [ 291.481413] ? mutex_lock+0x96/0x100 [ 291.481415] ? __mutex_lock_slowpath+0x30/0x30 [ 291.481418] ? arch_jump_label_transform+0x52/0x80 [ 291.481420] ? set_task_select_cpus+0x115/0x940 [ 291.481422] ? __jump_label_update+0x1a1/0x1e0 [ 291.481424] ? jump_label_update+0x2ee/0x3b0 [ 291.481427] ? static_key_slow_inc_cpuslocked+0x1c8/0x2d0 [ 291.481430] ? start_auto_affinity+0x190/0x200 [ 291.481432] ? tg_set_dynamic_affinity_mode+0xad/0xf0 [ 291.481435] ? cpu_affinity_mode_write_u64+0x22/0x30 [ 291.481437] ? cgroup_file_write+0x46f/0x660 [ 291.481439] ? cgroup_init_cftypes+0x300/0x300 [ 291.481441] ? __mutex_lock_slowpath+0x30/0x30

Signed-off-by: Hui Tang tanghui20@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- include/linux/sched/grid_qos.h | 12 ++++ kernel/sched/core.c | 9 +-- kernel/sched/fair.c | 107 ++++++++++++++++++++------------- kernel/sched/grid/qos.c | 14 +++-- 4 files changed, 88 insertions(+), 54 deletions(-)

diff --git a/include/linux/sched/grid_qos.h b/include/linux/sched/grid_qos.h index cea2bf651880..23d08dbb6ae6 100644 --- a/include/linux/sched/grid_qos.h +++ b/include/linux/sched/grid_qos.h @@ -2,6 +2,7 @@ #ifndef _LINUX_SCHED_GRID_QOS_H #define _LINUX_SCHED_GRID_QOS_H #include <linux/nodemask.h> +#include <linux/sched.h>

#ifdef CONFIG_QOS_SCHED_SMART_GRID enum sched_grid_qos_class { @@ -61,6 +62,7 @@ struct sched_grid_qos_power {

struct sched_grid_qos_affinity { nodemask_t mem_preferred_node_mask; + const struct cpumask *prefer_cpus; };

struct task_struct; @@ -72,6 +74,11 @@ struct sched_grid_qos { int (*affinity_set)(struct task_struct *p); };

+static inline int sched_qos_affinity_set(struct task_struct *p) +{ + return p->grid_qos->affinity_set(p); +} + int sched_grid_qos_fork(struct task_struct *p, struct task_struct *orig); void sched_grid_qos_free(struct task_struct *p);

@@ -88,5 +95,10 @@ sched_grid_preferred_nid(int preferred_nid, nodemask_t *nodemask) { return preferred_nid; } + +static inline int sched_qos_affinity_set(struct task_struct *p) +{ + return 0; +} #endif #endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9855cb5458c0..e5307eab43c2 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9530,9 +9530,6 @@ static inline s64 cpu_smt_expell_read(struct cgroup_subsys_state *css, int tg_set_dynamic_affinity_mode(struct task_group *tg, u64 mode) { struct auto_affinity *auto_affi = tg->auto_affinity; - int ret = 0; - - raw_spin_lock_irq(&auto_affi->lock);

/* auto mode*/ if (mode == 1) { @@ -9540,14 +9537,10 @@ int tg_set_dynamic_affinity_mode(struct task_group *tg, u64 mode) } else if (mode == 0) { stop_auto_affinity(auto_affi); } else { - raw_spin_unlock_irq(&auto_affi->lock); return -EINVAL; }

- auto_affi->mode = mode; - raw_spin_unlock_irq(&auto_affi->lock); - - return ret; + return 0; }

static u64 cpu_affinity_mode_read_u64(struct cgroup_subsys_state *css, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d0b7171cef5e..480260279a60 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -28,9 +28,7 @@ #include <linux/delay.h> #include <linux/tracehook.h> #endif -#ifdef CONFIG_QOS_SCHED_SMART_GRID #include <linux/sched/grid_qos.h> -#endif #include <linux/bpf_sched.h>

/* @@ -5821,6 +5819,7 @@ static inline unsigned long cpu_util(int cpu); static unsigned long capacity_of(int cpu); static int sched_idle_cpu(int cpu); static unsigned long cpu_runnable(struct rq *rq); +static inline bool prefer_cpus_valid(struct task_struct *p);

int sysctl_affinity_adjust_delay_ms = 5000;

@@ -5836,22 +5835,29 @@ static void smart_grid_usage_dec(void) static_key_slow_dec(&__smart_grid_used); }

-static void tg_update_task_prefer_cpus(struct task_group *tg) +static inline struct cpumask *task_prefer_cpus(struct task_struct *p) { - struct affinity_domain *ad = &tg->auto_affinity->ad; - struct task_struct *task; - struct css_task_iter it; + struct affinity_domain *ad;

- css_task_iter_start(&tg->css, 0, &it); - while ((task = css_task_iter_next(&it))) { - if (tg == &root_task_group && !task->mm) - continue; + if (!smart_grid_used()) + return p->prefer_cpus;

- set_prefer_cpus_ptr(task, ad->domains[ad->curr_level]); - /* grid_qos must not be NULL */ - task->grid_qos->affinity_set(task); - } - css_task_iter_end(&it); + if (task_group(p)->auto_affinity->mode == 0) + return (void *)p->cpus_ptr; + + ad = &task_group(p)->auto_affinity->ad; + return ad->domains[ad->curr_level]; +} + +static inline int dynamic_affinity_mode(struct task_struct *p) +{ + if (!prefer_cpus_valid(p)) + return -1; + + if (smart_grid_used()) + return task_group(p)->auto_affinity->mode == 0 ? -1 : 1; + + return 0; }

static void affinity_domain_up(struct task_group *tg) @@ -5872,8 +5878,6 @@ static void affinity_domain_up(struct task_group *tg)

if (level == ad->dcount) return; - - tg_update_task_prefer_cpus(tg); }

static void affinity_domain_down(struct task_group *tg) @@ -5894,8 +5898,6 @@ static void affinity_domain_down(struct task_group *tg)

if (!level) return; - - tg_update_task_prefer_cpus(tg); }

static enum hrtimer_restart sched_auto_affi_period_timer(struct hrtimer *timer) @@ -5961,8 +5963,6 @@ static int tg_update_affinity_domain_down(struct task_group *tg, void *data) if (!smart_grid_used()) return 0;

- if (auto_affi->mode) - tg_update_task_prefer_cpus(tg); return 0; }

@@ -5980,35 +5980,41 @@ void tg_update_affinity_domains(int cpu, int online)

void start_auto_affinity(struct auto_affinity *auto_affi) { - struct task_group *tg = auto_affi->tg; ktime_t delay_ms;

- if (auto_affi->period_active == 1) + raw_spin_lock_irq(&auto_affi->lock); + if (auto_affi->period_active == 1) { + raw_spin_unlock_irq(&auto_affi->lock); return; - - tg_update_task_prefer_cpus(tg); + }

auto_affi->period_active = 1; + auto_affi->mode = 1; delay_ms = ms_to_ktime(sysctl_affinity_adjust_delay_ms); hrtimer_forward_now(&auto_affi->period_timer, delay_ms); hrtimer_start_expires(&auto_affi->period_timer, HRTIMER_MODE_ABS_PINNED); + raw_spin_unlock_irq(&auto_affi->lock); + smart_grid_usage_inc(); }

void stop_auto_affinity(struct auto_affinity *auto_affi) { - struct task_group *tg = auto_affi->tg; struct affinity_domain *ad = &auto_affi->ad;

- if (auto_affi->period_active == 0) + raw_spin_lock_irq(&auto_affi->lock); + if (auto_affi->period_active == 0) { + raw_spin_unlock_irq(&auto_affi->lock); return; + }

hrtimer_cancel(&auto_affi->period_timer); auto_affi->period_active = 0; + auto_affi->mode = 0; ad->curr_level = ad->dcount > 0 ? ad->dcount - 1 : 0; + raw_spin_unlock_irq(&auto_affi->lock);

- tg_update_task_prefer_cpus(tg); smart_grid_usage_dec(); }

@@ -6226,6 +6232,19 @@ static void destroy_auto_affinity(struct task_group *tg) } #else static void destroy_auto_affinity(struct task_group *tg) {} + +#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY +static inline struct cpumask *task_prefer_cpus(struct task_struct *p) +{ + return p->prefer_cpus; +} +#endif + +static inline int dynamic_affinity_mode(struct task_struct *p) +{ + return 0; +} + #endif

/************************************************** @@ -7725,10 +7744,11 @@ int sysctl_sched_util_low_pct = 85;

static inline bool prefer_cpus_valid(struct task_struct *p) { - return p->prefer_cpus && - !cpumask_empty(p->prefer_cpus) && - !cpumask_equal(p->prefer_cpus, p->cpus_ptr) && - cpumask_subset(p->prefer_cpus, p->cpus_ptr); + struct cpumask *prefer_cpus = task_prefer_cpus(p); + + return !cpumask_empty(prefer_cpus) && + !cpumask_equal(prefer_cpus, p->cpus_ptr) && + cpumask_subset(prefer_cpus, p->cpus_ptr); }

static inline unsigned long taskgroup_cpu_util(struct task_group *tg, @@ -7763,20 +7783,23 @@ static void set_task_select_cpus(struct task_struct *p, int *idlest_cpu, long min_util = INT_MIN; struct task_group *tg; long spare; - int cpu; + int cpu, mode;

- p->select_cpus = p->cpus_ptr; - if (!prefer_cpus_valid(p)) + rcu_read_lock(); + mode = dynamic_affinity_mode(p); + if (mode == -1) { + rcu_read_unlock(); return; - - if (smart_grid_used()) { - p->select_cpus = p->prefer_cpus; + } else if (mode == 1) { + p->select_cpus = task_prefer_cpus(p); if (idlest_cpu) *idlest_cpu = cpumask_first(p->select_cpus); + sched_qos_affinity_set(p); + rcu_read_unlock(); return; }

- rcu_read_lock(); + /* manual mode */ tg = task_group(p); for_each_cpu(cpu, p->prefer_cpus) { if (idlest_cpu && (available_idle_cpu(cpu) || sched_idle_cpu(cpu))) { @@ -7844,13 +7867,13 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f time = schedstat_start_time();

/* - * required for stable ->cpus_allowed + * required for stable ->cpus_ptr */ lockdep_assert_held(&p->pi_lock);

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY p->select_cpus = p->cpus_ptr; - if (dynamic_affinity_used()) + if (dynamic_affinity_used() || smart_grid_used()) set_task_select_cpus(p, &idlest_cpu, sd_flag); #endif

@@ -9441,7 +9464,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY p->select_cpus = p->cpus_ptr; - if (dynamic_affinity_used()) + if (dynamic_affinity_used() || smart_grid_used()) set_task_select_cpus(p, NULL, 0); if (!cpumask_test_cpu(env->dst_cpu, p->select_cpus)) { #else diff --git a/kernel/sched/grid/qos.c b/kernel/sched/grid/qos.c index 3fb433d213fd..7081bee588ee 100644 --- a/kernel/sched/grid/qos.c +++ b/kernel/sched/grid/qos.c @@ -24,20 +24,26 @@ #include <linux/sched/grid_qos.h> #include "internal.h"

-static int qos_affinity_set(struct task_struct *p) +static inline int qos_affinity_set(struct task_struct *p) { int n; struct sched_grid_qos_affinity *affinity = &p->grid_qos->affinity;

- nodes_clear(affinity->mem_preferred_node_mask); + if (likely(affinity->prefer_cpus == p->select_cpus)) + return 0; + /* * We want the memory allocation to be as close to the CPU * as possible, and adjust after getting memory bandwidth usage. */ - for (n = 0; n < nr_node_ids; n++) - if (cpumask_intersects(cpumask_of_node(n), p->prefer_cpus)) + for (n = 0; n < nr_node_ids; n++) { + if (cpumask_intersects(cpumask_of_node(n), p->select_cpus)) node_set(n, affinity->mem_preferred_node_mask); + else + node_clear(n, affinity->mem_preferred_node_mask); + }

+ affinity->prefer_cpus = p->select_cpus; return 0; }

From: Hui Tang tanghui20@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7DSX6 CVE: NA

-------------------------------

Timer storm may be triggered if !cpumask_weight(ad->domains[i]) which is set in cpu offline.

Fixes: 713cfd2684fa ("sched: Introduce smart grid scheduling strategy for cfs") Signed-off-by: Hui Tang tanghui20@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- kernel/sched/fair.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e53911b7222c..3f226695ac9b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5871,15 +5871,13 @@ static void affinity_domain_up(struct task_group *tg) return;

while (level < ad->dcount) { - if (IS_DOMAIN_SET(level + 1, ad->domain_mask)) { + if (IS_DOMAIN_SET(level + 1, ad->domain_mask) && + cpumask_weight(ad->domains[level + 1]) > 0) { ad->curr_level = level + 1; - break; + return; } level++; } - - if (level == ad->dcount) - return; }

static void affinity_domain_down(struct task_group *tg) @@ -5891,15 +5889,15 @@ static void affinity_domain_down(struct task_group *tg) return;

while (level > 0) { + if (!cpumask_weight(ad->domains[level - 1])) + return; + if (IS_DOMAIN_SET(level - 1, ad->domain_mask)) { ad->curr_level = level - 1; - break; + return; } level--; } - - if (!level) - return; }

static enum hrtimer_restart sched_auto_affi_period_timer(struct hrtimer *timer) @@ -5919,11 +5917,8 @@ static enum hrtimer_restart sched_auto_affi_period_timer(struct hrtimer *timer) tg_capacity += capacity_of(cpu); }

- if (unlikely(!tg_capacity)) - return HRTIMER_RESTART; - raw_spin_lock_irqsave(&auto_affi->lock, flags); - if (util_avg_sum * 100 > tg_capacity * sysctl_sched_util_low_pct) { + if (util_avg_sum * 100 >= tg_capacity * sysctl_sched_util_low_pct) { affinity_domain_up(tg); } else if (util_avg_sum * 100 < tg_capacity * sysctl_sched_util_low_pct / 2) { @@ -5955,16 +5950,17 @@ static int tg_update_affinity_domain_down(struct task_group *tg, void *data) continue;

/* online */ - if (cpu_state[1]) + if (cpu_state[1]) { cpumask_set_cpu(cpu_state[0], ad->domains[i]); - else + } else { cpumask_clear_cpu(cpu_state[0], ad->domains[i]); + if (!cpumask_weight(ad->domains[i])) + affinity_domain_up(tg); + } + } raw_spin_unlock_irqrestore(&auto_affi->lock, flags);

- if (!smart_grid_used()) - return 0; - return 0; }

From: Hui Tang tanghui20@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7EBNA CVE: NA

-------------------------------

Fix memory leak on error branch for smart grid.

Fixes: 713cfd2684fa ("sched: Introduce smart grid scheduling strategy for cfs") Signed-off-by: Hui Tang tanghui20@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- kernel/sched/fair.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 467528589b8a..480e87d7e9fa 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6156,7 +6156,6 @@ static int init_affinity_domains(struct affinity_domain *ad) }

if (!sd) { - ad->dcount = 0; rcu_read_unlock(); return -EINVAL; } @@ -6164,8 +6163,10 @@ static int init_affinity_domains(struct affinity_domain *ad)

for (i = 0; i < dcount; i++) { ad->domains[i] = kmalloc(sizeof(cpumask_t), GFP_KERNEL); - if (!ad->domains[i]) + if (!ad->domains[i]) { + ad->dcount = i; goto err; + } }

rcu_read_lock();

From: Hui Tang tanghui20@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7EEF3 CVE: NA

-------------------------------

Adjust few parameters range for smart grid.

Fixes: 713cfd2684fa ("sched: Introduce smart grid scheduling strategy for cfs") Signed-off-by: Hui Tang tanghui20@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- kernel/sched/core.c | 2 +- kernel/sysctl.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 18191396f858..4c04ef0d3fe3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9568,7 +9568,7 @@ int tg_set_affinity_period(struct task_group *tg, u64 period_ms) if (unlikely(!tg->auto_affinity)) return -EPERM;

- if (period_ms > U64_MAX / NSEC_PER_MSEC) + if (!period_ms || period_ms > U64_MAX / NSEC_PER_MSEC) return -EINVAL;

raw_spin_lock_irq(&tg->auto_affinity->lock); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b7a0e9a34035..58eeca0db3d0 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -125,7 +125,7 @@ static int one_thousand = 1000; #ifdef CONFIG_PRINTK static int ten_thousand = 10000; #endif -#ifdef CONFIG_QOS_SCHED +#if defined(CONFIG_QOS_SCHED) || defined(CONFIG_QOS_SCHED_SMART_GRID) static int hundred_thousand = 100000; #endif #ifdef CONFIG_PERF_EVENTS @@ -2809,7 +2809,9 @@ static struct ctl_table kern_table[] = { .data = &sysctl_affinity_adjust_delay_ms, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &hundred_thousand, }, #endif { }

From: Hui Tang tanghui20@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7FBJM CVE: NA

----------------------------------------

Free ad->domains_orig[] in 'free_affinity_domains', otherwise the memory will leak.

Fixes: 713cfd2684fa ("sched: Introduce smart grid scheduling strategy for cfs") Signed-off-by: Hui Tang tanghui20@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- kernel/sched/fair.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index dcf2e258d9ac..54f61ccfa6d7 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6112,7 +6112,9 @@ void free_affinity_domains(struct affinity_domain *ad)

for (i = 0; i < ad->dcount; i++) { kfree(ad->domains[i]); + kfree(ad->domains_orig[i]); ad->domains[i] = NULL; + ad->domains_orig[i] = NULL; } ad->dcount = 0; } @@ -6133,7 +6135,7 @@ static int init_affinity_domains_orig(struct affinity_domain *ad) err: for (j = 0; j < i; j++) { kfree(ad->domains_orig[j]); - ad->domains_orig[i] = NULL; + ad->domains_orig[j] = NULL; } return -ENOMEM; } @@ -6199,7 +6201,7 @@ int init_auto_affinity(struct task_group *tg) struct auto_affinity *auto_affi; int ret;

- auto_affi = kmalloc(sizeof(*auto_affi), GFP_KERNEL); + auto_affi = kzalloc(sizeof(*auto_affi), GFP_KERNEL); if (!auto_affi) return -ENOMEM;

From: Hui Tang tanghui20@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7HFZV CVE: NA

----------------------------------------

There may be NULL pointer derefrence when hotplug running and creating taskgroup concurrently.

sched_autogroup_create_attach -> sched_create_group -> alloc_fair_sched_group -> init_auto_affinity -> init_affinity_domains -> cpumask_copy(xx, sched_domain_span(tmp)) { tmp may be free due rcu lock missing }

{ hotplug will rebuild sched domain } sched_cpu_activate -> build_sched_domains -> cpuset_cpu_active -> partition_sched_domains -> build_sched_domains -> cpu_attach_domain -> destroy_sched_domains -> call_rcu(&sd->rcu, destroy_sched_domains_rcu)

So sd should be protect with rcu lock in entire critical zone.

[ 599.811593] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000000 [ 600.112821] pc : init_affinity_domains+0xf4/0x200 [ 600.125918] lr : init_affinity_domains+0xd4/0x200 [ 600.331355] Call trace: [ 600.338734] init_affinity_domains+0xf4/0x200 [ 600.347955] init_auto_affinity+0x78/0xc0 [ 600.356622] alloc_fair_sched_group+0xd8/0x210 [ 600.365594] sched_create_group+0x48/0xc0 [ 600.373970] sched_autogroup_create_attach+0x54/0x190 [ 600.383311] ksys_setsid+0x110/0x130 [ 600.391014] __arm64_sys_setsid+0x18/0x24 [ 600.399156] el0_svc_common+0x118/0x170 [ 600.406818] el0_svc_handler+0x3c/0x80 [ 600.414188] el0_svc+0x8/0x640 [ 600.420719] Code: b40002c0 9104e002 f9402061 a9401444 (a9001424) [ 600.430504] SMP: stopping secondary CPUs [ 600.441751] Starting crashdump kernel...

Fixes: 713cfd2684fa ("sched: Introduce smart grid scheduling strategy for cfs") Signed-off-by: Hui Tang tanghui20@huawei.com Reviewed-by: Zhang Qiao zhangqiao22@huawei.com Signed-off-by: Zhang Changzhong zhangchangzhong@huawei.com Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- kernel/sched/fair.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 54f61ccfa6d7..24138c60442b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6110,7 +6110,7 @@ void free_affinity_domains(struct affinity_domain *ad) { int i;

- for (i = 0; i < ad->dcount; i++) { + for (i = 0; i < AD_LEVEL_MAX; i++) { kfree(ad->domains[i]); kfree(ad->domains_orig[i]); ad->domains[i] = NULL; @@ -6149,6 +6149,12 @@ static int init_affinity_domains(struct affinity_domain *ad) int i = 0; int cpu;

+ for (i = 0; i < AD_LEVEL_MAX; i++) { + ad->domains[i] = kmalloc(sizeof(cpumask_t), GFP_KERNEL); + if (!ad->domains[i]) + goto err; + } + rcu_read_lock(); cpu = cpumask_first_and(cpu_active_mask, housekeeping_cpumask(HK_FLAG_DOMAIN)); @@ -6157,21 +6163,12 @@ static int init_affinity_domains(struct affinity_domain *ad) dcount++; }

- if (!sd) { + if (!sd || dcount > AD_LEVEL_MAX) { rcu_read_unlock(); - return -EINVAL; - } - rcu_read_unlock(); - - for (i = 0; i < dcount; i++) { - ad->domains[i] = kmalloc(sizeof(cpumask_t), GFP_KERNEL); - if (!ad->domains[i]) { - ad->dcount = i; - goto err; - } + ret = -EINVAL; + goto err; }

- rcu_read_lock(); idlest = sd_find_idlest_group(sd); cpu = group_find_idlest_cpu(idlest); i = 0; @@ -6216,6 +6213,8 @@ int init_auto_affinity(struct task_group *tg) ret = init_affinity_domains(&auto_affi->ad); if (ret) { kfree(auto_affi); + if (ret == -EINVAL) + ret = 0; return ret; }

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I7ZBSR CVE: NA

----------------------------------------

Now, we can use /pro/pid/smart_grid_level to {read,set} task current qos level.

This allows to determine the scope of dynamic partitioning of the task in smart_grid.

SCHED_GRID_QOS_TASK_LEVEL was defined different QoS level. The lower number has the higher priority. (E.g. 0 was the highest).

Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- fs/proc/base.c | 76 ++++++++++++++++++++++++++++++++++ include/linux/sched/grid_qos.h | 11 ++++- kernel/sched/grid/stat.c | 15 +++++++ 3 files changed, 101 insertions(+), 1 deletion(-)

diff --git a/fs/proc/base.c b/fs/proc/base.c index 2a4cc5c796c7..e9a5550fbbe2 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -93,6 +93,10 @@ #include <linux/sched/coredump.h> #include <linux/sched/debug.h> #include <linux/sched/stat.h> +#ifdef CONFIG_QOS_SCHED_SMART_GRID +#include <linux/sched/grid_qos.h> +#include <linux/sched.h> +#endif #include <linux/posix-timers.h> #include <linux/time_namespace.h> #include <linux/resctrl.h> @@ -3493,6 +3497,75 @@ static int proc_stack_depth(struct seq_file *m, struct pid_namespace *ns, } #endif /* CONFIG_STACKLEAK_METRICS */

+#ifdef CONFIG_QOS_SCHED_SMART_GRID +static int smart_grid_level_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + if (p->grid_qos != NULL) + seq_printf(m, "%d\n", p->grid_qos->stat.class_lvl); + + put_task_struct(p); + + return 0; +} + +static int smart_grid_level_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, smart_grid_level_show, inode); +} + +static ssize_t smart_grid_level_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + char buffer[TASK_COMM_LEN]; + const size_t maxlen = sizeof(buffer) - 1; + unsigned long long level = SCHED_GRID_QOS_TASK_LEVEL_MAX; + unsigned int len; + int ret = 0; + + memset(buffer, 0, sizeof(buffer)); + if (copy_from_user(buffer, buf, count > maxlen ? maxlen : count)) + return -EFAULT; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + len = _parse_integer(buffer, 10, &level); + if (len & KSTRTOX_OVERFLOW || + level >= SCHED_GRID_QOS_TASK_LEVEL_MAX) { + put_task_struct(p); + return -EINVAL; + } + + if (p->grid_qos != NULL && + p->grid_qos->stat.set_class_lvl != NULL) + ret = p->grid_qos->stat.set_class_lvl(&p->grid_qos->stat, level); + + put_task_struct(p); + + if (ret) + return ret; + return count; +} + +static const struct file_operations proc_pid_sg_level_operations = { + .open = smart_grid_level_open, + .read = seq_read, + .write = smart_grid_level_write, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + /* * Thread groups */ @@ -3516,6 +3589,9 @@ static const struct pid_entry tgid_base_stuff[] = { #ifdef CONFIG_SCHED_DEBUG REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations), #endif +#ifdef CONFIG_QOS_SCHED_SMART_GRID + REG("smart_grid_level", 0644, proc_pid_sg_level_operations), +#endif #ifdef CONFIG_SCHED_AUTOGROUP REG("autogroup", S_IRUGO|S_IWUSR, proc_pid_sched_autogroup_operations), #endif diff --git a/include/linux/sched/grid_qos.h b/include/linux/sched/grid_qos.h index 3bfb10d9f58a..15a19aeb1d4f 100644 --- a/include/linux/sched/grid_qos.h +++ b/include/linux/sched/grid_qos.h @@ -17,6 +17,15 @@ enum sched_grid_qos_class { SCHED_GRID_QOS_CLASS_LEVEL_NR };

+/* + * SCHED_GRID_QOS_TASK_LEVEL was defined different QoS level. + * The lower number has the higher priority. (E.g. 0 was the highest) + * The enum sched_grid_qos_class defined the max level, the lowest level. + */ +#define SCHED_GRID_QOS_TASK_LEVEL_HIGHEST SCHED_GRID_QOS_CLASS_LEVEL_1 +#define SCHED_GRID_QOS_TASK_LEVEL_MAX (SCHED_GRID_QOS_CLASS_LEVEL_NR) +#define SCHED_GRID_QOS_TASK_LEVEL_DEFAULT (SCHED_GRID_QOS_CLASS_LEVEL_NR - 1) + enum { SCHED_GRID_QOS_IPS_INDEX = 0, SCHED_GRID_QOS_MEMBOUND_RATIO_INDEX = 1, @@ -50,7 +59,7 @@ struct sched_grid_qos_sample {

struct sched_grid_qos_stat { enum sched_grid_qos_class class_lvl; - int (*set_class_lvl)(struct sched_grid_qos_stat *qos_stat); + int (*set_class_lvl)(struct sched_grid_qos_stat *qos_stat, int level); struct sched_grid_qos_sample sample[SCHED_GRID_QOS_SAMPLE_NR]; };

diff --git a/kernel/sched/grid/stat.c b/kernel/sched/grid/stat.c index b40c75145608..68bbc060b811 100644 --- a/kernel/sched/grid/stat.c +++ b/kernel/sched/grid/stat.c @@ -19,8 +19,20 @@ #include <linux/sched/grid_qos.h> #include "internal.h"

+static int qos_stat_set_class_level(struct sched_grid_qos_stat *qos_stat, int level) +{ + if (qos_stat == NULL || level >= SCHED_GRID_QOS_TASK_LEVEL_MAX) + return -EINVAL; + + qos_stat->class_lvl = level; + return 0; +} + void qos_stat_init(struct sched_grid_qos_stat *stat) { + if (stat == NULL) + return; + stat->sample[SCHED_GRID_QOS_IPS_INDEX].name = "ips"; stat->sample[SCHED_GRID_QOS_IPS_INDEX].index = SCHED_GRID_QOS_IPS_INDEX; stat->sample[SCHED_GRID_QOS_MEMBOUND_RATIO_INDEX].name = "membound_ratio"; @@ -29,4 +41,7 @@ void qos_stat_init(struct sched_grid_qos_stat *stat) stat->sample[SCHED_GRID_QOS_MEMBANDWIDTH_INDEX].name = "memband_width"; stat->sample[SCHED_GRID_QOS_MEMBANDWIDTH_INDEX].index = SCHED_GRID_QOS_MEMBANDWIDTH_INDEX; + + stat->set_class_lvl = qos_stat_set_class_level; + stat->class_lvl = SCHED_GRID_QOS_TASK_LEVEL_DEFAULT; }

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I7ZBSR CVE: NA

----------------------------------------

In smart_grid zone, the system is divided into multiple zones. The typical configuration is one hot zone and the other warm zone.

This commit can configure an independent cpufreq governor for each zone. After the zone range in the system changes dynamically, it can Automatically configure the corresponding cpufreq governor.

The default configuration of hot zone is performance governor, and the default configuration of warm zone is powersave governor.

-------- -------- -------- | group0 | | group1 | | group2 | -------- -------- -------- | | | v v v --------------------------- ----------------- | | | | | hot zone | | warm zone | | | | | --------------------------- ----------------- ^ ^ | | ------------- ---------- | performance | <- cpufreq governor -> |powersave | ------------ ----------

Introduce two attributes in /sys/devices/system/cpu/cpufreq:

1. smart_grid_governor_enable

For smart_grid governor, set 1 for enable, 0 for disable, read to get current status.

For Notice, if smart_grid governor is enabled the governor of cpu may auto switch by smart_grid, so if user wants to config governor by itself smart_grid_governor shoudle be disabled.

2. smart_grid_governor

Only can set when smart_grid_governor has been enabled.

Set {level}-{governor name} formate to change each zone's governor:

0-performance will set all hot zone cpu's governor to performance. 1-powersave will set all warm zone cpu's governor to powersave.

Signed-off-by: Yipeng Zou zouyipeng@huawei.com --- drivers/cpufreq/cpufreq.c | 234 ++++++++++++++++++++++++++++++++++++++ include/linux/cpufreq.h | 11 ++ kernel/sched/grid/qos.c | 1 + 3 files changed, 246 insertions(+)

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 70f7a638b6db..969569a3facf 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2689,6 +2689,237 @@ int cpufreq_boost_enabled(void) } EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);

+#ifdef CONFIG_QOS_SCHED_SMART_GRID + +struct smart_grid_zone { + char governor_name[SMART_GRID_ZONE_NR][CPUFREQ_NAME_LEN]; + unsigned int enable; + struct irq_work irq_work; + struct work_struct work; + unsigned int is_init; +}; + +static struct smart_grid_zone sg_zone; +static DEFINE_MUTEX(sg_zone_lock); + +#define SG_WRITE_BUFF_LEN 30 + +void cpufreq_smart_grid_start_sync(void) +{ + if (likely(sg_zone.is_init)) + irq_work_queue(&sg_zone.irq_work); +} + +static ssize_t show_smart_grid_governor(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int len = 0; + int gov_index; + + mutex_lock(&sg_zone_lock); + if (!sg_zone.enable) { + mutex_unlock(&sg_zone_lock); + return sprintf(buf, "smart_grid governor disable\n"); + } + + for (gov_index = 0; gov_index < SMART_GRID_ZONE_NR; gov_index++) + len += sprintf(buf + len, "smart_grid-%d: %s\n", gov_index, + sg_zone.governor_name[gov_index]); + + mutex_unlock(&sg_zone_lock); + return len; +} + +static ssize_t store_smart_grid_governor(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct cpufreq_governor *target_gov = NULL; + unsigned int current_level; + char *level_string = NULL; + char buf_string[SG_WRITE_BUFF_LEN]; + char *gov_string = buf_string; + char save_string[CPUFREQ_NAME_LEN]; + int ret; + + mutex_lock(&sg_zone_lock); + if (!sg_zone.enable) { + ret = -EINVAL; + goto fail; + } + + if (strscpy(buf_string, buf, SG_WRITE_BUFF_LEN) <= 0) { + ret = -EINVAL; + goto fail; + } + + level_string = strsep(&gov_string, "-"); + if (level_string == NULL) { + ret = -EINVAL; + goto fail; + } + + if (kstrtouint(level_string, 10, &current_level)) { + ret = -EINVAL; + goto fail; + } + + if (current_level >= SMART_GRID_ZONE_NR) { + ret = -EINVAL; + goto fail; + } + + if (sscanf(gov_string, "%15s", save_string) != 1) { + ret = -EINVAL; + goto fail; + } + + target_gov = cpufreq_parse_governor(save_string); + if (target_gov == NULL) { + ret = -EINVAL; + goto fail; + } + module_put(target_gov->owner); + + strscpy(sg_zone.governor_name[current_level], save_string, CPUFREQ_NAME_LEN); + cpufreq_smart_grid_start_sync(); + mutex_unlock(&sg_zone_lock); + return count; + +fail: + mutex_unlock(&sg_zone_lock); + return ret; +} +define_one_global_rw(smart_grid_governor); + +static ssize_t show_smart_grid_governor_enable(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", sg_zone.enable); +} + +static void smart_grid_irq_work(struct irq_work *irq_work) +{ + struct smart_grid_zone *zone; + + zone = container_of(irq_work, struct smart_grid_zone, irq_work); + schedule_work_on(smp_processor_id(), &zone->work); +} + +static void smart_grid_work_handler(struct work_struct *work) +{ + struct smart_grid_zone *zone; + struct cpufreq_governor *target_gov = NULL; + struct cpufreq_policy *policy = NULL; + unsigned int cpu; + int gov_index; + + zone = container_of(work, struct smart_grid_zone, work); + + mutex_lock(&sg_zone_lock); + if (!sg_zone.enable) { + mutex_unlock(&sg_zone_lock); + return; + } + + for (gov_index = 0; gov_index < SMART_GRID_ZONE_NR; gov_index++) { + target_gov = cpufreq_parse_governor(sg_zone.governor_name[gov_index]); + if (target_gov == NULL) + continue; + + for_each_cpu(cpu, sched_grid_zone_cpumask(gov_index)) { + if (cpu_is_offline(cpu)) + continue; + + policy = cpufreq_cpu_acquire(cpu); + if (policy == NULL) + continue; + + if (policy->governor == target_gov) { + cpufreq_cpu_release(policy); + continue; + } + /*Try to switch governor */ + store_scaling_governor(policy, sg_zone.governor_name[gov_index], + CPUFREQ_NAME_LEN); + cpufreq_cpu_release(policy); + } + module_put(target_gov->owner); + } + mutex_unlock(&sg_zone_lock); +} + +static void sg_zone_set_enable(void) +{ + int gov_index; + + /* Set default smart_grid governor */ + for (gov_index = 0; gov_index < SMART_GRID_ZONE_NR; gov_index++) { + if (!gov_index) + strscpy(sg_zone.governor_name[gov_index], "performance", CPUFREQ_NAME_LEN); + else + strscpy(sg_zone.governor_name[gov_index], "powersave", CPUFREQ_NAME_LEN); + } + + sg_zone.enable = 1; + cpufreq_smart_grid_start_sync(); +} + +static void sg_zone_set_disable(void) +{ + irq_work_sync(&sg_zone.irq_work); + cancel_work_sync(&sg_zone.work); + sg_zone.enable = 0; +} + +static ssize_t store_smart_grid_governor_enable(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + unsigned int enable; + + if (kstrtouint(buf, 10, &enable)) + return -EINVAL; + + if (enable > 1) + return -EINVAL; + + mutex_lock(&sg_zone_lock); + if (sg_zone.enable == enable) { + mutex_unlock(&sg_zone_lock); + return -EINVAL; + } + + if (enable) + sg_zone_set_enable(); + else + sg_zone_set_disable(); + + mutex_unlock(&sg_zone_lock); + return count; +} +define_one_global_rw(smart_grid_governor_enable); + +static int create_smart_grid_sysfs_file(void) +{ + int ret; + + ret = sysfs_create_file(cpufreq_global_kobject, &smart_grid_governor.attr); + if (ret) + pr_err("%s: cannot register global smart_grid_governor sysfs file\n", + __func__); + + ret = sysfs_create_file(cpufreq_global_kobject, &smart_grid_governor_enable.attr); + if (ret) + pr_err("%s: cannot register global smart_grid_governor_enable sysfs file\n", + __func__); + + init_irq_work(&sg_zone.irq_work, smart_grid_irq_work); + INIT_WORK(&sg_zone.work, smart_grid_work_handler); + sg_zone.enable = 0; + sg_zone.is_init = 1; + return ret; +} +#endif + /********************************************************************* * REGISTER / UNREGISTER CPUFREQ DRIVER * *********************************************************************/ @@ -2861,6 +3092,9 @@ static int __init cpufreq_core_init(void) if (!strlen(default_governor)) strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN);

+#ifdef CONFIG_QOS_SCHED_SMART_GRID + create_smart_grid_sysfs_file(); +#endif return 0; } module_param(off, int, 0444); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index acbad3b36322..b715828cf357 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -16,6 +16,9 @@ #include <linux/pm_qos.h> #include <linux/spinlock.h> #include <linux/sysfs.h> +#ifdef CONFIG_QOS_SCHED_SMART_GRID +#include <linux/sched/grid_qos.h> +#endif

/********************************************************************* * CPUFREQ INTERFACE * @@ -606,6 +609,14 @@ int cpufreq_register_governor(struct cpufreq_governor *governor); void cpufreq_unregister_governor(struct cpufreq_governor *governor); int cpufreq_start_governor(struct cpufreq_policy *policy); void cpufreq_stop_governor(struct cpufreq_policy *policy); +#ifdef CONFIG_QOS_SCHED_SMART_GRID +/* Implement in cpufreq.c */ +#ifdef CONFIG_CPU_FREQ +void cpufreq_smart_grid_start_sync(void); +#else +static inline void cpufreq_smart_grid_start_sync(void) { return; } +#endif +#endif

#define cpufreq_governor_init(__governor) \ static int __init __governor##_init(void) \ diff --git a/kernel/sched/grid/qos.c b/kernel/sched/grid/qos.c index fee1b2c8c220..f8c5deffd3ea 100644 --- a/kernel/sched/grid/qos.c +++ b/kernel/sched/grid/qos.c @@ -198,6 +198,7 @@ int sched_grid_zone_update(bool is_locked) if (!is_locked) raw_spin_unlock_irqrestore(&sg_zone.lock, flags);

+ cpufreq_smart_grid_start_sync(); return 0; }