hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7D76D -------------------------------- Add cmdline of 'noqossched' which is enable by default. Use the static key to reduce noise for multi level prio, smt expell and prio load balance. Signed-off-by: Hui Tang <tanghui20@huawei.com> --- include/linux/sched.h | 26 ++++++++++++++--- kernel/sched/core.c | 11 +++++++ kernel/sched/fair.c | 67 +++++++++++++++++++++++++++++-------------- kernel/sched/sched.h | 6 +++- kernel/signal.c | 3 +- 5 files changed, 86 insertions(+), 27 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 3aae225f98a7..fdd3324cc858 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1869,15 +1869,19 @@ extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk); __get_task_comm(buf, sizeof(buf), tsk); \ }) +#ifdef CONFIG_SMP #ifdef CONFIG_QOS_SCHED_SMT_EXPELLER -void qos_smt_check_need_resched(void); +static __always_inline bool qos_sched_enabled(void); +extern struct static_key_true qos_smt_expell_switch; +extern void qos_smt_check_need_resched(void); #endif - -#ifdef CONFIG_SMP static __always_inline void scheduler_ipi(void) { #ifdef CONFIG_QOS_SCHED_SMT_EXPELLER - qos_smt_check_need_resched(); + + if (qos_sched_enabled() && + static_branch_likely(&qos_smt_expell_switch)) + qos_smt_check_need_resched(); #endif /* * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting @@ -2215,11 +2219,25 @@ static inline void sched_core_fork(struct task_struct *p) { } void sched_move_offline_task(struct task_struct *p); void sched_qos_offline_wait(void); int sched_qos_cpu_overload(void); + +extern struct static_key_true __qos_sched_switch; +static __always_inline bool qos_sched_enabled(void) +{ + if (static_branch_likely(&__qos_sched_switch)) + return true; + + return false; +} #else static inline int sched_qos_cpu_overload(void) { return 0; } + +static __always_inline bool qos_sched_enabled(void) +{ + return false; +} #endif #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 454bca0c9c6b..71c69c8161ff 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3617,6 +3617,17 @@ static void __init init_schedstats(void) set_schedstats(__sched_schedstats); } +#ifdef CONFIG_QOS_SCHED +DEFINE_STATIC_KEY_TRUE(__qos_sched_switch); + +static int __init no_qos_sched_switch_setup(char *__unused) +{ + static_branch_disable(&__qos_sched_switch); + return 1; +} +__setup("noqossched", no_qos_sched_switch_setup); +#endif + #ifdef CONFIG_PROC_SYSCTL int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ff209d25c21c..d7233d4da47d 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2997,13 +2997,17 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu) #endif /* CONFIG_NUMA_BALANCING */ #ifdef CONFIG_QOS_SCHED_PRIO_LB -static void +static __always_inline void adjust_rq_cfs_tasks(void (*list_op)(struct list_head *, struct list_head *), struct rq *rq, struct sched_entity *se) { - struct task_group *tg = task_group(task_of(se)); + struct task_group *tg; + if (!qos_sched_enabled()) + return; + + tg = task_group(task_of(se)); if (sysctl_sched_prio_load_balance_enabled && is_offline_level(tg->qos_level)) (*list_op)(&se->group_node, &rq->cfs_offline_tasks); else @@ -3235,7 +3239,8 @@ static long calc_group_shares(struct cfs_rq *cfs_rq) tg_shares = READ_ONCE(tg->shares); #ifdef CONFIG_QOS_SCHED_MULTILEVEL - tg_shares = qos_reweight(tg_shares, tg); + if (qos_sched_enabled()) + tg_shares = qos_reweight(tg_shares, tg); #endif load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); @@ -3286,7 +3291,8 @@ static void update_cfs_group(struct sched_entity *se) #ifndef CONFIG_SMP shares = READ_ONCE(gcfs_rq->tg->shares); #ifdef CONFIG_QOS_SCHED_MULTILEVEL - shares = qos_reweight(shares, gcfs_rq->tg); + if (qos_sched_enabled()) + shares = qos_reweight(shares, gcfs_rq->tg); #endif if (likely(se->load.weight == shares)) @@ -7071,7 +7077,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) #ifdef CONFIG_JUMP_LABEL static DEFINE_STATIC_KEY_FALSE(__dynamic_affinity_used); -static inline bool dynamic_affinity_used(void) +static __always_inline bool dynamic_affinity_used(void) { return static_branch_unlikely(&__dynamic_affinity_used); } @@ -7082,7 +7088,7 @@ void dynamic_affinity_enable(void) } #else /* CONFIG_JUMP_LABEL */ -static bool dynamic_affinity_used(void) +static __always_inline bool dynamic_affinity_used(void) { return true; } @@ -7097,9 +7103,6 @@ int sysctl_sched_util_low_pct = 85; static inline bool prefer_cpus_valid(struct task_struct *p) { - if (!dynamic_affinity_used()) - return false; - return p->prefer_cpus && !cpumask_empty(p->prefer_cpus) && !cpumask_equal(p->prefer_cpus, p->cpus_ptr) && @@ -7208,7 +7211,9 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f lockdep_assert_held(&p->pi_lock); #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY - set_task_select_cpus(p, &idlest_cpu, sd_flag); + p->select_cpus = p->cpus_ptr; + if (dynamic_affinity_used()) + set_task_select_cpus(p, &idlest_cpu, sd_flag); #endif if (sd_flag & SD_BALANCE_WAKE) { @@ -7770,9 +7775,13 @@ static int __unthrottle_qos_cfs_rqs(int cpu) return res; } -static int unthrottle_qos_cfs_rqs(int cpu) +static __always_inline int unthrottle_qos_cfs_rqs(int cpu) { int res; + + if (!qos_sched_enabled()) + return 0; + res = __unthrottle_qos_cfs_rqs(cpu); if (qos_timer_is_activated(cpu) && !qos_smt_expelled(cpu)) @@ -7781,8 +7790,11 @@ static int unthrottle_qos_cfs_rqs(int cpu) return res; } -static bool check_qos_cfs_rq(struct cfs_rq *cfs_rq) +static __always_inline bool check_qos_cfs_rq(struct cfs_rq *cfs_rq) { + if (!qos_sched_enabled()) + return false; + if (unlikely(__this_cpu_read(qos_cpu_overload))) { return false; } @@ -7864,8 +7876,11 @@ void init_qos_hrtimer(int cpu) * we should schedule offline tasks to run so that they can leave kernel * critical sections, and throttle them before returning to user mode. */ -static void qos_schedule_throttle(struct task_struct *p) +static __always_inline void qos_schedule_throttle(struct task_struct *p) { + if (!qos_sched_enabled()) + return; + if (unlikely(current->flags & PF_KTHREAD)) return; @@ -7876,7 +7891,7 @@ static void qos_schedule_throttle(struct task_struct *p) } #ifndef CONFIG_QOS_SCHED_SMT_EXPELLER -static bool qos_smt_expelled(int this_cpu) +static __always_inline bool qos_smt_expelled(int this_cpu) { return false; } @@ -7952,8 +7967,11 @@ static bool qos_sched_idle_cpu(int this_cpu) rq->nr_running); } -static bool qos_smt_expelled(int this_cpu) +static __always_inline bool qos_smt_expelled(int this_cpu) { + if (!qos_sched_enabled()) + return false; + if (!static_branch_likely(&qos_smt_expell_switch)) return false; @@ -8011,8 +8029,11 @@ static void qos_smt_send_ipi(int this_cpu) } } -static void qos_smt_expel(int this_cpu, struct task_struct *p) +static __always_inline void qos_smt_expel(int this_cpu, struct task_struct *p) { + if (!qos_sched_enabled()) + return; + if (!static_branch_likely(&qos_smt_expell_switch)) return; @@ -8145,7 +8166,8 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf #ifdef CONFIG_FAIR_GROUP_SCHED if (!prev || prev->sched_class != &fair_sched_class) { #ifdef CONFIG_QOS_SCHED - if (cfs_rq->idle_h_nr_running != 0 && rq->online) + if (qos_sched_enabled() && + cfs_rq->idle_h_nr_running != 0 && rq->online) goto qos_simple; else #endif @@ -8661,7 +8683,7 @@ static int task_hot(struct task_struct *p, struct lb_env *env) #ifdef CONFIG_QOS_SCHED_PRIO_LB /* Preempt sched idle cpu do not consider migration cost */ - if (sysctl_sched_prio_load_balance_enabled && + if (qos_sched_enabled() && sysctl_sched_prio_load_balance_enabled && cpus_share_cache(env->src_cpu, env->dst_cpu) && sched_idle_cpu(env->dst_cpu)) return 0; @@ -8770,7 +8792,9 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) return 0; #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY - set_task_select_cpus(p, NULL, 0); + p->select_cpus = p->cpus_ptr; + if (dynamic_affinity_used()) + set_task_select_cpus(p, NULL, 0); if (!cpumask_test_cpu(env->dst_cpu, p->select_cpus)) { #else if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { @@ -8916,7 +8940,7 @@ static struct task_struct *detach_one_task(struct lb_env *env) return p; } #ifdef CONFIG_QOS_SCHED_PRIO_LB - if (sysctl_sched_prio_load_balance_enabled) { + if (qos_sched_enabled() && sysctl_sched_prio_load_balance_enabled) { loop++; if (loop == 1) { tasks = &env->src_rq->cfs_offline_tasks; @@ -9055,7 +9079,8 @@ static int detach_tasks(struct lb_env *env) } #ifdef CONFIG_QOS_SCHED_PRIO_LB - if (sysctl_sched_prio_load_balance_enabled && env->imbalance > 0) { + if (qos_sched_enabled() && sysctl_sched_prio_load_balance_enabled && + env->imbalance > 0) { /* * Avoid offline tasks starve to death if env->loop exceed * env->loop_max, so we should set env->loop to 0 and detach diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 27fd1240ac85..95ae9ac553fc 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3065,8 +3065,12 @@ static inline int is_offline_level(long qos_level) #endif #ifdef CONFIG_QOS_SCHED_SMT_EXPELLER -static inline int task_has_qos_idle_policy(struct task_struct *p) +static __always_inline int task_has_qos_idle_policy(struct task_struct *p) { + if (!qos_sched_enabled() || + !static_branch_likely(&qos_smt_expell_switch)) + return 0; + return qos_idle_policy(task_group(p)->qos_level) && p->policy == SCHED_IDLE; } #endif diff --git a/kernel/signal.c b/kernel/signal.c index cf498d949f2f..dd6076c6235f 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1050,7 +1050,8 @@ static void complete_signal(int sig, struct task_struct *p, enum pid_type type) t = p; do { #ifdef CONFIG_QOS_SCHED - sched_move_offline_task(t); + if (qos_sched_enabled()) + sched_move_offline_task(t); #endif task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK); sigaddset(&t->pending.signal, SIGKILL); -- 2.17.1