From: Peter Zijlstra <peterz@infradead.org> mainline inclusion from mainline-v6.12-rc1 commit fd03c5b8585562d60f8b597b4332d28f48abfe7d category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IDC9YK Reference: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commi... -------------------------------- The current rule is that: pick_next_task() := pick_task() + set_next_task(.first = true) And many classes implement it directly as such. Change things around to make pick_next_task() optional while also changing the definition to: pick_next_task(prev) := pick_task() + put_prev_task() + set_next_task(.first = true) The reason is that sched_ext would like to have a 'final' call that knows the next task. By placing put_prev_task() right next to set_next_task() (as it already is for sched_core) this becomes trivial. As a bonus, this is a nice cleanup on its own. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lore.kernel.org/r/20240813224016.051225657@infradead.org Conflicts: kernel/sched/core.c kernel/sched/deadline.c kernel/sched/fair.c kernel/sched/ext.c kernel/sched/idle.c [conflict details: idle.c: COnflicts with a91091aed1fa ("sched: More flexible use of CPU quota when CPU is idle"), so only pick the part modified by this mainline, and keep the logic of a91091aed1fa. fair.c: Conflicts with hulk inclusion ceea34dd528c ("sched: Implement the function of qos smt expeller"), only pick the part modified by this mainline, and keep the logic of ceea34dd528c. deadline.c: Use the mainline patch pick_task_dl() to replace the pick_next_task_dl(). core.c: Combine the logic in mainline into current version in function __pick_next_task(). ext.c: Add params with the core.c, due to lack of d7b01aef9dbd ("Merge branch 'tip/sched/core' into for-6.12") which is the mainline patch.] Signed-off-by: Zicheng Qu <quzicheng@huawei.com> --- kernel/sched/core.c | 32 +++++++++++++++++++++++--------- kernel/sched/deadline.c | 14 +------------- kernel/sched/ext.c | 9 ++++++++- kernel/sched/fair.c | 11 +++++------ kernel/sched/idle.c | 18 +++--------------- kernel/sched/rt.c | 13 +------------ kernel/sched/sched.h | 16 ++++++++++++---- kernel/sched/stop_task.c | 13 +------------ 8 files changed, 54 insertions(+), 72 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index e95934b083a4..6c5d59b2a2ec 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5944,8 +5944,9 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) /* Assume the next prioritized class is idle_sched_class */ if (!p) { + p = pick_task_idle(rq); put_prev_task(rq, prev); - p = pick_next_task_idle(rq); + set_next_task_first(rq, p); } return p; @@ -5953,16 +5954,29 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) restart: prev_balance(rq, prev, rf); - put_prev_task(rq, prev); for_each_active_class(class) { - p = class->pick_next_task(rq); - if (p) { - const struct sched_class *prev_class = prev->sched_class; + if (class->pick_next_task) { + p = class->pick_next_task(rq, prev); + if (p) { + const struct sched_class *prev_class = prev->sched_class; + + if (class != prev_class && prev_class->switch_class) + prev_class->switch_class(rq, p); + return p; + } + } else { + p = class->pick_task(rq); + if (p) { + const struct sched_class *prev_class = prev->sched_class; - if (class != prev_class && prev_class->switch_class) - prev_class->switch_class(rq, p); - return p; + put_prev_task(rq, prev); + set_next_task_first(rq, p); + + if (class != prev_class && prev_class->switch_class) + prev_class->switch_class(rq, p); + return p; + } } } @@ -6059,7 +6073,6 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) } prev_balance(rq, prev, rf); - put_prev_task(rq, prev); smt_mask = cpu_smt_mask(cpu); need_sync = !!rq->core->core_cookie; @@ -6226,6 +6239,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) } out_set_next: + put_prev_task(rq, prev); set_next_task_first(rq, next); out: if (rq->core->core_forceidle_count && next == rq->idle) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 30dec34d4e33..3b299653170c 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2031,17 +2031,6 @@ static struct task_struct *pick_task_dl(struct rq *rq) return p; } -static struct task_struct *pick_next_task_dl(struct rq *rq) -{ - struct task_struct *p; - - p = pick_task_dl(rq); - if (p) - set_next_task_dl(rq, p, true); - - return p; -} - static void put_prev_task_dl(struct rq *rq, struct task_struct *p) { struct sched_dl_entity *dl_se = &p->dl; @@ -2737,13 +2726,12 @@ DEFINE_SCHED_CLASS(dl) = { .wakeup_preempt = wakeup_preempt_dl, - .pick_next_task = pick_next_task_dl, + .pick_task = pick_task_dl, .put_prev_task = put_prev_task_dl, .set_next_task = set_next_task_dl, #ifdef CONFIG_SMP .balance = balance_dl, - .pick_task = pick_task_dl, .select_task_rq = select_task_rq_dl, .migrate_task_rq = migrate_task_rq_dl, .set_cpus_allowed = set_cpus_allowed_dl, diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index d0a8c936b10b..022fa6e7d093 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2783,14 +2783,21 @@ static struct task_struct *first_local_task(struct rq *rq) struct task_struct, scx.dsq_list.node); } -static struct task_struct *pick_next_task_scx(struct rq *rq) +static struct task_struct *pick_next_task_scx(struct rq *rq, + struct task_struct *prev) { struct task_struct *p; + if (prev->sched_class == &ext_sched_class) + put_prev_task_scx(rq, prev); + p = first_local_task(rq); if (!p) return NULL; + if (prev->sched_class != &ext_sched_class) + prev->sched_class->put_prev_task(rq, prev); + set_next_task_scx(rq, p, true); if (unlikely(!p->scx.slice)) { diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1b40cf7aecab..e0e1b7922978 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10247,7 +10247,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf se = &p->se; #ifdef CONFIG_FAIR_GROUP_SCHED - if (!prev || prev->sched_class != &fair_sched_class) { + if (prev->sched_class != &fair_sched_class) { #ifdef CONFIG_QOS_SCHED if (cfs_rq->idle_h_nr_running != 0 && rq->online) goto qos_simple; @@ -10323,8 +10323,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf simple: #endif - if (prev) - put_prev_task(rq, prev); + put_prev_task(rq, prev); set_next_task_fair(rq, p, true); return p; @@ -10382,9 +10381,9 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf return NULL; } -static struct task_struct *__pick_next_task_fair(struct rq *rq) +static struct task_struct *__pick_next_task_fair(struct rq *rq, struct task_struct *prev) { - return pick_next_task_fair(rq, NULL, NULL); + return pick_next_task_fair(rq, prev, NULL); } /* @@ -15283,13 +15282,13 @@ DEFINE_SCHED_CLASS(fair) = { .wakeup_preempt = check_preempt_wakeup_fair, + .pick_task = pick_task_fair, .pick_next_task = __pick_next_task_fair, .put_prev_task = put_prev_task_fair, .set_next_task = set_next_task_fair, #ifdef CONFIG_SMP .balance = balance_fair, - .pick_task = pick_task_fair, .select_task_rq = select_task_rq_fair, .migrate_task_rq = migrate_task_rq_fair, diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 8c3ae98a16f0..12081077893e 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -458,17 +458,8 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir schedstat_inc(rq->sched_goidle); } -#ifdef CONFIG_SMP -static struct task_struct *pick_task_idle(struct rq *rq) -{ - return rq->idle; -} -#endif - -struct task_struct *pick_next_task_idle(struct rq *rq) +struct task_struct *pick_task_idle(struct rq *rq) { - struct task_struct *next = rq->idle; - #ifdef CONFIG_SCHED_SOFT_QUOTA if (sched_feat(SOFT_QUOTA)) { if (unthrottle_cfs_rq_soft_quota(rq) && rq->cfs.nr_running) @@ -476,9 +467,7 @@ struct task_struct *pick_next_task_idle(struct rq *rq) } #endif - set_next_task_idle(rq, next, true); - - return next; + return rq->idle; } /* @@ -534,13 +523,12 @@ DEFINE_SCHED_CLASS(idle) = { .wakeup_preempt = wakeup_preempt_idle, - .pick_next_task = pick_next_task_idle, + .pick_task = pick_task_idle, .put_prev_task = put_prev_task_idle, .set_next_task = set_next_task_idle, #ifdef CONFIG_SMP .balance = balance_idle, - .pick_task = pick_task_idle, .select_task_rq = select_task_rq_idle, .set_cpus_allowed = set_cpus_allowed_common, #endif diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 7e624355d836..1e5dd3398349 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1808,16 +1808,6 @@ static struct task_struct *pick_task_rt(struct rq *rq) return p; } -static struct task_struct *pick_next_task_rt(struct rq *rq) -{ - struct task_struct *p = pick_task_rt(rq); - - if (p) - set_next_task_rt(rq, p, true); - - return p; -} - static void put_prev_task_rt(struct rq *rq, struct task_struct *p) { struct sched_rt_entity *rt_se = &p->rt; @@ -2706,13 +2696,12 @@ DEFINE_SCHED_CLASS(rt) = { .wakeup_preempt = wakeup_preempt_rt, - .pick_next_task = pick_next_task_rt, + .pick_task = pick_task_rt, .put_prev_task = put_prev_task_rt, .set_next_task = set_next_task_rt, #ifdef CONFIG_SMP .balance = balance_rt, - .pick_task = pick_task_rt, .select_task_rq = select_task_rq_rt, .set_cpus_allowed = set_cpus_allowed_common, .rq_online = rq_online_rt, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 80ccf379c6d4..05c98693f93b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2530,7 +2530,17 @@ struct sched_class { KABI_REPLACE(void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int flags), void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags)) - struct task_struct *(*pick_next_task)(struct rq *rq); + struct task_struct *(*pick_task)(struct rq *rq); + /* + * Optional! When implemented pick_next_task() should be equivalent to: + * + * next = pick_task(); + * if (next) { + * put_prev_task(prev); + * set_next_task_first(next); + * } + */ + struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *prev); void (*put_prev_task)(struct rq *rq, struct task_struct *p); void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); @@ -2540,8 +2550,6 @@ struct sched_class { struct task_struct *prev, struct rq_flags *rf)) int (*select_task_rq)(struct task_struct *p, int task_cpu, int flags); - struct task_struct * (*pick_task)(struct rq *rq); - void (*migrate_task_rq)(struct task_struct *p, int new_cpu); void (*task_woken)(struct rq *this_rq, struct task_struct *task); @@ -2692,7 +2700,7 @@ static inline bool sched_fair_runnable(struct rq *rq) } extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); -extern struct task_struct *pick_next_task_idle(struct rq *rq); +extern struct task_struct *pick_task_idle(struct rq *rq); #define SCA_CHECK 0x01 #define SCA_MIGRATE_DISABLE 0x02 diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index 4cf02074fa9e..0fd5352ff0ce 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -41,16 +41,6 @@ static struct task_struct *pick_task_stop(struct rq *rq) return rq->stop; } -static struct task_struct *pick_next_task_stop(struct rq *rq) -{ - struct task_struct *p = pick_task_stop(rq); - - if (p) - set_next_task_stop(rq, p, true); - - return p; -} - static void enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) { @@ -112,13 +102,12 @@ DEFINE_SCHED_CLASS(stop) = { .wakeup_preempt = wakeup_preempt_stop, - .pick_next_task = pick_next_task_stop, + .pick_task = pick_task_stop, .put_prev_task = put_prev_task_stop, .set_next_task = set_next_task_stop, #ifdef CONFIG_SMP .balance = balance_stop, - .pick_task = pick_task_stop, .select_task_rq = select_task_rq_stop, .set_cpus_allowed = set_cpus_allowed_common, #endif -- 2.34.1