From: Peter Zijlstra <peterz@infradead.org> mainline inclusion from mainline-v6.12-rc1 commit 436f3eed5c69c1048a5754df6e3dbb291e5cccbd category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IDC9YK Reference: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commi... -------------------------------- Ensure the last put_prev_task() and the first set_next_task() always go together. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lore.kernel.org/r/20240813224016.158454756@infradead.org Conflicts: kernel/sched/core.c [Combine the logic in mainline into current version in function __pick_next_task().] Signed-off-by: Zicheng Qu <quzicheng@huawei.com> --- kernel/sched/core.c | 19 +++++-------------- kernel/sched/fair.c | 3 +-- kernel/sched/sched.h | 10 +++++++++- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6c5d59b2a2ec..37d25534a7a6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5945,8 +5945,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) /* Assume the next prioritized class is idle_sched_class */ if (!p) { p = pick_task_idle(rq); - put_prev_task(rq, prev); - set_next_task_first(rq, p); + put_prev_set_next_task(rq, prev, p); } return p; @@ -5970,9 +5969,8 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) if (p) { const struct sched_class *prev_class = prev->sched_class; - put_prev_task(rq, prev); - set_next_task_first(rq, p); - + put_prev_set_next_task(rq, prev, p); + if (class != prev_class && prev_class->switch_class) prev_class->switch_class(rq, p); return p; @@ -6063,13 +6061,8 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); next = rq->core_pick; - if (next != prev) { - put_prev_task(rq, prev); - set_next_task_first(rq, next); - } - rq->core_pick = NULL; - goto out; + goto out_set_next; } prev_balance(rq, prev, rf); @@ -6239,9 +6232,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) } out_set_next: - put_prev_task(rq, prev); - set_next_task_first(rq, next); -out: + put_prev_set_next_task(rq, prev, next); if (rq->core->core_forceidle_count && next == rq->idle) queue_core_balance(rq); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e0e1b7922978..f3a602864ed5 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10323,8 +10323,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf simple: #endif - put_prev_task(rq, prev); - set_next_task_fair(rq, p, true); + put_prev_set_next_task(rq, prev, p); return p; idle: diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index eaab584caf45..45c432c53716 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2609,8 +2609,16 @@ static inline void set_next_task(struct rq *rq, struct task_struct *next) next->sched_class->set_next_task(rq, next, false); } -static inline void set_next_task_first(struct rq *rq, struct task_struct *next) +static inline void put_prev_set_next_task(struct rq *rq, + struct task_struct *prev, + struct task_struct *next) { + WARN_ON_ONCE(rq->curr != prev); + + if (next == prev) + return; + + prev->sched_class->put_prev_task(rq, prev); next->sched_class->set_next_task(rq, next, true); } -- 2.34.1