[PATCH openEuler-5.10 32/38] sched/core: Fix forceidle balancing

From: Peter Zijlstra peterz@infradead.org

mainline inclusion from mainline-v5.18-rc2 commit 5b6547ed97f4f5dfc23f8e3970af6d11d7b7ed7e category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I5OOWG CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

--------------------------------------------------------------------------

Steve reported that ChromeOS encounters the forceidle balancer being ran from rt_mutex_setprio()'s balance_callback() invocation and explodes.

Now, the forceidle balancer gets queued every time the idle task gets selected, set_next_task(), which is strictly too often. rt_mutex_setprio() also uses set_next_task() in the 'change' pattern:

queued = task_on_rq_queued(p); /* p->on_rq == TASK_ON_RQ_QUEUED */ running = task_current(rq, p); /* rq->curr == p */

if (queued) dequeue_task(...); if (running) put_prev_task(...);

/* change task properties */

if (queued) enqueue_task(...); if (running) set_next_task(...);

However, rt_mutex_setprio() will explicitly not run this pattern on the idle task (since priority boosting the idle task is quite insane). Most other 'change' pattern users are pidhash based and would also not apply to idle.

Also, the change pattern doesn't contain a __balance_callback() invocation and hence we could have an out-of-band balance-callback, which *should* trigger the WARN in rq_pin_lock() (which guards against this exact anti-pattern).

So while none of that explains how this happens, it does indicate that having it in set_next_task() might not be the most robust option.

Instead, explicitly queue the forceidle balancer from pick_next_task() when it does indeed result in forceidle selection. Having it here, ensures it can only be triggered under the __schedule() rq->lock instance, and hence must be ran from that context.

This also happens to clean up the code a little, so win-win.

Fixes: d2dfa17bc7de ("sched: Trivial forced-newidle balancer") Reported-by: Steven Rostedt rostedt@goodmis.org Signed-off-by: Peter Zijlstra (Intel) peterz@infradead.org Tested-by: T.J. Alumbaugh talumbau@chromium.org Link: https://lkml.kernel.org/r/20220330160535.GN8939@worktop.programming.kicks-as... Signed-off-by: Lin Shengwang linshengwang1@huawei.com Reviewed-by: lihua hucool.lihua@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/sched/core.c | 14 ++++++++++---- kernel/sched/idle.c | 1 - kernel/sched/sched.h | 6 ------ 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 058ad14cc162..1341087361ac 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4810,6 +4810,8 @@ static inline struct task_struct *pick_task(struct rq *rq)

extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);

+static void queue_core_balance(struct rq *rq); + static struct task_struct * pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) { @@ -4858,7 +4860,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) }

rq->core_pick = NULL; - return next; + goto out; }

put_prev_task_balance(rq, prev, rf); @@ -4900,7 +4902,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) */ WARN_ON_ONCE(fi_before); task_vruntime_update(rq, next, false); - goto done; + goto out_set_next; } }

@@ -5009,8 +5011,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) resched_curr(rq_i); }

-done: +out_set_next: set_next_task(rq, next); +out: + if (rq->core->core_forceidle && next == rq->idle) + queue_core_balance(rq); + return next; }

@@ -5107,7 +5113,7 @@ static void sched_core_balance(struct rq *rq)

static DEFINE_PER_CPU(struct callback_head, core_balance_head);

-void queue_core_balance(struct rq *rq) +static void queue_core_balance(struct rq *rq) { if (!sched_core_enabled(rq)) return; diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 58bcb9517dfe..3c6396d61a04 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -430,7 +430,6 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir { update_idle_core(rq); schedstat_inc(rq->sched_goidle); - queue_core_balance(rq); }

#ifdef CONFIG_SMP diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 610d126a7f44..4f057b00be4c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1265,8 +1265,6 @@ static inline bool sched_group_cookie_match(struct rq *rq, return false; }

-extern void queue_core_balance(struct rq *rq); - static inline bool sched_core_enqueued(struct task_struct *p) { return !RB_EMPTY_NODE(&p->core_node); @@ -1300,10 +1298,6 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq) return &rq->__lock; }

-static inline void queue_core_balance(struct rq *rq) -{ -} - static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p) { return true;