From: Peter Zijlstra <peterz@infradead.org> mainline inclusion from mainline-v5.14-rc1 commit 875feb41fd20f6bd6054c9e79a5bcd9da6d8d2b2 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I5OOWG CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------------------------------------------------- Stuff the meat of sched_core_put() into a work such that we can use sched_core_put() from atomic context. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Tested-by: Don Hiatt <dhiatt@digitalocean.com> Tested-by: Hongyu Ning <hongyu.ning@linux.intel.com> Tested-by: Vincent Guittot <vincent.guittot@linaro.org> Link: https://lkml.kernel.org/r/20210422123308.377455632@infradead.org Signed-off-by: Lin Shengwang <linshengwang1@huawei.com> Reviewed-by: lihua <hucool.lihua@huawei.com> Signed-off-by: Zheng Zengkai <zhengzengkai@huawei.com> --- kernel/sched/core.c | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 89f34366fd74..86a256b490fa 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -93,7 +93,7 @@ DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); */ static DEFINE_MUTEX(sched_core_mutex); -static int sched_core_count; +static atomic_t sched_core_count; static struct cpumask sched_core_mask; static void __sched_core_flip(bool enabled) @@ -161,18 +161,39 @@ static void __sched_core_disable(void) void sched_core_get(void) { + if (atomic_inc_not_zero(&sched_core_count)) + return; + mutex_lock(&sched_core_mutex); - if (!sched_core_count++) + if (!atomic_read(&sched_core_count)) __sched_core_enable(); + + smp_mb__before_atomic(); + atomic_inc(&sched_core_count); mutex_unlock(&sched_core_mutex); } -void sched_core_put(void) +static void __sched_core_put(struct work_struct *work) { - mutex_lock(&sched_core_mutex); - if (!--sched_core_count) + if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { __sched_core_disable(); - mutex_unlock(&sched_core_mutex); + mutex_unlock(&sched_core_mutex); + } +} + +void sched_core_put(void) +{ + static DECLARE_WORK(_work, __sched_core_put); + + /* + * "There can be only one" + * + * Either this is the last one, or we don't actually need to do any + * 'work'. If it is the last *again*, we rely on + * WORK_STRUCT_PENDING_BIT. + */ + if (!atomic_add_unless(&sched_core_count, -1, 1)) + schedule_work(&_work); } #endif /* CONFIG_SCHED_CORE */ -- 2.20.1