sched: load tracking optimization.
Yu Jiahua (3): sched: Introcude config option SCHED_OPTIMIZE_LOAD_TRACKING sched: Add switch for update_blocked_averages sched: Add frequency control for load update in scheduler_tick
include/linux/sched/sysctl.h | 13 ++++ init/Kconfig | 8 +++ kernel/sched/fair.c | 126 +++++++++++++++++++++++++++++++++++ kernel/sysctl.c | 27 ++++++++ 4 files changed, 174 insertions(+)
From: Yu Jiahua yujiahua1@huawei.com
hulk inclusion category: feature bugzilla: 175551 https://gitee.com/openeuler/kernel/issues/I4DVI3
--------------------------------
Introduce kernel config SCHED_OPTIMIZE_LOAD_TRACKING to prepare for kernel scheduler loadtracking optimize.
Signed-off-by: Yu Jiahua yujiahua1@huawei.com Reviewed-by: Chen Hui judy.chenhui@huawei.com Signed-off-by: Chen Jun chenjun102@huawei.com --- init/Kconfig | 8 ++++++++ 1 file changed, 8 insertions(+)
diff --git a/init/Kconfig b/init/Kconfig index b095baa2b83c..66d7f2708b95 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2077,6 +2077,14 @@ endmenu # General setup
source "arch/Kconfig"
+config SCHED_OPTIMIZE_LOAD_TRACKING + bool "Optimize scheduler load tracking" + default n + help + Optimize scheduler load tracking, when load balance is not important + in system, we close some load tracking in tick and enqueue or dequeue + task, in this way, we can save some unnecessary cpu overhead. + config RT_MUTEXES bool
From: Yu Jiahua yujiahua1@huawei.com
hulk inclusion category: feature bugzilla: 175551 https://gitee.com/openeuler/kernel/issues/I4DVI3
--------------------------------
Avaliable for switch on/off for update_blocked_averages, it helps reducing cpu usage when system in heavy load.
Signed-off-by: Yu Jiahua yujiahua1@huawei.com Reviewed-by: Chen Hui judy.chenhui@huawei.com Signed-off-by: Chen Jun chenjun102@huawei.com --- include/linux/sched/sysctl.h | 7 +++++++ kernel/sched/fair.c | 40 ++++++++++++++++++++++++++++++++++++ kernel/sysctl.c | 11 ++++++++++ 3 files changed, 58 insertions(+)
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 3c31ba88aca5..f07787335bf9 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -98,4 +98,11 @@ int sched_energy_aware_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); #endif
+#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING +extern int sysctl_blocked_averages(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +extern struct static_key_true sched_blocked_averages; +#endif + #endif /* _LINUX_SCHED_SYSCTL_H */ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7e8c23119867..c2597f5ddeea 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8001,6 +8001,39 @@ static void attach_tasks(struct lb_env *env) rq_unlock(env->dst_rq, &rf); }
+#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING +DEFINE_STATIC_KEY_TRUE(sched_blocked_averages); + +static void set_blocked_averages(bool enabled) +{ + if (enabled) + static_branch_enable(&sched_blocked_averages); + else + static_branch_disable(&sched_blocked_averages); +} + +int sysctl_blocked_averages(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table t; + int err; + int state = static_branch_likely(&sched_blocked_averages); + + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + t = *table; + t.data = &state; + err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (err < 0) + return err; + if (write) + set_blocked_averages(state); + + return err; +} +#endif + #ifdef CONFIG_NO_HZ_COMMON static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { @@ -8204,6 +8237,13 @@ static void update_blocked_averages(int cpu) rq_lock_irqsave(rq, &rf); update_rq_clock(rq);
+#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING + if (!static_branch_unlikely(&sched_blocked_averages)) { + rq_unlock_irqrestore(rq, &rf); + return; + } +#endif + decayed |= __update_blocked_others(rq, &done); decayed |= __update_blocked_fair(rq, &done);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index d88c28573393..7c12f9f50ef5 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1764,6 +1764,17 @@ static struct ctl_table kern_table[] = { }, #endif /* CONFIG_NUMA_BALANCING */ #endif /* CONFIG_SCHED_DEBUG */ +#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING + { + .procname = "sched_blocked_averages", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_blocked_averages, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif { .procname = "sched_rt_period_us", .data = &sysctl_sched_rt_period,
From: Yu Jiahua yujiahua1@huawei.com
hulk inclusion category: feature bugzilla: 175551 https://gitee.com/openeuler/kernel/issues/I4DVI3
--------------------------------
Avaliable for control thread load update frequency manually, by echo 10~30 > /proc/sys/kernel/sched_tick_update_load. Available for switch on/off update thread load in scheduler_tick.
Signed-off-by: Yu Jiahua yujiahua1@huawei.com Reviewed-by: Chen Hui judy.chenhui@huawei.com Signed-off-by: Chen Jun chenjun102@huawei.com --- include/linux/sched/sysctl.h | 6 +++ kernel/sched/fair.c | 86 ++++++++++++++++++++++++++++++++++++ kernel/sysctl.c | 16 +++++++ 3 files changed, 108 insertions(+)
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index f07787335bf9..b9ad7db37614 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -101,7 +101,13 @@ int sched_energy_aware_handler(struct ctl_table *table, int write, #ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING extern int sysctl_blocked_averages(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sysctl_tick_update_load(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sysctl_update_load_latency(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos);
+extern unsigned int sysctl_load_tracking_latency; +extern struct static_key_true sched_tick_update_load; extern struct static_key_true sched_blocked_averages; #endif
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c2597f5ddeea..0183b177ee6b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -38,6 +38,35 @@ unsigned int sysctl_sched_latency = 6000000ULL; static unsigned int normalized_sysctl_sched_latency = 6000000ULL;
+#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING +#define LANTENCY_MIN 10 +#define LANTENCY_MAX 30 +unsigned int sysctl_load_tracking_latency = LANTENCY_MIN; + +int sysctl_update_load_latency(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret; + int min = LANTENCY_MIN; + int max = LANTENCY_MAX; + int latency = sysctl_load_tracking_latency; + struct ctl_table t; + + t = *table; + t.data = &latency; + t.extra1 = &min; + t.extra2 = &max; + + ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (ret || !write) + return ret; + + sysctl_load_tracking_latency = latency; + + return 0; +} +#endif + /* * The initial- and re-scaling of tunables is configurable * @@ -3805,16 +3834,34 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s { u64 now = cfs_rq_clock_pelt(cfs_rq); int decayed; +#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING + u64 delta; +#endif
/* * Track task load average for carrying it to new CPU after migrated, and * track group sched_entity load average for task_h_load calc in migration */ +#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING + delta = now - se->avg.last_update_time; + delta >>= sysctl_load_tracking_latency; + + if (!delta) + return; + if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) __update_load_avg_se(now, cfs_rq, se);
decayed = update_cfs_rq_load_avg(now, cfs_rq); decayed |= propagate_entity_load_avg(se); +#else + if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) + __update_load_avg_se(now, cfs_rq, se); + + decayed = update_cfs_rq_load_avg(now, cfs_rq); + decayed |= propagate_entity_load_avg(se); +#endif +
if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
@@ -4551,6 +4598,38 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) cfs_rq->curr = NULL; }
+#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING +DEFINE_STATIC_KEY_TRUE(sched_tick_update_load); +static void set_tick_update_load(bool enabled) +{ + if (enabled) + static_branch_enable(&sched_tick_update_load); + else + static_branch_disable(&sched_tick_update_load); +} + +int sysctl_tick_update_load(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table t; + int err; + int state = static_branch_likely(&sched_tick_update_load); + + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + t = *table; + t.data = &state; + err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (err < 0) + return err; + if (write) + set_tick_update_load(state); + + return err; +} +#endif + static void entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) { @@ -4562,8 +4641,15 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) /* * Ensure that runnable average is periodically updated. */ +#ifdef CONFIG_SCHED_OPTIMIZE_LOAD_TRACKING + if (static_branch_likely(&sched_tick_update_load)) { + update_load_avg(cfs_rq, curr, UPDATE_TG); + update_cfs_group(curr); + } +#else update_load_avg(cfs_rq, curr, UPDATE_TG); update_cfs_group(curr); +#endif
#ifdef CONFIG_SCHED_HRTICK /* diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 7c12f9f50ef5..513cb217232a 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1774,6 +1774,22 @@ static struct ctl_table kern_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_tick_update_load", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_tick_update_load, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_load_tracking_latency", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_update_load_latency, + }, #endif { .procname = "sched_rt_period_us",
-
Zheng Zengkai