From: Tengda Wu <wutengda2@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/ICAOAT -------------------------------- Now that the throttle metric is in place, rundelay no longer needs to track throttle time. Signed-off-by: Tengda Wu <wutengda2@huawei.com> Signed-off-by: Pu Lehui <pulehui@huawei.com> --- include/linux/sched.h | 4 ++++ kernel/sched/core.c | 18 ++++++++++-------- kernel/sched/deadline.c | 4 ++-- kernel/sched/fair.c | 8 +++++--- kernel/sched/rt.c | 4 ++-- kernel/sched/sched.h | 11 ++++++++++- kernel/sched/stats.h | 27 +++++++++++++++++++++++++-- 7 files changed, 58 insertions(+), 18 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index a694cc11dea5..b31db870e23b 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -994,6 +994,10 @@ struct task_struct { /* delay due to memory thrashing */ unsigned in_thrashing:1; #endif +#ifdef CONFIG_CGROUP_IFS + /* Run delayed due to bandwidth throttling */ + KABI_FILL_HOLE(unsigned in_throttle:1) +#endif unsigned long atomic_flags; /* Flags requiring atomic access. */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7b77fa528c3c..082056ba8bdc 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -154,6 +154,16 @@ const_debug unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; __read_mostly int scheduler_running; +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) +int sched_task_is_throttled(struct task_struct *p, int cpu) +{ + if (p->sched_class->task_is_throttled) + return p->sched_class->task_is_throttled(p, cpu); + + return 0; +} +#endif + #ifdef CONFIG_SCHED_CORE DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); @@ -268,14 +278,6 @@ void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) resched_curr(rq); } -static int sched_task_is_throttled(struct task_struct *p, int cpu) -{ - if (p->sched_class->task_is_throttled) - return p->sched_class->task_is_throttled(p, cpu); - - return 0; -} - static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) { struct rb_node *node = &p->core_node; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index dd13bbe8c0e4..3aa12a73c9ce 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2715,7 +2715,7 @@ static void prio_changed_dl(struct rq *rq, struct task_struct *p, #endif } -#ifdef CONFIG_SCHED_CORE +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) static int task_is_throttled_dl(struct task_struct *p, int cpu) { return p->dl.dl_throttled; @@ -2754,7 +2754,7 @@ DEFINE_SCHED_CLASS(dl) = { .switched_to = switched_to_dl, .update_curr = update_curr_dl, -#ifdef CONFIG_SCHED_CORE +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) .task_is_throttled = task_is_throttled_dl, #endif }; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index be95fd6394bc..cf3f147cba4d 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -14348,7 +14348,11 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, return delta > 0; } +#else +static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} +#endif +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) static int task_is_throttled_fair(struct task_struct *p, int cpu) { struct cfs_rq *cfs_rq; @@ -14360,8 +14364,6 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu) #endif return throttled_hierarchy(cfs_rq); } -#else -static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} #endif #ifdef CONFIG_SCHED_STEAL @@ -15096,7 +15098,7 @@ DEFINE_SCHED_CLASS(fair) = { .task_change_group = task_change_group_fair, #endif -#ifdef CONFIG_SCHED_CORE +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) .task_is_throttled = task_is_throttled_fair, #endif diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 77bb7ee8cce0..e2eaa8ffd009 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2692,7 +2692,7 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) return 0; } -#ifdef CONFIG_SCHED_CORE +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) static int task_is_throttled_rt(struct task_struct *p, int cpu) { struct rt_rq *rt_rq; @@ -2740,7 +2740,7 @@ DEFINE_SCHED_CLASS(rt) = { .update_curr = update_curr_rt, -#ifdef CONFIG_SCHED_CORE +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) .task_is_throttled = task_is_throttled_rt, #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 761870540a21..41ba8ad1a753 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -546,6 +546,15 @@ extern void sched_release_group(struct task_group *tg); extern void sched_move_task(struct task_struct *tsk); +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) +extern int sched_task_is_throttled(struct task_struct *p, int cpu); +#else +static inline int sched_task_is_throttled(struct task_struct *p, int cpu) +{ + return 0; +} +#endif + #ifdef CONFIG_QOS_SCHED_SMART_GRID extern void start_auto_affinity(struct auto_affinity *auto_affi); extern void stop_auto_affinity(struct auto_affinity *auto_affi); @@ -2451,7 +2460,7 @@ struct sched_class { void (*task_change_group)(struct task_struct *p); #endif -#ifdef CONFIG_SCHED_CORE +#if defined(CONFIG_SCHED_CORE) || defined(CONFIG_CGROUP_IFS) int (*task_is_throttled)(struct task_struct *p, int cpu); #endif KABI_RESERVE(1) diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 6d40d4d55b27..12966561d5d4 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -134,6 +134,24 @@ __schedstats_from_se(struct sched_entity *se) #define QOS_THROTTLED 2 #endif +#ifdef CONFIG_CGROUP_IFS +static inline void ifs_account_rundelay(struct task_struct *task, u64 delta) +{ + /* + * No need to include bandwidth throttling time in rundelay, + * leave it to the throttle metric. + */ + if (unlikely(task->in_throttle)) { + task->in_throttle = 0; + return; + } + + cgroup_ifs_account_rundelay(task, delta); +} +#else +static inline void ifs_account_rundelay(struct task_struct *task, u64 delta) {} +#endif + #ifdef CONFIG_PSI void psi_task_change(struct task_struct *task, int clear, int set); void psi_task_switch(struct task_struct *prev, struct task_struct *next, @@ -241,7 +259,7 @@ static inline void sched_info_dequeue(struct rq *rq, struct task_struct *t) t->sched_info.last_queued = 0; t->sched_info.run_delay += delta; - cgroup_ifs_account_rundelay(t, delta); + ifs_account_rundelay(t, delta); rq_sched_info_dequeue(rq, delta); } @@ -264,7 +282,7 @@ static void sched_info_arrive(struct rq *rq, struct task_struct *t) t->sched_info.last_arrival = now; t->sched_info.pcount++; - cgroup_ifs_account_rundelay(t, delta); + ifs_account_rundelay(t, delta); rq_sched_info_arrive(rq, delta); } @@ -277,6 +295,11 @@ static inline void sched_info_enqueue(struct rq *rq, struct task_struct *t) { if (!t->sched_info.last_queued) t->sched_info.last_queued = rq_clock(rq); + +#ifdef CONFIG_CGROUP_IFS + if (!t->in_throttle && sched_task_is_throttled(t, task_cpu(t))) + t->in_throttle = 1; +#endif } /* -- 2.34.1