Signed-off-by: Liu Xinpeng liuxp11@chinatelecom.cn Signed-off-by: Ctyun Kernel ctyuncommiter01@chinatelecom.cn --- include/linux/cgroup-defs.h | 14 +++++++++++--- include/linux/cgroup.h | 3 ++- include/linux/kernfs.h | 8 +++++--- include/linux/kthread.h | 3 +++ include/linux/sched.h | 14 +++++++++----- kernel/cgroup/cgroup.c | 11 ++++++++--- kernel/fork.c | 2 +- kernel/kthread.c | 2 ++ kernel/sched/psi.c | 18 +++++++++++------- 9 files changed, 52 insertions(+), 23 deletions(-)
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index daf2233..421c9c9 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -443,9 +443,6 @@ struct cgroup { /* used to schedule release agent */ struct work_struct release_agent_work;
- /* used to track pressure stalls */ - struct psi_group psi; - /* used to store eBPF programs */ struct cgroup_bpf bpf;
@@ -456,6 +453,17 @@ struct cgroup { int ancestor_ids[]; };
+struct cgroup_psi { + struct cgroup cgrp; + /* used to track pressure stalls */ + struct psi_group psi; +}; + +static inline struct cgroup_psi *get_cgroup_psi(struct cgroup *cgrp) +{ + return container_of(cgrp, struct cgroup_psi, cgrp); +} + /* * A cgroup_root represents the root of a cgroup hierarchy, and may be * associated with a kernfs_root to form an active hierarchy. This is diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 0e1f062..d4a512eb 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -673,7 +673,8 @@ static inline void pr_cont_cgroup_path(struct cgroup *cgrp)
static inline struct psi_group *cgroup_psi(struct cgroup *cgrp) { - return &cgrp->psi; + struct cgroup_psi *cgroup = get_cgroup_psi(cgrp); + return &cgroup->psi; }
static inline void cgroup_init_kthreadd(void) diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index ab81a22..b8d7720 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -269,16 +269,18 @@ struct kernfs_ops { ssize_t (*write)(struct kernfs_open_file *of, char *buf, size_t bytes, loff_t off);
- __poll_t (*poll)(struct kernfs_open_file *of, - struct poll_table_struct *pt); - int (*mmap)(struct kernfs_open_file *of, struct vm_area_struct *vma);
#ifdef CONFIG_DEBUG_LOCK_ALLOC struct lock_class_key lockdep_key; #endif
+#ifndef __GENKSYMS__ + __poll_t (*poll)(struct kernfs_open_file *of, + struct poll_table_struct *pt); +#else KABI_RESERVE(1) +#endif KABI_RESERVE(2) };
diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 206f0c3..de1b645 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -4,6 +4,9 @@ /* Simple interface for creating and stopping kernel threads without mess. */ #include <linux/err.h> #include <linux/sched.h> +#ifdef __GENKSYMS__ +#include <linux/cgroup.h> +#endif
__printf(4, 5) struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), diff --git a/include/linux/sched.h b/include/linux/sched.h index aa9af0a..22915f2 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -715,7 +715,7 @@ struct task_struct { unsigned sched_contributes_to_load:1; unsigned sched_migrated:1; unsigned sched_remote_wakeup:1; -#ifdef CONFIG_PSI +#if (defined(CONFIG_PSI) && !defined(__GENKSYMS__)) unsigned sched_psi_wake_requeue:1; #endif
@@ -972,10 +972,6 @@ struct task_struct { siginfo_t *last_siginfo;
struct task_io_accounting ioac; -#ifdef CONFIG_PSI - /* Pressure stall state */ - unsigned int psi_flags; -#endif #ifdef CONFIG_TASK_XACCT /* Accumulated RSS usage: */ u64 acct_rss_mem1; @@ -1228,7 +1224,15 @@ struct task_struct { */ randomized_struct_fields_end
+#if (defined(CONFIG_PSI) && !defined(__GENKSYMS__)) +/* Pressure stall state */ + union { + unsigned int psi_flags; + unsigned long psi_flags_padding; + } s; +#else KABI_RESERVE(1) +#endif KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index c9f5127..32dd248 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3428,21 +3428,21 @@ static int cpu_stat_show(struct seq_file *seq, void *v) static int cgroup_io_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgroup = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup->id == 1 ? &psi_system : &cgroup->psi; + struct psi_group *psi = cgroup->id == 1 ? &psi_system : cgroup_psi(cgroup);
return psi_show(seq, psi, PSI_IO); } static int cgroup_memory_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgroup = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup->id == 1 ? &psi_system : &cgroup->psi; + struct psi_group *psi = cgroup->id == 1 ? &psi_system : cgroup_psi(cgroup);
return psi_show(seq, psi, PSI_MEM); } static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgroup = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup->id == 1 ? &psi_system : &cgroup->psi; + struct psi_group *psi = cgroup->id == 1 ? &psi_system : cgroup_psi(cgroup);
return psi_show(seq, psi, PSI_CPU); } @@ -4932,8 +4932,13 @@ static struct cgroup *cgroup_create(struct cgroup *parent) int ret;
/* allocate the cgroup and its ID, 0 is reserved for the root */ +#ifdef CONFIG_PSI + cgrp = kzalloc(struct_size(cgrp, ancestor_ids, (level + 1)) + sizeof + (cgroup_psi), GFP_KERNEL); +#else cgrp = kzalloc(struct_size(cgrp, ancestor_ids, (level + 1)), GFP_KERNEL); +#endif if (!cgrp) return ERR_PTR(-ENOMEM);
diff --git a/kernel/fork.c b/kernel/fork.c index 36d35d3..493b65a 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1899,7 +1899,7 @@ static __latent_entropy struct task_struct *copy_process( p->default_timer_slack_ns = current->timer_slack_ns;
#ifdef CONFIG_PSI - p->psi_flags = 0; + p->s.psi_flags = 0; #endif
task_io_accounting_init(&p->ioac); diff --git a/kernel/kthread.c b/kernel/kthread.c index 2ec3583..0c857a7 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -11,7 +11,9 @@ #include <linux/kthread.h> #include <linux/completion.h> #include <linux/err.h> +#ifndef __GENKSYMS__ #include <linux/cgroup.h> +#endif #include <linux/cpuset.h> #include <linux/unistd.h> #include <linux/file.h> diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 35b8c79..e0d3278 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -544,17 +544,17 @@ void psi_task_change(struct task_struct *task, int clear, int set) if (!task->pid) return;
- if (((task->psi_flags & set) || - (task->psi_flags & clear) != clear) && + if (((task->s.psi_flags & set) || + (task->s.psi_flags & clear) != clear) && !psi_bug) { printk_deferred(KERN_ERR "psi: inconsistent task state! task=%d:%s cpu=%d psi_flags=%x clear=%x set=%x\n", task->pid, task->comm, cpu, - task->psi_flags, clear, set); + task->s.psi_flags, clear, set); psi_bug = 1; }
- task->psi_flags &= ~clear; - task->psi_flags |= set; + task->s.psi_flags &= ~clear; + task->s.psi_flags |= set;
/* * Periodic aggregation shuts off if there is a period of no @@ -650,11 +650,13 @@ void psi_memstall_leave(unsigned long *flags) }
#ifdef CONFIG_CGROUPS -int psi_cgroup_alloc(struct cgroup *cgroup) +int psi_cgroup_alloc(struct cgroup *cgrp) { + struct cgroup_psi *cgroup; if (static_branch_likely(&psi_disabled)) return 0;
+ cgroup = get_cgroup_psi(cgrp); cgroup->psi.pcpu = alloc_percpu(struct psi_group_cpu); if (!cgroup->psi.pcpu) return -ENOMEM; @@ -662,11 +664,13 @@ int psi_cgroup_alloc(struct cgroup *cgroup) return 0; }
-void psi_cgroup_free(struct cgroup *cgroup) +void psi_cgroup_free(struct cgroup *cgrp) { + struct cgroup_psi *cgroup; if (static_branch_likely(&psi_disabled)) return;
+ cgroup = get_cgroup_psi(cgrp); cancel_delayed_work_sync(&cgroup->psi.avgs_work); free_percpu(cgroup->psi.pcpu); }