hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/ICAOAT -------------------------------- Add cgroup_ifs_enter/leave_lock helper for lock type interference. Signed-off-by: Pu Lehui <pulehui@huawei.com> --- include/linux/cgroup.h | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f4d155d3ef46..0e21c8b3fec0 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -10,6 +10,7 @@ */ #include <linux/sched.h> +#include <linux/sched/clock.h> #include <linux/cpumask.h> #include <linux/nodemask.h> #include <linux/rculist.h> @@ -860,8 +861,6 @@ static inline void cgroup_bpf_put(struct cgroup *cgrp) {} void cgroup_move_task_to_root(struct task_struct *tsk); #endif -#ifdef CONFIG_CGROUP_IFS - enum ifs_types { IFS_SMT, IFS_RUNDELAY, @@ -877,6 +876,8 @@ enum ifs_types { NR_IFS_TYPES, }; +#ifdef CONFIG_CGROUP_IFS + struct cgroup_ifs_cpu { /* total time for each interference, in ns */ u64 time[NR_IFS_TYPES]; @@ -927,6 +928,40 @@ static inline void cgroup_ifs_account_delta(struct cgroup_ifs_cpu *ifsc, ifsc->time[type] += delta; } +static inline u64 cgroup_ifs_time_counter(void) +{ + return sched_clock(); +} + +static inline void cgroup_ifs_enter_lock(u64 *clock) +{ + struct cgroup_ifs *ifs; + + if (!cgroup_ifs_enabled()) + return; + + ifs = current_ifs(); + if (ifs) + *clock = cgroup_ifs_time_counter(); +} + +static inline void cgroup_ifs_leave_lock(u64 clock, enum ifs_types t) +{ + u64 delta; + struct cgroup_ifs *ifs; + struct cgroup_ifs_cpu *ifsc; + + if (!cgroup_ifs_enabled()) + return; + + ifs = current_ifs(); + if (ifs) { + ifsc = this_cpu_ptr(ifs->pcpu); + delta = cgroup_ifs_time_counter() - clock; + cgroup_ifs_account_delta(ifsc, t, delta); + } +} + void cgroup_ifs_account_smttime(struct task_struct *prev, struct task_struct *next, struct task_struct *idle); @@ -1035,6 +1070,8 @@ void cgroup_ifs_enable_sleep_account(void); #endif #else /* !CONFIG_CGROUP_IFS */ +static inline void cgroup_ifs_enter_lock(u64 *clock) {} +static inline void cgroup_ifs_leave_lock(u64 clock, enum ifs_types t) {} static inline void cgroup_ifs_account_smttime(struct task_struct *prev, struct task_struct *next, struct task_struct *idle) {} -- 2.34.1

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/ICAOAT -------------------------------- Add mutex interference track support Signed-off-by: Pu Lehui <pulehui@huawei.com> --- include/linux/cgroup.h | 1 + kernel/cgroup/ifs.c | 3 +++ kernel/locking/mutex.c | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f8d65dedcd85..562e222202b2 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -863,6 +863,7 @@ void cgroup_move_task_to_root(struct task_struct *tsk); enum ifs_types { IFS_SPINLOCK, + IFS_MUTEX, IFS_SMT, IFS_RUNDELAY, IFS_WAKELAT, diff --git a/kernel/cgroup/ifs.c b/kernel/cgroup/ifs.c index 82817c7d821a..8af919a63ba5 100644 --- a/kernel/cgroup/ifs.c +++ b/kernel/cgroup/ifs.c @@ -201,6 +201,9 @@ static const char *ifs_type_name(int type) case IFS_SPINLOCK: name = "spinlock"; break; + case IFS_MUTEX: + name = "mutex"; + break; case IFS_SMT: name = "smt"; break; diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index d973fe6041bf..1003afe8dd74 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -24,6 +24,7 @@ #include <linux/sched/rt.h> #include <linux/sched/wake_q.h> #include <linux/sched/debug.h> +#include <linux/cgroup.h> #include <linux/export.h> #include <linux/spinlock.h> #include <linux/interrupt.h> @@ -572,6 +573,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas { struct mutex_waiter waiter; struct ww_mutex *ww; + u64 ifs_clock; int ret; if (!use_ww_ctx) @@ -647,6 +649,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas set_current_state(state); trace_contention_begin(lock, LCB_F_MUTEX); + cgroup_ifs_enter_lock(&ifs_clock); for (;;) { bool first; @@ -719,6 +722,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas skip_wait: /* got the lock - cleanup and rejoice! */ lock_acquired(&lock->dep_map, ip); + cgroup_ifs_leave_lock(ifs_clock, IFS_MUTEX); trace_contention_end(lock, 0); if (ww_ctx) @@ -732,6 +736,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas __set_current_state(TASK_RUNNING); __mutex_remove_waiter(lock, &waiter); err_early_kill: + cgroup_ifs_leave_lock(ifs_clock, IFS_MUTEX); trace_contention_end(lock, ret); raw_spin_unlock(&lock->wait_lock); debug_mutex_free_waiter(&waiter); -- 2.34.1

From: Xu Kuohai <xukuohai@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/ICAOAT -------------------------------- Add distribution histogram support Signed-off-by: Xu Kuohai <xukuohai@huawei.com> Signed-off-by: Pu Lehui <pulehui@huawei.com> --- include/linux/cgroup.h | 13 +++- kernel/cgroup/ifs.c | 158 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 531024a54442..ba66564f46a3 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -880,7 +880,14 @@ enum ifs_types { #ifdef CONFIG_CGROUP_IFS +#define CGROUP_IFS_HIST_SLOTS 64 + +struct cgroup_ifs_hist { + u64 counts[NR_IFS_TYPES][CGROUP_IFS_HIST_SLOTS]; +}; + struct cgroup_ifs_cpu { + struct cgroup_ifs_hist hist; /* total time for each interference, in ns */ u64 time[NR_IFS_TYPES]; }; @@ -926,8 +933,12 @@ static inline void cgroup_ifs_account_delta(struct cgroup_ifs_cpu *ifsc, if (!cgroup_ifs_enabled()) return; - if (delta > 0) + if (delta > 0) { + int idx = __builtin_clzll(delta); + + ifsc->hist.counts[type][idx]++; ifsc->time[type] += delta; + } } static inline u64 cgroup_ifs_time_counter(void) diff --git a/kernel/cgroup/ifs.c b/kernel/cgroup/ifs.c index 9aaadd649739..6243c9a98ffe 100644 --- a/kernel/cgroup/ifs.c +++ b/kernel/cgroup/ifs.c @@ -8,6 +8,21 @@ #include <linux/sched/clock.h> #include "cgroup-internal.h" +#define TDESC_MAX_SLOT 64 +#define TDESC_BUF_SIZE 32 + +enum { + IFS_TIMER_CLK, + IFS_TIMER_TSC, + IFS_TIMER_NUM, +}; + +/* time range description, for printing */ +struct ifs_tdesc { + char tdesc_str[TDESC_MAX_SLOT][TDESC_BUF_SIZE]; + int tdesc_num; +}; + /* smt interference */ struct smt_itf { u64 total_time; /* total time of all smt interferences */ @@ -26,6 +41,8 @@ static DEFINE_PER_CPU(struct smt_itf, smt_itf); static DEFINE_PER_CPU(struct smt_info, smt_info); static DEFINE_PER_CPU_READ_MOSTLY(int, smt_sibling) = -1; +static struct ifs_tdesc ifs_tdesc[IFS_TIMER_NUM]; + static DEFINE_PER_CPU(struct cgroup_ifs_cpu, cgrp_root_ifs_cpu); struct cgroup_ifs cgroup_root_ifs = { .pcpu = &cgrp_root_ifs_cpu, @@ -237,6 +254,70 @@ static const char *ifs_type_name(int type) return name; } +static int tdesc_print(char *buf, int size, u64 nsecs) +{ + int ret; + + if (nsecs < NSEC_PER_USEC) + ret = snprintf(buf, size, "%llu ns", nsecs); + else if (nsecs < NSEC_PER_MSEC) + ret = snprintf(buf, size, "%llu.%02llu us", + nsecs / NSEC_PER_USEC, + (nsecs % NSEC_PER_USEC) / 10); + else if (nsecs < NSEC_PER_SEC) + ret = snprintf(buf, size, "%llu.%02llu ms", + nsecs / NSEC_PER_MSEC, + (nsecs % NSEC_PER_MSEC) / 10000); + else + ret = snprintf(buf, size, "%llu.%02llu s", + nsecs / NSEC_PER_SEC, + (nsecs % NSEC_PER_SEC) / 10000000); + return ret; +} + +static void tdesc_init(struct ifs_tdesc *desc, u64 freq) +{ + u64 start = 1, end; + u64 left, right; + int len, size; + int i = 0; + char *buf; + + do { + end = start << 1; + if (freq == NSEC_PER_SEC) + left = start; + else + left = (((u64)NSEC_PER_SEC << 5) / freq * start) >> 5; + right = left << 1; + + len = 0; + size = TDESC_BUF_SIZE; + + buf = &desc->tdesc_str[TDESC_MAX_SLOT - 1 - i][0]; + len += snprintf(buf, size, "["); + len += tdesc_print(buf + len, size - len, left); + + /* less than 1 hour*/ + if (left < (u64)NSEC_PER_SEC * 3600ULL) { + len += snprintf(buf + len, size - len, ", "); + len += tdesc_print(buf + len, size - len, right); + len += snprintf(buf + len, size - len, ")"); + } else { + snprintf(buf + len, size - len, " .... )"); + desc->tdesc_num = TDESC_MAX_SLOT - 1 - i; + break; + } + start = end; + } while (++i < TDESC_MAX_SLOT); +} + +static void cgroup_ifs_tdesc_init(void) +{ + tdesc_init(&ifs_tdesc[IFS_TIMER_CLK], NSEC_PER_SEC); + tdesc_init(&ifs_tdesc[IFS_TIMER_TSC], this_cpu_read(ifs_tsc_freq)); +} + static u64 tsc_cycles_to_nsec(u64 tsc_cycles) { #if defined(__aarch64__) || defined(__x86_64__) @@ -305,6 +386,77 @@ static int print_sum_time(struct cgroup_ifs *ifs, struct seq_file *seq) seq_printf(seq, "%-18s%llu\n", ifs_type_name(i), time[i]); } + seq_puts(seq, "\n"); + + return 0; +} + +static int print_hist_count(struct cgroup_ifs *ifs, struct seq_file *seq) +{ + struct cgroup_ifs_hist *h; + struct ifs_tdesc *desc; + bool is_print_title; + const char *name; + u64 start; + int count; + int i, j; + int cpu; + + h = kzalloc(sizeof(struct cgroup_ifs_hist), GFP_KERNEL); + if (!h) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct cgroup_ifs_cpu *ifsc = per_cpu_ptr(ifs->pcpu, cpu); + + for (i = 0; i < NR_IFS_TYPES; i++) { + if (!should_print(i)) + continue; + for (j = 0; j < CGROUP_IFS_HIST_SLOTS; j++) + h->counts[i][j] += ifsc->hist.counts[i][j]; + } + } + + for (i = 0; i < NR_IFS_TYPES; i++) { + name = ifs_type_name(i); + is_print_title = false; + start = 1ULL << 63; + + if (!should_print(i)) + continue; + + if (i == IFS_SPINLOCK || i == IFS_MUTEX) + desc = &ifs_tdesc[IFS_TIMER_TSC]; + else + desc = &ifs_tdesc[IFS_TIMER_CLK]; + + count = 0; + for (j = 0; j < CGROUP_IFS_HIST_SLOTS; j++) { + if (j < desc->tdesc_num) { + count += h->counts[i][j]; + continue; + } else if (j == desc->tdesc_num) { + count += h->counts[i][j]; + } else { + count = h->counts[i][j]; + } + + if (count) { + if (unlikely(!is_print_title)) { + is_print_title = true; + seq_printf(seq, "%s distribution\n", name); + } + seq_printf(seq, "%-24s: %d\n", + desc->tdesc_str[j], count); + } + start /= 2; + } + + if (is_print_title) + seq_puts(seq, "\n"); + } + + kfree(h); return 0; } @@ -323,6 +475,10 @@ static int cgroup_ifs_show(struct seq_file *seq, void *v) if (ret) return ret; + ret = print_hist_count(ifs, seq); + if (ret) + return ret; + return 0; } @@ -345,6 +501,7 @@ static ssize_t cgroup_ifs_write(struct kernfs_open_file *of, char *buf, for_each_possible_cpu(cpu) { ifsc = per_cpu_ptr(ifs->pcpu, cpu); memset(ifsc->time, 0, sizeof(ifsc->time)); + memset(ifsc->hist.counts, 0, sizeof(ifsc->hist.counts)); } } @@ -379,6 +536,7 @@ void cgroup_ifs_init(void) return; BUG_ON(cgroup_init_cftypes(NULL, cgroup_ifs_files)); + cgroup_ifs_tdesc_init(); static_branch_enable(&cgrp_ifs_enabled); } -- 2.34.1