From: ZhangPeng zhangpeng362@huawei.com

maillist inclusion category: performance bugzilla: https://gitee.com/openeuler/kernel/issues/I9IA1I CVE: NA

Reference: https://lore.kernel.org/all/20240418142008.2775308-1-zhangpeng362@huawei.com...

--------------------------------

Depending on whether counters is NULL, we can support two modes: atomic mode and perpcu mode. We implement both modes by grouping the s64 count and atomic64_t count_atomic in a union. At the same time, we create the interface for adding and reading in atomic mode and for switching atomic mode to percpu mode.

Suggested-by: Jan Kara jack@suse.cz Signed-off-by: ZhangPeng zhangpeng362@huawei.com Signed-off-by: Kefeng Wang wangkefeng.wang@huawei.com --- include/linux/percpu_counter.h | 48 +++++++++++++++++++++++++++++++--- lib/percpu_counter.c | 35 +++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 5 deletions(-)

diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h index d01351b1526f..1a0f25a27d7b 100644 --- a/include/linux/percpu_counter.h +++ b/include/linux/percpu_counter.h @@ -21,7 +21,18 @@

struct percpu_counter { raw_spinlock_t lock; - s64 count; + /* + * Depending on whether counters is NULL, we can support two modes, + * atomic mode using count_atomic and perpcu mode using count. + * The single-thread processes should use atomic mode to reduce the + * memory consumption and performance regression. + * The multiple-thread processes should use percpu mode to reduce the + * error margin. + */ + union { + s64 count; + atomic64_t count_atomic; + }; #ifdef CONFIG_HOTPLUG_CPU struct list_head list; /* All percpu_counters are on a list */ #endif @@ -32,14 +43,14 @@ extern int percpu_counter_batch;

int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount, gfp_t gfp, u32 nr_counters, - struct lock_class_key *key); + struct lock_class_key *key, bool switch_mode);

#define percpu_counter_init_many(fbc, value, gfp, nr_counters) \ ({ \ static struct lock_class_key __key; \ \ __percpu_counter_init_many(fbc, value, gfp, nr_counters,\ - &__key); \ + &__key, false); \ })

@@ -121,6 +132,20 @@ static inline bool percpu_counter_initialized(struct percpu_counter *fbc) return (fbc->counters != NULL); }

+static inline s64 percpu_counter_atomic_read(struct percpu_counter *fbc) +{ + return atomic64_read(&fbc->count_atomic); +} + +static inline void percpu_counter_atomic_add(struct percpu_counter *fbc, + s64 amount) +{ + atomic64_add(amount, &fbc->count_atomic); +} + +int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc, + u32 nr_counters); + #else /* !CONFIG_SMP */

struct percpu_counter { @@ -230,6 +255,23 @@ static inline bool percpu_counter_initialized(struct percpu_counter *fbc) static inline void percpu_counter_sync(struct percpu_counter *fbc) { } + +static inline s64 percpu_counter_atomic_read(struct percpu_counter *fbc) +{ + return fbc->count; +} + +static inline void percpu_counter_atomic_add(struct percpu_counter *fbc, + s64 amount) +{ + percpu_counter_add(fbc, amount); +} + +static inline int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc, + u32 nr_counters) +{ + return 0; +} #endif /* CONFIG_SMP */

static inline void percpu_counter_inc(struct percpu_counter *fbc) diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index 9073430dc865..7d2eaba4db1d 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -153,7 +153,7 @@ EXPORT_SYMBOL(__percpu_counter_sum);

int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount, gfp_t gfp, u32 nr_counters, - struct lock_class_key *key) + struct lock_class_key *key, bool switch_mode) { unsigned long flags __maybe_unused; size_t counter_size; @@ -174,7 +174,8 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount, #ifdef CONFIG_HOTPLUG_CPU INIT_LIST_HEAD(&fbc[i].list); #endif - fbc[i].count = amount; + if (likely(!switch_mode)) + fbc[i].count = amount; fbc[i].counters = (void *)counters + (i * counter_size);

debug_percpu_counter_activate(&fbc[i]); @@ -278,6 +279,36 @@ int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch) } EXPORT_SYMBOL(__percpu_counter_compare);

+/* + * percpu_counter_switch_to_pcpu_many: Converts struct percpu_counters from + * atomic mode to percpu mode. + * + * Return: 0 if percpu_counter is already in atomic mode or successfully + * switched to atomic mode; -ENOMEM if perpcu memory allocation fails, + * perpcu_counter is still in atomic mode. + */ +int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc, + u32 nr_counters) +{ + static struct lock_class_key __key; + unsigned long flags; + bool ret = 0; + + if (percpu_counter_initialized(fbc)) + return 0; + + preempt_disable(); + local_irq_save(flags); + if (likely(!percpu_counter_initialized(fbc))) + ret = __percpu_counter_init_many(fbc, 0, + GFP_ATOMIC|__GFP_NOWARN|__GFP_ZERO, + nr_counters, &__key, true); + local_irq_restore(flags); + preempt_enable(); + + return ret; +} + static int __init percpu_counter_startup(void) { int ret;

From: ZhangPeng zhangpeng362@huawei.com

hulk inclusion category: performance bugzilla: https://gitee.com/openeuler/kernel/issues/I9IA1I

---------------------------

Depending on whether counters is NULL, we can support two modes for mm counter, atomic mode and perpcu mode. Currently, the mm counter atomic mode is enabled by default. Introduce cmdline interface disable_mm_counter_atomic to disable mm counter atomic mode, which changes mm_counter_atomic_enable from true to false.

Signed-off-by: ZhangPeng zhangpeng362@huawei.com --- kernel/fork.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+)

diff --git a/kernel/fork.c b/kernel/fork.c index d75c18892eb6..9b16283a125c 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -154,6 +154,8 @@ int lockdep_tasklist_lock_is_held(void) EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); #endif /* #ifdef CONFIG_PROVE_RCU */

+static bool __ro_after_init mm_counter_atomic_enable = true; + int nr_processes(void) { int cpu; @@ -1307,6 +1309,25 @@ static void mm_init_uprobes_state(struct mm_struct *mm) #endif }

+static __always_inline int mm_counter_init(struct mm_struct *mm) +{ + /* + * Depending on whether counters is NULL, we can support two modes for + * mm counter, atomic mode and perpcu mode. Currently, the mm counter + * atomic mode is enabled by default. Introduce cmdline interface + * disable_mm_counter_atomic to disable mm counter atomic mode, which + * changes mm_counter_atomic_enable from true to false. + */ + if (mm_counter_atomic_enable) + return 0; + + if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT, + NR_MM_COUNTERS)) + return -ENOMEM; + + return 0; +} + static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, struct user_namespace *user_ns) { @@ -1357,11 +1378,16 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, if (mm_alloc_cid(mm)) goto fail_cid;

+ if (mm_counter_init(mm)) + goto fail_pcpu; + sp_init_mm(mm); mm->user_ns = get_user_ns(user_ns); lru_gen_init_mm(mm); return mm;

+fail_pcpu: + mm_destroy_cid(mm); fail_cid: destroy_context(mm); fail_nocontext: @@ -3627,3 +3653,11 @@ int sysctl_max_threads(struct ctl_table *table, int write,

return 0; } + +static int __init disable_mm_counter_atomic(char *buf) +{ + mm_counter_atomic_enable = false; + + return 0; +} +early_param("disable_mm_counter_atomic", disable_mm_counter_atomic);