[PATCH openEuler-1.0-LTS 03/41] share_pool: Introduce struct spg_proc_stat

From: Tang Yizhou tangyizhou@huawei.com

ascend inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4EUVI CVE: NA

-------------------------------------------------

In multi-group mode, we introduce struct spg_proc_stat which represents statistics in an sp_group for a process. And struct sp_proc_stat is the accumulation of all structs of spg_proc_stat for a process.

Signed-off-by: Tang Yizhou tangyizhou@huawei.com Reviewed-by: Ding Tianhong dingtianhong@huawei.com Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- include/linux/share_pool.h | 6 + mm/share_pool.c | 365 +++++++++++++++++++++++++++---------- 2 files changed, 276 insertions(+), 95 deletions(-)

diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 8ab4cfb2b2509..d4c0ec944adc1 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -6,6 +6,7 @@ #include <linux/notifier.h> #include <linux/vmalloc.h> #include <linux/printk.h> +#include <linux/hashtable.h>

#define SP_HUGEPAGE (1 << 0) #define SP_HUGEPAGE_ONLY (1 << 1) @@ -146,11 +147,16 @@ struct sp_walk_data { pmd_t *pmd; };

+/* we estimate a process ususally belongs to at most 16 sp-group */ +#define SP_PROC_HASH_BITS 4 + /* per process memory usage statistics indexed by tgid */ struct sp_proc_stat { atomic_t use_count; int tgid; struct mm_struct *mm; + struct mutex lock; /* protect hashtable */ + DECLARE_HASHTABLE(hash, SP_PROC_HASH_BITS); char comm[TASK_COMM_LEN]; /* * alloc amount minus free amount, may be negative when freed by diff --git a/mm/share_pool.c b/mm/share_pool.c index 3980e5632cdcf..4063dd0861cce 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -63,6 +63,8 @@ #define MAX_GROUP_FOR_TASK 3000 #define MAX_PROC_PER_GROUP 1024

+#define GROUP_NONE 0 + #define PF_DOMAIN_CORE 0x10000000 /* AOS CORE processes in sched.h */

/* mdc scene hack */ @@ -153,88 +155,214 @@ static struct sp_proc_stat *sp_get_proc_stat_ref_locked(int tgid) return stat; }

-/* - * The caller must - * 1. ensure no concurrency problem for task_struct and mm_struct. - * 2. hold sp_group_sem for sp_group_master (pay attention to ABBA deadlock) - */ -static struct sp_proc_stat *sp_init_proc_stat(struct task_struct *tsk, - struct mm_struct *mm) +static struct sp_proc_stat *sp_get_proc_stat(int tgid) { - struct sp_group_master *master; - bool exist; struct sp_proc_stat *stat; - int id, tgid = tsk->tgid; - int ret;

- master = sp_init_group_master_locked(mm, &exist); - if (IS_ERR(master)) - return (struct sp_proc_stat *)master; + down_read(&sp_proc_stat_sem); + stat = sp_get_proc_stat_locked(tgid); + up_read(&sp_proc_stat_sem); + return stat; +} + +/* user must call sp_proc_stat_drop() after use */ +struct sp_proc_stat *sp_get_proc_stat_ref(int tgid) +{ + struct sp_proc_stat *stat; + + down_read(&sp_proc_stat_sem); + stat = sp_get_proc_stat_ref_locked(tgid); + up_read(&sp_proc_stat_sem); + return stat; +} + +static struct sp_proc_stat *create_proc_stat(struct mm_struct *mm, + struct task_struct *tsk) +{ + struct sp_proc_stat *stat; + + stat = kmalloc(sizeof(*stat), GFP_KERNEL); + if (stat == NULL) { + pr_err_ratelimited("share pool: alloc proc stat failed, lack of memory\n"); + return ERR_PTR(-ENOMEM); + } + + atomic_set(&stat->use_count, 1); + atomic64_set(&stat->alloc_size, 0); + atomic64_set(&stat->k2u_size, 0); + stat->tgid = tsk->tgid; + stat->mm = mm; + mutex_init(&stat->lock); + hash_init(stat->hash); + get_task_comm(stat->comm, tsk); + + return stat; +} + +static struct sp_proc_stat *sp_init_proc_stat(struct sp_group_master *master, + struct mm_struct *mm, struct task_struct *tsk) +{ + struct sp_proc_stat *stat; + int id, alloc_id, tgid = tsk->tgid;

down_write(&sp_proc_stat_sem); id = master->sp_stat_id; if (id) { - /* other threads in the same process may have initialized it */ + /* may have been initialized */ stat = sp_get_proc_stat_locked(tgid); + up_write(&sp_proc_stat_sem); if (stat) { - up_write(&sp_proc_stat_sem); return stat; } else { up_write(&sp_proc_stat_sem); /* if enter this branch, that's our mistake */ - pr_err_ratelimited("share pool: proc stat invalid id %d\n", id); + WARN(1, "share pool: proc stat invalid id %d\n", id); return ERR_PTR(-EBUSY); } }

- stat = kzalloc(sizeof(*stat), GFP_KERNEL); - if (stat == NULL) { + stat = create_proc_stat(mm, tsk); + if (IS_ERR(stat)) { up_write(&sp_proc_stat_sem); - pr_err_ratelimited("share pool: alloc proc stat failed due to lack of memory\n"); - return ERR_PTR(-ENOMEM); + return stat; }

- atomic_set(&stat->use_count, 1); - atomic64_set(&stat->alloc_size, 0); - atomic64_set(&stat->k2u_size, 0); - stat->tgid = tgid; - stat->mm = mm; - get_task_comm(stat->comm, tsk); - - ret = idr_alloc(&sp_proc_stat_idr, stat, tgid, tgid + 1, GFP_KERNEL); - if (ret < 0) { + alloc_id = idr_alloc(&sp_proc_stat_idr, stat, tgid, tgid + 1, GFP_KERNEL); + if (alloc_id < 0) { up_write(&sp_proc_stat_sem); - pr_err_ratelimited("share pool: proc stat idr alloc failed %d\n", ret); + pr_err_ratelimited("share pool: proc stat idr alloc failed %d\n", alloc_id); kfree(stat); - return ERR_PTR(ret); + return ERR_PTR(alloc_id); }

- master->sp_stat_id = ret; + master->sp_stat_id = alloc_id; up_write(&sp_proc_stat_sem); + return stat; }

-static struct sp_proc_stat *sp_get_proc_stat(int tgid) +/* per process/sp-group memory usage statistics */ +struct spg_proc_stat { + int tgid; + int spg_id; /* 0 for non-group data, such as k2u_task */ + struct hlist_node pnode; /* hlist node in sp_proc_stat->hash */ + struct sp_proc_stat *proc_stat; + /* + * alloc amount minus free amount, may be negative when freed by + * another task in the same sp group. + */ + atomic64_t alloc_size; + atomic64_t k2u_size; +}; + +static void update_spg_proc_stat_alloc(unsigned long size, bool inc, + struct spg_proc_stat *stat) { - struct sp_proc_stat *stat; + struct sp_proc_stat *proc_stat = stat->proc_stat; + + if (inc) { + atomic64_add(size, &stat->alloc_size); + atomic64_add(size, &proc_stat->alloc_size); + } else { + atomic64_sub(size, &stat->alloc_size); + atomic64_sub(size, &proc_stat->alloc_size); + } +} + +static void update_spg_proc_stat_k2u(unsigned long size, bool inc, + struct spg_proc_stat *stat) +{ + struct sp_proc_stat *proc_stat = stat->proc_stat; + + if (inc) { + atomic64_add(size, &stat->k2u_size); + atomic64_add(size, &proc_stat->k2u_size); + + } else { + atomic64_sub(size, &stat->k2u_size); + atomic64_sub(size, &proc_stat->k2u_size); + } +} + +static struct spg_proc_stat *find_spg_proc_stat( + struct sp_proc_stat *proc_stat, int tgid, int spg_id) +{ + struct spg_proc_stat *stat = NULL; + + mutex_lock(&proc_stat->lock); + hash_for_each_possible(proc_stat->hash, stat, pnode, spg_id) { + if (stat->spg_id == spg_id) + break; + } + mutex_unlock(&proc_stat->lock);

- down_read(&sp_proc_stat_sem); - stat = sp_get_proc_stat_locked(tgid); - up_read(&sp_proc_stat_sem); return stat; }

-/* user must call sp_proc_stat_drop() after use */ -struct sp_proc_stat *sp_get_proc_stat_ref(int tgid) +static struct spg_proc_stat *create_spg_proc_stat( + struct sp_proc_stat *proc_stat, int tgid, int spg_id) { - struct sp_proc_stat *stat; + struct spg_proc_stat *stat;

- down_read(&sp_proc_stat_sem); - stat = sp_get_proc_stat_ref_locked(tgid); - up_read(&sp_proc_stat_sem); + stat = kmalloc(sizeof(struct spg_proc_stat), GFP_KERNEL); + if (stat == NULL) { + pr_err_ratelimited("share pool: no memory for spg proc stat\n"); + return ERR_PTR(-ENOMEM); + } + + stat->tgid = tgid; + stat->spg_id = spg_id; + stat->proc_stat = proc_stat; + atomic64_set(&stat->alloc_size, 0); + atomic64_set(&stat->k2u_size, 0); + + return stat; +} + +static struct spg_proc_stat *sp_init_spg_proc_stat( + struct sp_proc_stat *proc_stat, int tgid, int spg_id) +{ + struct spg_proc_stat *stat; + + stat = find_spg_proc_stat(proc_stat, tgid, spg_id); + if (stat) + return stat; + + stat = create_spg_proc_stat(proc_stat, tgid, spg_id); + if (IS_ERR(stat)) + return stat; + + mutex_lock(&proc_stat->lock); + hash_add(proc_stat->hash, &stat->pnode, stat->spg_id); + mutex_unlock(&proc_stat->lock); return stat; }

+/* + * The caller must + * 1. ensure no concurrency problem for task_struct and mm_struct. + * 2. hold sp_group_sem for sp_group_master (pay attention to ABBA deadlock) + */ +static struct spg_proc_stat *sp_init_process_stat(struct task_struct *tsk, + struct mm_struct *mm, int spg_id) +{ + struct sp_group_master *master; + bool exist; + struct sp_proc_stat *proc_stat; + struct spg_proc_stat *spg_proc_stat; + + master = sp_init_group_master_locked(mm, &exist); + if (IS_ERR(master)) + return (struct spg_proc_stat *)master; + + proc_stat = sp_init_proc_stat(master, mm, tsk); + if (IS_ERR(proc_stat)) + return (struct spg_proc_stat *)proc_stat; + + spg_proc_stat = sp_init_spg_proc_stat(proc_stat, tsk->tgid, spg_id); + return spg_proc_stat; +} + /* statistics of all sp area, protected by sp_area_lock */ struct sp_spa_stat { unsigned int total_num; @@ -373,6 +501,44 @@ static int spa_dec_usage(enum spa_type type, unsigned long size, bool is_dvpp) return 0; }

+static void update_spg_proc_stat(unsigned long size, bool inc, + struct spg_proc_stat *stat, enum spa_type type) +{ + if (unlikely(!stat)) { + sp_dump_stack(); + WARN(1, "share pool: null process stat\n"); + return; + } + + switch (type) { + case SPA_TYPE_ALLOC: + update_spg_proc_stat_alloc(size, inc, stat); + break; + case SPA_TYPE_K2TASK: + case SPA_TYPE_K2SPG: + update_spg_proc_stat_k2u(size, inc, stat); + break; + default: + WARN(1, "share pool: invalid stat type\n"); + } +} + +static void sp_update_process_stat(struct task_struct *tsk, bool inc, + int spg_id, struct sp_area *spa) +{ + struct spg_proc_stat *stat; + unsigned long size = spa->real_size; + enum spa_type type = spa->type; + + down_write(&sp_group_sem); + stat = sp_init_process_stat(tsk, tsk->mm, spg_id); + up_write(&sp_group_sem); + if (unlikely(IS_ERR(stat))) + return; + + update_spg_proc_stat(size, inc, stat, type); +} + static inline void check_interrupt_context(void) { if (unlikely(in_interrupt())) @@ -751,7 +917,7 @@ int sp_group_add_task(int pid, int spg_id) int ret = 0; bool id_newly_generated = false; struct sp_area *spa, *prev = NULL; - struct sp_proc_stat *stat; + struct spg_proc_stat *stat;

check_interrupt_context();

@@ -872,10 +1038,10 @@ int sp_group_add_task(int pid, int spg_id) }

/* per process statistics initialization */ - stat = sp_init_proc_stat(tsk, mm); + stat = sp_init_process_stat(tsk, mm, spg_id); if (IS_ERR(stat)) { ret = PTR_ERR(stat); - pr_err_ratelimited("share pool: init proc stat failed, ret %lx\n", PTR_ERR(stat)); + pr_err_ratelimited("share pool: init process stat failed, ret %lx\n", PTR_ERR(stat)); goto out_drop_group; }

@@ -954,9 +1120,7 @@ int sp_group_add_task(int pid, int spg_id) spin_unlock(&sp_area_lock); up_write(&spg->rw_lock);

- if (unlikely(ret)) - sp_proc_stat_drop(stat); - + /* no need to free spg_proc_stat, will be freed when process exits */ out_drop_group: if (unlikely(ret)) { if (mm->sp_group_master->count == 0) { @@ -1406,7 +1570,6 @@ static void __sp_free(struct sp_group *spg, unsigned long addr, int sp_free(unsigned long addr) { struct sp_area *spa; - struct sp_proc_stat *stat; int mode; loff_t offset; int ret = 0; @@ -1474,15 +1637,10 @@ int sp_free(unsigned long addr) up_read(&spa->spg->rw_lock);

/* pointer stat may be invalid because of kthread buff_module_guard_work */ - if (current->mm == NULL) { + if (current->mm == NULL) atomic64_sub(spa->real_size, &kthread_stat.alloc_size); - } else { - stat = sp_get_proc_stat(current->mm->sp_group_master->sp_stat_id); - if (stat) - atomic64_sub(spa->real_size, &stat->alloc_size); - else - WARN(1, "share pool: %s: null process stat\n", __func__); - } + else + sp_update_process_stat(current, false, spa->spg->id, spa);

drop_spa: __sp_area_drop(spa); @@ -1540,7 +1698,6 @@ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id) { struct sp_group *spg, *spg_tmp; struct sp_area *spa = NULL; - struct sp_proc_stat *stat; unsigned long sp_addr; unsigned long mmap_addr; void *p; /* return value */ @@ -1732,13 +1889,8 @@ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id) out: up_read(&spg->rw_lock);

- if (!IS_ERR(p)) { - stat = sp_get_proc_stat(current->mm->sp_group_master->sp_stat_id); - if (stat) - atomic64_add(size_aligned, &stat->alloc_size); - else - WARN(1, "share pool: %s: null process stat\n", __func__); - } + if (!IS_ERR(p)) + sp_update_process_stat(current, true, spg->id, spa);

/* this will free spa if mmap failed */ if (spa && !IS_ERR(spa)) @@ -1983,7 +2135,6 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size, unsigned int page_size = PAGE_SIZE; struct task_struct *tsk; struct mm_struct *mm; - struct sp_proc_stat *stat; int ret = 0, is_hugepage;

check_interrupt_context(); @@ -2025,25 +2176,27 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size, goto out_put_task; }

- /* - * Process statistics initialization. if the target process has been - * added to a sp group, then stat will be returned immediately. - */ - stat = sp_init_proc_stat(tsk, mm); - if (IS_ERR(stat)) { - uva = stat; - pr_err_ratelimited("share pool: init proc stat failed, ret %lx\n", PTR_ERR(stat)); - goto out_put_mm; - } - spg = __sp_find_spg(pid, SPG_ID_DEFAULT); if (spg == NULL) { /* k2u to task */ + struct spg_proc_stat *stat; + if (spg_id != SPG_ID_NONE && spg_id != SPG_ID_DEFAULT) { pr_err_ratelimited("share pool: k2task invalid spg id %d\n", spg_id); uva = ERR_PTR(-EINVAL); goto out_put_mm; } + + down_write(&sp_group_sem); + stat = sp_init_process_stat(tsk, mm, GROUP_NONE); + up_write(&sp_group_sem); + if (IS_ERR(stat)) { + uva = stat; + pr_err_ratelimited("share pool: k2u(task) init process stat failed, ret %lx\n", + PTR_ERR(stat)); + goto out_put_mm; + } + spa = sp_alloc_area(size_aligned, sp_flags, NULL, SPA_TYPE_K2TASK, tsk->tgid); if (IS_ERR(spa)) { pr_err_ratelimited("share pool: k2u(task) failed due to alloc spa failure " @@ -2058,7 +2211,12 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size, }

uva = sp_make_share_kva_to_task(kva_aligned, spa, mm); - goto accounting; + + if (!IS_ERR(uva)) + update_spg_proc_stat(size_aligned, true, stat, + SPA_TYPE_K2TASK); + + goto finish; }

down_read(&spg->rw_lock); @@ -2094,7 +2252,6 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size, uva = sp_make_share_kva_to_spg(kva_aligned, spa, spg); else uva = sp_make_share_kva_to_task(kva_aligned, spa, mm); - } else { /* group is dead, return -ENODEV */ pr_err_ratelimited("share pool: failed to make k2u, sp group is dead\n"); @@ -2102,10 +2259,12 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size, } up_read(&spg->rw_lock);

-accounting: + if (!IS_ERR(uva)) + sp_update_process_stat(tsk, true, spg_id, spa); + +finish: if (!IS_ERR(uva)) { uva = uva + (kva - kva_aligned); - atomic64_add(size_aligned, &stat->k2u_size); } else { /* associate vma and spa */ if (!vmalloc_area_clr_flag(spa, kva_aligned, VM_SHAREPOOL)) @@ -2445,7 +2604,6 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp unsigned long uva_aligned; unsigned long size_aligned; unsigned int page_size; - struct sp_proc_stat *stat; struct sp_group_node *spg_node;

/* @@ -2533,6 +2691,12 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp pr_err("share pool: failed to unmap VA %pK when munmap in unshare uva\n", (void *)uva_aligned); } + + if (unlikely(!current->mm)) + WARN(1, "share pool: unshare uva(to task) unexpected active kthread"); + else + sp_update_process_stat(current, false, GROUP_NONE, spa); + } else if (spa->type == SPA_TYPE_K2SPG) { if (spg_id < 0) { pr_err_ratelimited("share pool: unshare uva(to group) failed, invalid spg id %d\n", spg_id); @@ -2574,21 +2738,17 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp down_read(&spa->spg->rw_lock); __sp_free(spa->spg, uva_aligned, size_aligned, NULL); up_read(&spa->spg->rw_lock); - } - - sp_dump_stack();

- /* pointer stat may be invalid because of kthread buff_module_guard_work */ - if (current->mm == NULL) { - atomic64_sub(spa->real_size, &kthread_stat.k2u_size); - } else { - stat = sp_get_proc_stat(current->mm->sp_group_master->sp_stat_id); - if (stat) - atomic64_sub(spa->real_size, &stat->k2u_size); + if (current->mm == NULL) + atomic64_sub(spa->real_size, &kthread_stat.k2u_size); else - WARN(1, "share pool: %s: null process stat\n", __func__); + sp_update_process_stat(current, false, spa->spg->id, spa); + } else { + WARN(1, "share pool: unshare uva invalid spa type"); }

+ sp_dump_stack(); + out_clr_flag: /* deassociate vma and spa */ if (!vmalloc_area_clr_flag(spa, spa->kva, VM_SHAREPOOL)) @@ -2874,8 +3034,23 @@ __setup("enable_sp_multi_group_mode", enable_sp_multi_group_mode);

/*** Statistical and maintenance functions ***/

+static void free_process_spg_proc_stat(struct sp_proc_stat *proc_stat) +{ + int i; + struct spg_proc_stat *stat; + struct hlist_node *tmp; + + /* traverse proc_stat->hash locklessly as process is exiting */ + hash_for_each_safe(proc_stat->hash, i, tmp, stat, pnode) { + hash_del(&stat->pnode); + kfree(stat); + } +} + static void free_sp_proc_stat(struct sp_proc_stat *stat) { + free_process_spg_proc_stat(stat); + down_write(&sp_proc_stat_sem); stat->mm->sp_group_master->sp_stat_id = 0; idr_remove(&sp_proc_stat_idr, stat->tgid);