From: Ding Tianhong <dingtianhong(a)huawei.com>
ascend inclusion
category: feature
bugzilla: NA
CVE: NA
-------------------------------------------------
The sp_mutex is used to protect all critical path for share pool,
it has serious affected the performance of the the memory alloc
and release interface when there is a lot of process in the same
memory group, it will serious break the scailability of the system,
so add a new read semaphore lock to instead of the big lock for allocation
and release critical path.
The scailability has been greatly improved by this modification.
Show the test result:
number of process: alloc 4M avg time:
Before the patch: 1 32us
3 96us
10 330us
after the patch: 1 32us
3 40us
10 60us
v2: fix some conflicts and clean some code.
Signed-off-by: Ding Tianhong <dingtianhong(a)huawei.com>
Reviewed-by: Tang Yizhou <tangyizhou(a)huawei.com>
Reviewed-by: Kefeng Wang <wangkefeng.wang(a)huawei.com>
Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com>
---
include/linux/share_pool.h | 5 ++
kernel/fork.c | 4 +-
mm/share_pool.c | 170 ++++++++++++++++++++-----------------
3 files changed, 100 insertions(+), 79 deletions(-)
diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h
index 70b841d0eb8e..f2d17cb85fa5 100644
--- a/include/linux/share_pool.h
+++ b/include/linux/share_pool.h
@@ -93,6 +93,8 @@ struct sp_group {
unsigned long dvpp_va_start;
unsigned long dvpp_size;
atomic_t use_count;
+ /* protect the group internal elements */
+ struct rw_semaphore rw_lock;
};
struct sp_walk_data {
@@ -238,6 +240,8 @@ extern void *vmalloc_hugepage_user(unsigned long size);
extern void *buff_vzalloc_user(unsigned long size);
extern void *buff_vzalloc_hugepage_user(unsigned long size);
+void sp_exit_mm(struct mm_struct *mm);
+
#else
static inline int sp_group_add_task(int pid, int spg_id)
@@ -400,6 +404,7 @@ static inline void *buff_vzalloc_hugepage_user(unsigned long size)
{
return NULL;
}
+
#endif
#endif /* LINUX_SHARE_POOL_H */
diff --git a/kernel/fork.c b/kernel/fork.c
index d1d8ac083c80..61496b70cfb8 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1055,8 +1055,6 @@ static inline void __mmput(struct mm_struct *mm)
{
VM_BUG_ON(atomic_read(&mm->mm_users));
- sp_group_exit(mm);
-
uprobe_clear_state(mm);
exit_aio(mm);
ksm_exit(mm);
@@ -1084,6 +1082,8 @@ void mmput(struct mm_struct *mm)
{
might_sleep();
+ sp_group_exit(mm);
+
if (atomic_dec_and_test(&mm->mm_users))
__mmput(mm);
}
diff --git a/mm/share_pool.c b/mm/share_pool.c
index e326c95104da..27792a641401 100644
--- a/mm/share_pool.c
+++ b/mm/share_pool.c
@@ -197,6 +197,16 @@ static bool host_svm_sp_enable = false;
int sysctl_share_pool_hugepage_enable = 1;
+static void free_sp_group(struct sp_group *spg);
+
+static bool sp_group_get(struct sp_group *spg)
+{
+ if (spg_valid(spg) && atomic_inc_not_zero(&spg->use_count))
+ return true;
+
+ return false;
+}
+
static unsigned long spa_size(struct sp_area *spa)
{
return spa->real_size;
@@ -337,7 +347,9 @@ static struct sp_group *__sp_find_spg(int pid, int spg_id)
put_task_struct(tsk);
} else {
+ mutex_lock(&sp_mutex);
spg = idr_find(&sp_group_idr, spg_id);
+ mutex_unlock(&sp_mutex);
}
return spg;
@@ -392,6 +404,8 @@ static struct sp_group *find_or_alloc_sp_group(int spg_id)
INIT_LIST_HEAD(&spg->procs);
INIT_LIST_HEAD(&spg->spa_list);
+ init_rwsem(&spg->rw_lock);
+
ret = idr_alloc(&sp_group_idr, spg, spg_id, spg_id+1,
GFP_KERNEL);
if (ret < 0) {
@@ -422,9 +436,8 @@ static struct sp_group *find_or_alloc_sp_group(int spg_id)
goto out_fput;
}
} else {
- if (!spg_valid(spg))
+ if (!sp_group_get(spg))
return ERR_PTR(-ENODEV);
- atomic_inc(&spg->use_count);
}
return spg;
@@ -607,6 +620,8 @@ int sp_group_add_task(int pid, int spg_id)
}
mm->sp_group = spg;
+
+ down_write(&spg->rw_lock);
/* We reactive the spg even the spg exists already. */
spg->is_alive = true;
list_add_tail(&mm->sp_node, &spg->procs);
@@ -675,11 +690,14 @@ int sp_group_add_task(int pid, int spg_id)
mm->sp_group = NULL;
}
+ up_write(&spg->rw_lock);
out_drop_group:
if (unlikely(ret))
__sp_group_drop_locked(spg);
out_put_mm:
- mmput(mm);
+ /* No need to put the mm if the sp group add this mm success.*/
+ if (unlikely(ret))
+ mmput(mm);
out_put_task:
put_task_struct(tsk);
out_unlock:
@@ -712,44 +730,12 @@ static void spg_exit_unlock(bool unlock)
mutex_unlock(&sp_mutex);
}
-/*
- * Do cleanup when a process exits.
- */
-void sp_group_exit(struct mm_struct *mm)
-{
- bool is_alive = true;
- bool unlock;
-
- /*
- * Nothing to do if this thread group doesn't belong to any sp_group.
- * No need to protect this check with lock because we can add a task
- * to a group if !PF_EXITING.
- */
- if (!mm->sp_group)
- return;
-
- spg_exit_lock(&unlock);
- if (list_is_singular(&mm->sp_group->procs))
- is_alive = mm->sp_group->is_alive = false;
- list_del(&mm->sp_node);
- spg_exit_unlock(unlock);
-
- /*
- * To avoid calling this with sp_mutex held, we first mark the
- * sp_group as dead and then send the notification and then do
- * the real cleanup in sp_group_post_exit().
- */
- if (!is_alive)
- blocking_notifier_call_chain(&sp_notifier_chain, 0,
- mm->sp_group);
-}
-
void sp_group_post_exit(struct mm_struct *mm)
{
struct sp_proc_stat *stat;
bool unlock;
- if (!mm->sp_group)
+ if (!enable_ascend_share_pool || !mm->sp_group)
return;
spg_exit_lock(&unlock);
@@ -1139,8 +1125,6 @@ static void sp_munmap(struct mm_struct *mm, unsigned long addr,
{
int err;
- if (!mmget_not_zero(mm))
- return;
down_write(&mm->mmap_sem);
err = do_munmap(mm, addr, size, NULL);
@@ -1150,7 +1134,6 @@ static void sp_munmap(struct mm_struct *mm, unsigned long addr,
}
up_write(&mm->mmap_sem);
- mmput(mm);
}
/* The caller must hold sp_mutex. */
@@ -1183,8 +1166,6 @@ int sp_free(unsigned long addr)
check_interrupt_context();
- mutex_lock(&sp_mutex);
-
/*
* Access control: a share pool addr can only be freed by another task
* in the same spg or a kthread (such as buff_module_guard_work)
@@ -1217,6 +1198,8 @@ int sp_free(unsigned long addr)
sp_dump_stack();
+ down_read(&spa->spg->rw_lock);
+
__sp_free(spa->spg, spa->va_start, spa_size(spa), NULL);
/* Free the memory of the backing shmem or hugetlbfs */
@@ -1226,6 +1209,9 @@ int sp_free(unsigned long addr)
if (ret)
pr_err("share pool: sp free fallocate failed: %d\n", ret);
+ up_read(&spa->spg->rw_lock);
+
+ mutex_lock(&sp_mutex);
/* pointer stat may be invalid because of kthread buff_module_guard_work */
if (current->mm == NULL) {
kthread_stat.alloc_size -= spa->real_size;
@@ -1236,12 +1222,11 @@ int sp_free(unsigned long addr)
else
BUG();
}
+ mutex_unlock(&sp_mutex);
drop_spa:
__sp_area_drop(spa);
out:
- mutex_unlock(&sp_mutex);
-
sp_try_to_compact();
return ret;
}
@@ -1317,9 +1302,7 @@ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id)
if (sp_flags & SP_HUGEPAGE_ONLY)
sp_flags |= SP_HUGEPAGE;
- mutex_lock(&sp_mutex);
spg = __sp_find_spg(current->pid, SPG_ID_DEFAULT);
- mutex_unlock(&sp_mutex);
if (!spg) { /* DVPP pass through scene: first call sp_alloc() */
/* mdc scene hack */
if (enable_mdc_default_group)
@@ -1336,14 +1319,16 @@ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id)
ret);
return ERR_PTR(ret);
}
- mutex_lock(&sp_mutex);
spg = current->mm->sp_group;
} else { /* other scenes */
- mutex_lock(&sp_mutex);
if (spg_id != SPG_ID_DEFAULT) {
+ mutex_lock(&sp_mutex);
/* the caller should be a member of the sp group */
- if (spg != idr_find(&sp_group_idr, spg_id))
+ if (spg != idr_find(&sp_group_idr, spg_id)) {
+ mutex_unlock(&sp_mutex);
goto out;
+ }
+ mutex_unlock(&sp_mutex);
}
}
@@ -1352,6 +1337,7 @@ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id)
goto out;
}
+ down_read(&spg->rw_lock);
if (sp_flags & SP_HUGEPAGE) {
file = spg->file_hugetlb;
size_aligned = ALIGN(size, PMD_SIZE);
@@ -1376,31 +1362,25 @@ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id)
unsigned long populate = 0;
struct vm_area_struct *vma;
- if (!mmget_not_zero(mm))
- continue;
-
down_write(&mm->mmap_sem);
mmap_addr = sp_mmap(mm, file, spa, &populate);
if (IS_ERR_VALUE(mmap_addr)) {
up_write(&mm->mmap_sem);
p = (void *)mmap_addr;
__sp_free(spg, sp_addr, size_aligned, mm);
- mmput(mm);
pr_err("share pool: allocation sp mmap failed, ret %ld\n", mmap_addr);
goto out;
}
- p =(void *)mmap_addr; /* success */
+ p = (void *)mmap_addr; /* success */
if (populate == 0) {
up_write(&mm->mmap_sem);
- mmput(mm);
continue;
}
vma = find_vma(mm, sp_addr);
if (unlikely(!vma)) {
up_write(&mm->mmap_sem);
- mmput(mm);
pr_err("share pool: allocation failed due to find %pK vma failure\n",
(void *)sp_addr);
p = ERR_PTR(-EINVAL);
@@ -1461,24 +1441,22 @@ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id)
size_aligned = ALIGN(size, PAGE_SIZE);
sp_flags &= ~SP_HUGEPAGE;
__sp_area_drop(spa);
- mmput(mm);
goto try_again;
}
}
-
- mmput(mm);
break;
}
- mmput(mm);
}
+out:
+ up_read(&spg->rw_lock);
+
+ mutex_lock(&sp_mutex);
if (!IS_ERR(p)) {
stat = idr_find(&sp_stat_idr, current->mm->sp_stat_id);
if (stat)
stat->alloc_size += size_aligned;
}
-
-out:
mutex_unlock(&sp_mutex);
/* this will free spa if mmap failed */
@@ -1556,10 +1534,6 @@ static unsigned long sp_remap_kva_to_vma(unsigned long kva, struct sp_area *spa,
}
}
- if (!mmget_not_zero(mm)) {
- ret_addr = -ESPGMMEXIT;
- goto put_file;
- }
down_write(&mm->mmap_sem);
ret_addr = sp_mmap(mm, file, spa, &populate);
@@ -1604,8 +1578,7 @@ static unsigned long sp_remap_kva_to_vma(unsigned long kva, struct sp_area *spa,
put_mm:
up_write(&mm->mmap_sem);
- mmput(mm);
-put_file:
+
if (!spa->spg && file)
fput(file);
@@ -1769,10 +1742,12 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size,
*/
stat = sp_init_proc_stat(tsk, mm);
if (IS_ERR(stat)) {
+ mutex_unlock(&sp_mutex);
uva = stat;
pr_err("share pool: init proc stat failed, ret %lx\n", PTR_ERR(stat));
goto out_unlock;
}
+ mutex_unlock(&sp_mutex);
spg = __sp_find_spg(pid, SPG_ID_DEFAULT);
if (spg == NULL) {
@@ -1794,6 +1769,7 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size,
}
if (!vmalloc_area_set_flag(spa, kva_aligned, VM_SHAREPOOL)) {
+ up_read(&spg->rw_lock);
pr_err("share pool: %s: the kva %pK is not valid\n", __func__, (void *)kva_aligned);
goto out_drop_spa;
}
@@ -1808,12 +1784,14 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size,
goto out_unlock;
}
+ down_read(&spg->rw_lock);
if (enable_share_k2u_spg)
spa = sp_alloc_area(size_aligned, sp_flags, spg, SPA_TYPE_K2SPG);
else
spa = sp_alloc_area(size_aligned, sp_flags, NULL, SPA_TYPE_K2TASK);
if (IS_ERR(spa)) {
+ up_read(&spg->rw_lock);
if (printk_ratelimit())
pr_err("share pool: k2u(spg) failed due to alloc spa failure "
"(potential no enough virtual memory when -75): %ld\n",
@@ -1831,14 +1809,18 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size,
uva = sp_make_share_kva_to_spg(kva_aligned, spa, spg);
else
uva = sp_make_share_kva_to_task(kva_aligned, spa, mm);
+
+ up_read(&spg->rw_lock);
} else {
/* group is dead, return -ENODEV */
pr_err("share pool: failed to make k2u, sp group is dead\n");
}
if (!IS_ERR(uva)) {
+ mutex_lock(&sp_mutex);
uva = uva + (kva - kva_aligned);
stat->k2u_size += size_aligned;
+ mutex_unlock(&sp_mutex);
} else {
/* associate vma and spa */
if (!vmalloc_area_clr_flag(spa, kva_aligned, VM_SHAREPOOL))
@@ -1849,7 +1831,6 @@ void *sp_make_share_k2u(unsigned long kva, unsigned long size,
out_drop_spa:
__sp_area_drop(spa);
out_unlock:
- mutex_unlock(&sp_mutex);
mmput(mm);
out_put_task:
put_task_struct(tsk);
@@ -2144,7 +2125,6 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp
unsigned int page_size;
struct sp_proc_stat *stat;
- mutex_lock(&sp_mutex);
/*
* at first we guess it's a hugepage addr
* we can tolerate at most PMD_SIZE or PAGE_SIZE which is matched in k2u
@@ -2157,7 +2137,7 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp
if (printk_ratelimit())
pr_err("share pool: invalid input uva %pK in unshare uva\n",
(void *)uva);
- goto out_unlock;
+ goto out;
}
}
@@ -2259,10 +2239,14 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp
goto out_drop_area;
}
+ down_read(&spa->spg->rw_lock);
__sp_free(spa->spg, uva_aligned, size_aligned, NULL);
+ up_read(&spa->spg->rw_lock);
}
sp_dump_stack();
+
+ mutex_lock(&sp_mutex);
/* pointer stat may be invalid because of kthread buff_module_guard_work */
if (current->mm == NULL) {
kthread_stat.k2u_size -= spa->real_size;
@@ -2273,6 +2257,7 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp
else
WARN(1, "share_pool: %s: null process stat\n", __func__);
}
+ mutex_unlock(&sp_mutex);
out_clr_flag:
/* deassociate vma and spa */
@@ -2281,8 +2266,7 @@ static int sp_unshare_uva(unsigned long uva, unsigned long size, int pid, int sp
out_drop_area:
__sp_area_drop(spa);
-out_unlock:
- mutex_unlock(&sp_mutex);
+out:
return ret;
}
@@ -2446,7 +2430,7 @@ bool sp_config_dvpp_range(size_t start, size_t size, int device_id, int pid)
check_interrupt_context();
if (device_id < 0 || device_id >= MAX_DEVID || pid < 0 || size <= 0 ||
- size> MMAP_SHARE_POOL_16G_SIZE)
+ size > MMAP_SHARE_POOL_16G_SIZE)
return false;
mutex_lock(&sp_mutex);
@@ -2468,9 +2452,10 @@ EXPORT_SYMBOL_GPL(sp_config_dvpp_range);
/* Check whether the address belongs to the share pool. */
bool is_sharepool_addr(unsigned long addr)
{
- if (host_svm_sp_enable == false)
- return addr >= MMAP_SHARE_POOL_START && addr < (MMAP_SHARE_POOL_16G_START + MMAP_SHARE_POOL_16G_SIZE);
- return addr >= MMAP_SHARE_POOL_START && addr < MMAP_SHARE_POOL_END;
+ if (host_svm_sp_enable == false)
+ return addr >= MMAP_SHARE_POOL_START && addr < (MMAP_SHARE_POOL_16G_START + MMAP_SHARE_POOL_16G_SIZE);
+
+ return addr >= MMAP_SHARE_POOL_START && addr < MMAP_SHARE_POOL_END;
}
EXPORT_SYMBOL_GPL(is_sharepool_addr);
@@ -2515,7 +2500,8 @@ int proc_sp_group_state(struct seq_file *m, struct pid_namespace *ns,
return 0;
}
-static void rb_spa_stat_show(struct seq_file *seq) {
+static void rb_spa_stat_show(struct seq_file *seq)
+{
struct rb_node *node;
struct sp_area *spa;
@@ -2814,6 +2800,36 @@ vm_fault_t sharepool_no_page(struct mm_struct *mm,
}
EXPORT_SYMBOL(sharepool_no_page);
+#define MM_WOULD_FREE 2
+
+void sp_group_exit(struct mm_struct *mm)
+{
+ struct sp_group *spg = NULL;
+ bool is_alive = true, unlock;
+
+ if (!enable_ascend_share_pool)
+ return;
+
+ spg = mm->sp_group;
+
+ /* If the mm_users is 2, it means that the mm is ready to be freed
+ because the last owner of this mm is in exiting process.
+ */
+ if (spg_valid(spg) && atomic_read(&mm->mm_users) == MM_WOULD_FREE) {
+ spg_exit_lock(&unlock);
+ down_write(&spg->rw_lock);
+ if (list_is_singular(&spg->procs))
+ is_alive = spg->is_alive = false;
+ list_del(&mm->sp_node);
+ up_write(&spg->rw_lock);
+ if (!is_alive)
+ blocking_notifier_call_chain(&sp_notifier_chain, 0,
+ mm->sp_group);
+ atomic_dec(&mm->mm_users);
+ spg_exit_unlock(unlock);
+ }
+}
+
struct page *sp_alloc_pages(struct vm_struct *area, gfp_t mask,
unsigned int page_order, int node)
{
--
2.25.1
