From: Fang Lijun fanglijun3@huawei.com
ascend inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4EUVI CVE: NA
------------------------------------------------- The reason of exporting __vmalloc_node() is that gfp_mask __GFP_ACCOUNT is used in mbuff to limit memory usage of vmalloc() with memory cgroup.
We add a new parameter vmflags for __vmalloc_node() because VM_USERMAP and VM_HUGE_PAGES is for vmalloc_hugepage_user().
By selecting HAVE_ARCH_HUGE_VMALLOC, vmalloc_hugepage_user() can allocate hugepage memory. Also, vmalloc() will allocate hugepage memory if possible. Reference: https://lwn.net/Articles/839107/
Signed-off-by: Tang Yizhou tangyizhou@huawei.com Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Reviewed-by: Ding Tianhong dingtianhong@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- arch/Kconfig | 4 -- arch/arm64/Kconfig | 1 - include/linux/share_pool.h | 25 ---------- include/linux/vmalloc.h | 7 ++- mm/share_pool.c | 93 -------------------------------------- mm/vmalloc.c | 74 +++++++++++++++++++++++------- 6 files changed, 64 insertions(+), 140 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig index e877b083238de..00f55932ba781 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -562,10 +562,6 @@ config HAVE_ARCH_HUGE_VMAP config HAVE_ARCH_HUGE_VMALLOC depends on HAVE_ARCH_HUGE_VMAP bool - help - Archs that select this would be capable of PMD-sized vmaps (i.e., - arch_vmap_pmd_supported() returns true), and they must make no - assumptions that vmalloc memory is mapped with PAGE_SIZE ptes.
config HAVE_ARCH_SOFT_DIRTY bool diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index f2106f2cb6edf..9d49b9524e1d4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1467,7 +1467,6 @@ config ASCEND_SHARE_POOL default n select ARCH_USES_HIGH_VMA_FLAGS select MM_OWNER - depends on HAVE_ARCH_HUGE_VMALLOC help This feature allows multiple processes to share virtual memory both in kernel and user level, which is only enabled for ascend platform. diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 6bd03c3504c42..442ee1ed5fa65 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -326,11 +326,6 @@ vm_fault_t sharepool_no_page(struct mm_struct *mm, struct address_space *mapping, pgoff_t idx, unsigned long address, pte_t *ptep, unsigned int flags);
-extern void *vmalloc_hugepage(unsigned long size); -extern void *vmalloc_hugepage_user(unsigned long size); -extern void *vzalloc_user_account(unsigned long size, int node); -extern void *vzalloc_hugepage_user_account(unsigned long size, int node); - void sp_exit_mm(struct mm_struct *mm);
static inline bool is_vmalloc_huge(unsigned long vm_flags) @@ -593,26 +588,6 @@ static inline bool ascend_sp_oom_show(void) return false; }
-static inline void *vmalloc_hugepage(unsigned long size) -{ - return NULL; -} - -static inline void *vmalloc_hugepage_user(unsigned long size) -{ - return NULL; -} - -static inline void *vzalloc_user_account(unsigned long size, int node) -{ - return NULL; -} - -static inline void *vzalloc_hugepage_user_account(unsigned long size, int node) -{ - return NULL; -} - static inline bool is_vmalloc_huge(unsigned long vm_flags) { return NULL; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 7322909aed157..f7cc858732e29 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -94,11 +94,16 @@ extern void *vmalloc_exec(unsigned long size); extern void *vmalloc_32(unsigned long size); extern void *vmalloc_32_user(unsigned long size); extern void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot); +extern void *__vmalloc_node(unsigned long size, unsigned long align, + gfp_t gfp_mask, pgprot_t prot, + unsigned long vm_flags, + int node, const void *caller); extern void *__vmalloc_node_range(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, pgprot_t prot, unsigned long vm_flags, int node, const void *caller); - +extern void *vmalloc_hugepage(unsigned long size); +extern void *vmalloc_hugepage_user(unsigned long size); #ifndef CONFIG_MMU extern void *__vmalloc_node_flags(unsigned long size, int node, gfp_t flags); static inline void *__vmalloc_node_flags_caller(unsigned long size, int node, diff --git a/mm/share_pool.c b/mm/share_pool.c index 001c37d76a4b2..8b18648414344 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -4467,104 +4467,11 @@ struct page *sp_alloc_pages(struct vm_struct *area, gfp_t mask, return alloc_pages_node(node, mask, page_order); }
-/** - * vmalloc_hugepage - allocate virtually contiguous hugetlb memory - * @size: allocation size - * - * Allocate enough huge pages to cover @size and map them into - * contiguous kernel virtual space. - * - * The allocation size is aligned to PMD_SIZE automatically - */ -void *vmalloc_hugepage(unsigned long size) -{ - /* PMD hugepage aligned */ - size = PMD_ALIGN(size); - - return __vmalloc_node_range(size, PMD_SIZE, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, PAGE_KERNEL, - VM_HUGE_PAGES, NUMA_NO_NODE, - __builtin_return_address(0)); -} -EXPORT_SYMBOL(vmalloc_hugepage); - -/** - * vmalloc_hugepage_user - allocate virtually contiguous hugetlb memory - * for userspace - * @size: allocation size - * - * Allocate enough huge pages to cover @size and map them into - * contiguous kernel virtual space. The resulting memory area - * is zeroed so it can be mapped to userspace without leaking data. - * - * The allocation size is aligned to PMD_SIZE automatically - */ -void *vmalloc_hugepage_user(unsigned long size) -{ - /* PMD hugepage aligned */ - size = PMD_ALIGN(size); - - return __vmalloc_node_range(size, PMD_SIZE, VMALLOC_START, VMALLOC_END, - GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, - VM_HUGE_PAGES | VM_USERMAP, NUMA_NO_NODE, - __builtin_return_address(0)); -} -EXPORT_SYMBOL(vmalloc_hugepage_user); - -/** - * vzalloc_user_account - allocate zeroed virtually contiguous memory - * for userspace - * @size: allocation size - * @node: NUMA node id - * - * The resulting memory area is zeroed so it can be mapped to userspace - * without leaking data. - * - * Compare to vmalloc_user(), this is a customized function because - * __GFP_ACCOUNT is used to limit memory usage. - */ -void *vzalloc_user_account(unsigned long size, int node) -{ - return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, - GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT, PAGE_KERNEL, - VM_USERMAP, node, - __builtin_return_address(0)); -} -EXPORT_SYMBOL(vzalloc_user_account); - -/** - * vzalloc_hugepage_user_account - allocate virtually contiguous hugetlb memory - * for userspace - * @size: allocation size - * @node: NUMA node id - * - * Allocate enough huge pages to cover @size and map them into - * contiguous kernel virtual space. The resulting memory area - * is zeroed so it can be mapped to userspace without leaking data. - * - * The allocation size is aligned to PMD_SIZE automatically - * - * Compare to vmalloc_hugepage_user(), this is a customized function because - * __GFP_ACCOUNT is used to limit memory usage. - */ -void *vzalloc_hugepage_user_account(unsigned long size, int node) -{ - /* PMD hugepage aligned */ - size = PMD_ALIGN(size); - - return __vmalloc_node_range(size, PMD_SIZE, VMALLOC_START, VMALLOC_END, - GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT, PAGE_KERNEL, - VM_HUGE_PAGES | VM_USERMAP, node, - __builtin_return_address(0)); -} -EXPORT_SYMBOL(vzalloc_hugepage_user_account); - int enable_ascend_share_pool;
static int __init enable_share_pool(char *s) { enable_ascend_share_pool = 1; - vmap_allow_huge = true;
pr_info("Ascend enable share pool features\n");
diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 36f0e6d94a957..c3df82a8ee422 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -43,7 +43,7 @@ #include "internal.h"
#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC -bool __ro_after_init vmap_allow_huge; +bool __ro_after_init vmap_allow_huge = true;
static int __init set_nohugevmalloc(char *str) { @@ -2574,9 +2574,6 @@ void *vmap_hugepage(struct page **pages, unsigned int count, } EXPORT_SYMBOL(vmap_hugepage);
-static void *__vmalloc_node(unsigned long size, unsigned long align, - gfp_t gfp_mask, pgprot_t prot, - int node, const void *caller); static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot, unsigned int page_shift, int node) { @@ -2599,7 +2596,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, /* Please note that the recursion is strictly bounded. */ if (array_size > PAGE_SIZE) { pages = __vmalloc_node(array_size, 1, nested_gfp|highmem_mask, - PAGE_KERNEL, node, area->caller); + PAGE_KERNEL, 0, node, area->caller); } else { pages = kmalloc_node(array_size, nested_gfp, node); } @@ -2732,6 +2729,7 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, * @size: allocation size * @align: desired alignment * @gfp_mask: flags for the page level allocator + * @vm_flags: flags for vm_struct * @prot: protection mask for the allocated pages * @node: node to use for allocation or NUMA_NO_NODE * @caller: caller's return address @@ -2747,17 +2745,18 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, * with mm people. * */ -static void *__vmalloc_node(unsigned long size, unsigned long align, - gfp_t gfp_mask, pgprot_t prot, +void *__vmalloc_node(unsigned long size, unsigned long align, + gfp_t gfp_mask, pgprot_t prot, unsigned long vm_flags, int node, const void *caller) { return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, - gfp_mask, prot, 0, node, caller); + gfp_mask, prot, vm_flags, node, caller); } +EXPORT_SYMBOL(__vmalloc_node);
void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) { - return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE, + return __vmalloc_node(size, 1, gfp_mask, prot, 0, NUMA_NO_NODE, __builtin_return_address(0)); } EXPORT_SYMBOL(__vmalloc); @@ -2765,7 +2764,7 @@ EXPORT_SYMBOL(__vmalloc); static inline void *__vmalloc_node_flags(unsigned long size, int node, gfp_t flags) { - return __vmalloc_node(size, 1, flags, PAGE_KERNEL, + return __vmalloc_node(size, 1, flags, PAGE_KERNEL, 0, node, __builtin_return_address(0)); }
@@ -2773,7 +2772,7 @@ static inline void *__vmalloc_node_flags(unsigned long size, void *__vmalloc_node_flags_caller(unsigned long size, int node, gfp_t flags, void *caller) { - return __vmalloc_node(size, 1, flags, PAGE_KERNEL, node, caller); + return __vmalloc_node(size, 1, flags, PAGE_KERNEL, 0, node, caller); }
/** @@ -2823,7 +2822,7 @@ void *vmalloc_user(unsigned long size)
ret = __vmalloc_node(size, SHMLBA, GFP_KERNEL | __GFP_ZERO, - PAGE_KERNEL, NUMA_NO_NODE, + PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); if (ret) { area = find_vm_area(ret); @@ -2846,7 +2845,7 @@ EXPORT_SYMBOL(vmalloc_user); */ void *vmalloc_node(unsigned long size, int node) { - return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL, + return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL, 0, node, __builtin_return_address(0)); } EXPORT_SYMBOL(vmalloc_node); @@ -2884,7 +2883,7 @@ EXPORT_SYMBOL(vzalloc_node);
void *vmalloc_exec(unsigned long size) { - return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL_EXEC, + return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, __builtin_return_address(0)); }
@@ -2909,7 +2908,7 @@ void *vmalloc_exec(unsigned long size) */ void *vmalloc_32(unsigned long size) { - return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL, + return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); } EXPORT_SYMBOL(vmalloc_32); @@ -2927,7 +2926,7 @@ void *vmalloc_32_user(unsigned long size) void *ret;
ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, - NUMA_NO_NODE, __builtin_return_address(0)); + 0, NUMA_NO_NODE, __builtin_return_address(0)); if (ret) { area = find_vm_area(ret); area->flags |= VM_USERMAP; @@ -2936,6 +2935,49 @@ void *vmalloc_32_user(unsigned long size) } EXPORT_SYMBOL(vmalloc_32_user);
+#ifdef CONFIG_ASCEND_SHARE_POOL +/** + * vmalloc_hugepage - allocate virtually contiguous hugetlb memory + * @size: allocation size + * + * Allocate enough huge pages to cover @size and map them into + * contiguous kernel virtual space. + * + * The allocation size is aligned to PMD_SIZE automatically + */ +void *vmalloc_hugepage(unsigned long size) +{ + /* PMD hugepage aligned */ + size = PMD_ALIGN(size); + + return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL, + VM_HUGE_PAGES, NUMA_NO_NODE, + __builtin_return_address(0)); +} +EXPORT_SYMBOL(vmalloc_hugepage); + +/** + * vmalloc_hugepage_user - allocate virtually contiguous hugetlb memory + * for userspace + * @size: allocation size + * + * Allocate enough huge pages to cover @size and map them into + * contiguous kernel virtual space. The resulting memory area + * is zeroed so it can be mapped to userspace without leaking data. + * + * The allocation size is aligned to PMD_SIZE automatically + */ +void *vmalloc_hugepage_user(unsigned long size) +{ + /* PMD hugepage aligned */ + size = PMD_ALIGN(size); + + return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, + VM_HUGE_PAGES | VM_USERMAP, NUMA_NO_NODE, + __builtin_return_address(0)); +} +EXPORT_SYMBOL(vmalloc_hugepage_user); +#endif
/* * small helper routine , copy contents to buf from addr.