hulk inclusion category: feature bugzilla: NA ---------------------------------------- Since the ascend hardware only support fixed offset of nc mapping, the devm alloc 8T address space with fixed start 0x100000000000, and ascend svsp reserved 8T after devm end. We could define the offset is 16T, which is same of sharepool size. Virtual address space has been adjusted as follows: Virtual address space before: 0x100000000000: devm start 0x180000000000: devm end and svsp start 0x200000000000: svsp end 0xe00000000000: nc map start 0xe80000000000: share pool normal start and nc map end Virtual address space After: 0x100000000000: devm start 0x180000000000: devm end and svsp start 0x200000000000: svsp end and nc map start 0x280000000000: nc map end 0xe00000000000: share pool normal start 0xe7f000000000: share pool normal end and share pool ro start 0xe80000000000: share pool ro end and share pool dvpp start 0xf00000000000: share pool dvpp end and share pool nc map start 0xf80000000000: share pool nc map end Signed-off-by: Yongqiang Liu <liuyongqiang13@huawei.com> Signed-off-by: Yin Tirui <yintirui@huawei.com> --- arch/arm64/include/asm/ascend_vm.h | 24 +++++++ arch/arm64/mm/ascend_vm.c | 34 +++++++++ fs/Kconfig | 2 +- include/linux/ascend_vm.h | 28 ++++++++ include/linux/share_pool.h | 13 +--- mm/mmap.c | 70 +++++++++++++++++++ mm/share_pool.c | 107 +++++++++++++++++++++++++++++ 7 files changed, 267 insertions(+), 11 deletions(-) diff --git a/arch/arm64/include/asm/ascend_vm.h b/arch/arm64/include/asm/ascend_vm.h index ab2a23ba8b91..1019e0fb75fb 100644 --- a/arch/arm64/include/asm/ascend_vm.h +++ b/arch/arm64/include/asm/ascend_vm.h @@ -6,12 +6,25 @@ #ifndef __ASSEMBLY__ +#ifdef CONFIG_SHARE_POOL +DECLARE_STATIC_KEY_FALSE(share_pool_enabled_key); +static inline bool sp_is_enabled(void) +{ + return static_branch_likely(&share_pool_enabled_key); +} +#endif #ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW DECLARE_STATIC_KEY_FALSE(enable_pt_multi_view); #define N_PT_MULTI_VIEW_MMAP_END 0x280000000000UL #define N_PT_MULTI_VIEW_MMAP_SIZE 0x080000000000UL /* 8T */ #define N_PT_MULTI_VIEW_MMAP_START (N_PT_MULTI_VIEW_MMAP_END - N_PT_MULTI_VIEW_MMAP_SIZE) +#ifdef CONFIG_SHARE_POOL +#define MMAP_SHARE_POOL_NC_START 0xf00000000000UL /* share pool end */ +#define MMAP_SHARE_POOL_NC_END 0xf80000000000UL +#define MMAP_SHARE_POOL_NC_OFFSET 0x100000000000UL /* 16T */ +#endif + #define ASCEND_PTMV_PGD_ORDER 1 static inline int ascend_pt_multi_view_enabled(void) @@ -28,6 +41,13 @@ static inline unsigned long ascend_p4d_offset(unsigned long addr) return (addr >> P4D_SHIFT) * sizeof(p4d_t); } +#ifdef CONFIG_SHARE_POOL +static inline bool ascend_spnc_offset_valid(unsigned long offset) +{ + return offset >= ascend_p4d_offset(MMAP_SHARE_POOL_NC_START) && + offset < ascend_p4d_offset(MMAP_SHARE_POOL_NC_END); +} +#endif static inline bool ascend_nc_offset_valid(unsigned long offset) { @@ -46,6 +66,10 @@ static inline bool ascend_check_p4d_nocopy(p4d_t *p4dp) return true; offset = ((unsigned long)p4dp) & ~PAGE_MASK; +#ifdef CONFIG_SHARE_POOL + if (sp_is_enabled() && ascend_spnc_offset_valid(offset)) + return true; +#endif return ascend_nc_offset_valid(offset); } diff --git a/arch/arm64/mm/ascend_vm.c b/arch/arm64/mm/ascend_vm.c index a33c7046f57f..5c5e119193a7 100644 --- a/arch/arm64/mm/ascend_vm.c +++ b/arch/arm64/mm/ascend_vm.c @@ -31,6 +31,20 @@ static inline bool ascend_check_range_valid(unsigned long addr, unsigned long le { struct ascend_vm_area range; +#ifdef CONFIG_SHARE_POOL + if (!sp_is_enabled()) + goto skip; + + range.va_start = MMAP_SHARE_POOL_NC_START; + range.va_end = MMAP_SHARE_POOL_NC_END; + + if ((flags & MAP_NPTMV) && (flags & MAP_SHARE_POOL)) + return check_range_valid(addr, addr + len, &range); + + if (check_range_overlap(addr, addr + len, &range)) + return false; +skip: +#endif range.va_start = N_PT_MULTI_VIEW_MMAP_START; range.va_end = N_PT_MULTI_VIEW_MMAP_END; @@ -66,10 +80,30 @@ void n_pt_multi_view_mmap_get_area(struct vm_unmapped_area_info *info, info->high_limit = N_PT_MULTI_VIEW_MMAP_END; info->flags |= VM_UNMAPPED_AREA_NC; } +#ifdef CONFIG_SHARE_POOL + if (!sp_is_enabled()) + return; + + if ((flags & MAP_NPTMV) && (flags & MAP_SHARE_POOL)) { + info->low_limit = MMAP_SHARE_POOL_NC_START; + info->high_limit = MMAP_SHARE_POOL_NC_END; + } +#endif +} + +#ifdef CONFIG_SHARE_POOL +static bool ascend_spnc_map_valid(unsigned long addr) +{ + return addr >= MMAP_SHARE_POOL_NC_START && addr < MMAP_SHARE_POOL_NC_END; } +#endif bool ascend_nc_map_valid(unsigned long addr) { +#ifdef CONFIG_SHARE_POOL + if (sp_is_enabled() && ascend_spnc_map_valid(addr)) + return true; +#endif return addr >= N_PT_MULTI_VIEW_MMAP_START && addr < N_PT_MULTI_VIEW_MMAP_END; } diff --git a/fs/Kconfig b/fs/Kconfig index f80b4f76093c..fb395d8b060e 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -300,7 +300,7 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP config HUGETLB_PMD_PAGE_TABLE_SHARING def_bool HUGETLB_PAGE - depends on ARCH_WANT_HUGE_PMD_SHARE && SPLIT_PMD_PTLOCKS + depends on ARCH_WANT_HUGE_PMD_SHARE && SPLIT_PMD_PTLOCKS && !ASCEND_PAGE_TABLE_MULTI_VIEW config HUGETLB_ALLOC_LIMIT bool "Limit hugeTLB pages allocation" diff --git a/include/linux/ascend_vm.h b/include/linux/ascend_vm.h index fd596e77b0f3..f959800492e0 100644 --- a/include/linux/ascend_vm.h +++ b/include/linux/ascend_vm.h @@ -95,6 +95,34 @@ static inline int ascend_mmap_check_addr(unsigned long addr, unsigned long len, #endif #ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + +#define ASCEND_VMA_INFO_LOW_END N_PT_MULTI_VIEW_MMAP_START +#define ASCEND_VMA_INFO_HIGH_BASE N_PT_MULTI_VIEW_MMAP_END +#ifdef CONFIG_SHARE_POOL +#define ASCEND_VMA_INFO_HIGH_END MMAP_SHARE_POOL_NC_START +#endif + +static inline void ascend_set_vma_info_low(struct vm_unmapped_area_info *info) +{ + info->high_limit = min(info->high_limit, ASCEND_VMA_INFO_LOW_END); +} + +static inline void ascend_set_vma_info_high(struct vm_unmapped_area_info *info) +{ + info->low_limit = max(info->low_limit, ASCEND_VMA_INFO_HIGH_BASE); +#ifdef CONFIG_SHARE_POOL + if (!sp_is_enabled()) + return; + + info->high_limit = min(info->high_limit, ASCEND_VMA_INFO_HIGH_END); +#endif +} + +static inline bool ascend_vm_info_valid(struct vm_unmapped_area_info *info) +{ + return info->flags & ASCEND_VM_INFO_MASK; +} + void n_pt_multi_view_mmap_get_area(struct vm_unmapped_area_info *info, unsigned long flags); #else diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 2db4689616e4..a3caedb99c5c 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -50,8 +50,6 @@ #define MAX_DEVID 8 /* the max num of Da-vinci devices */ -extern struct static_key_false share_pool_enabled_key; - struct sp_walk_data { struct page **pages; unsigned int page_count; @@ -74,8 +72,8 @@ struct sp_walk_data { #define MMAP_SHARE_POOL_DVPP_SIZE 0x80000000000UL /* 16G size */ #define MMAP_SHARE_POOL_16G_SIZE 0x400000000UL -/* skip 8T for stack */ -#define MMAP_SHARE_POOL_SKIP 0x80000000000UL +/* skip 16T for stack */ +#define MMAP_SHARE_POOL_SKIP 0x100000000000UL #define MMAP_SHARE_POOL_END (TASK_SIZE - MMAP_SHARE_POOL_SKIP) #define MMAP_SHARE_POLL_DVPP_END (MMAP_SHARE_POOL_END) /* MMAP_SHARE_POOL_DVPP_START should be align to 16G */ @@ -119,11 +117,6 @@ extern bool mg_is_sharepool_addr(unsigned long addr); extern int mg_sp_id_of_current(void); -static inline bool sp_is_enabled(void) -{ - return static_branch_likely(&share_pool_enabled_key); -} - extern void __sp_mm_clean(struct mm_struct *mm); static inline void sp_mm_clean(struct mm_struct *mm) { @@ -137,7 +130,7 @@ static inline void sp_area_work_around(struct vm_unmapped_area_info *info, /* the MAP_SVSP couldn't work with MAP_SHARE_POOL. In addition, the * address ranges corresponding to the two flags must not overlap. */ - if (sp_is_enabled() && !(flags & MAP_SVSP)) + if (sp_is_enabled() && !(flags & (MAP_SVSP | MAP_NPTMV))) info->high_limit = min(info->high_limit, MMAP_SHARE_POOL_START); } diff --git a/mm/mmap.c b/mm/mmap.c index fbdefa4af6c3..a0c230400223 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1734,6 +1734,71 @@ static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) return gap; } +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + +unsigned long vm_unmapped_area_no_reserve(struct vm_unmapped_area_info *info) +{ + unsigned long addr; + unsigned long low_limit, high_limit; + + low_limit = info->low_limit; + high_limit = info->high_limit; + + if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) { + ascend_set_vma_info_high(info); + addr = unmapped_area_topdown(info); + if (IS_ERR_VALUE(addr)) { + info->low_limit = low_limit; +#ifdef CONFIG_SHARE_POOL + if (sp_is_enabled()) + info->high_limit = high_limit; +#endif + ascend_set_vma_info_low(info); + addr = unmapped_area_topdown(info); + return addr; + } + + return addr; + } + + ascend_set_vma_info_low(info); + addr = unmapped_area(info); + if (IS_ERR_VALUE(addr)) { + info->high_limit = high_limit; + ascend_set_vma_info_high(info); + addr = unmapped_area(info); + return addr; + } + + return addr; +} + +static unsigned long vm_unmapped_area_common(struct vm_unmapped_area_info *info) +{ + unsigned long addr; + + if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) + addr = unmapped_area_topdown(info); + else + addr = unmapped_area(info); + + return addr; +} + +static unsigned long ascend_vm_unmapped_area(struct vm_unmapped_area_info *info) +{ + unsigned long addr; + + if (ascend_vm_info_valid(info)) + addr = vm_unmapped_area_common(info); + else + addr = vm_unmapped_area_no_reserve(info); + + trace_vm_unmapped_area(addr, info); + return addr; +} +#endif + /* * Search for an unmapped address range. * @@ -1747,6 +1812,11 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) { unsigned long addr; +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + if (ascend_pt_multi_view_enabled()) + return ascend_vm_unmapped_area(info); +#endif + if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) addr = unmapped_area_topdown(info); else diff --git a/mm/share_pool.c b/mm/share_pool.c index 548b85e61989..e2369747c983 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -704,6 +704,9 @@ struct sp_area { pid_t applier; /* the original applier process */ int preferred_node_id; /* memory node */ struct work_struct work; +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + unsigned long nc_va_start; /* nocache vma start */ +#endif }; static unsigned long spa_size(struct sp_area *spa) @@ -1637,6 +1640,9 @@ static struct sp_area *sp_area_alloc(unsigned long size, unsigned long flags, spa->applier = applier; spa->preferred_node_id = node_id; atomic_set(&spa->use_count, 1); +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + spa->nc_va_start = spa->va_start + MMAP_SHARE_POOL_NC_OFFSET; +#endif /* the link location could be saved before, to be optimized */ spm_insert_area(mapping, spa); @@ -1802,6 +1808,14 @@ static void __sp_free(struct sp_area *spa, struct mm_struct *stop) if (mm == stop) break; sp_munmap(mm, spa->va_start, spa_size(spa)); +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + if (ascend_pt_multi_view_enabled()) { + if (spa->type != SPA_TYPE_ALLOC || spa->flags & SP_DVPP) + continue; + + sp_munmap(mm, spa->nc_va_start, spa_size(spa)); + } +#endif } } @@ -2075,6 +2089,62 @@ static int sp_alloc_populate(struct mm_struct *mm, struct sp_area *spa, return ret; } +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW +#ifndef PTE_PBHA0 +#define PTE_PBHA0 (_AT(pteval_t, 1) << 59) /* PBHA 59 bit */ +#endif +#define sp_pgprot_writethrough(prot) \ + __pgprot_modify(prot, PTE_ATTRINDX_MASK, \ + PTE_ATTRINDX(MT_NORMAL_WT) | PTE_PXN | PTE_UXN | PTE_PBHA0) + +static int sp_nc_mmap(struct mm_struct *mm, struct sp_area *spa, unsigned long prot) +{ + int ret; + unsigned long populate = 0; + unsigned long size = spa_size(spa); + unsigned long addr = spa->nc_va_start; + unsigned long flags = MAP_FIXED_NOREPLACE | MAP_SHARED | MAP_POPULATE | + MAP_SHARE_POOL | MAP_NPTMV; + unsigned long vm_flags = VM_NORESERVE | VM_SHARE_POOL | VM_DONTCOPY; + unsigned long pgoff = addr_offset(spa) >> PAGE_SHIFT; + struct vm_area_struct *vma; + + down_write(&mm->mmap_lock); + if (spa->flags & SP_PROT_RO) + prot &= ~PROT_WRITE; + + atomic_inc(&spa->use_count); + addr = __do_mmap_mm(mm, spa_file(spa), addr, size, prot, flags, vm_flags, pgoff, + &populate, NULL); + if (IS_ERR_VALUE(addr)) { + atomic_dec(&spa->use_count); + up_write(&mm->mmap_lock); + return -ENOMEM; + } + + BUG_ON(addr != spa->nc_va_start); + vma = find_vma(mm, addr); + vma->spa = spa; + + if (prot & PROT_WRITE) + vma->vm_page_prot = __pgprot(((~PTE_RDONLY) & + vma->vm_page_prot.pgprot) | PTE_DIRTY); + else + vm_flags_clear(vma, VM_MAYWRITE); + + vma->vm_page_prot = sp_pgprot_writethrough(vma->vm_page_prot); + up_write(&mm->mmap_lock); + + ret = do_mm_populate(mm, addr, populate, 0); + if (ret) { + down_write(&mm->mmap_lock); + do_munmap(mm, addr, spa_size(spa), NULL); + up_write(&mm->mmap_lock); + } + + return ret; +} +#endif static int sp_k2u_populate(struct mm_struct *mm, struct sp_area *spa); #define SP_SKIP_ERR 1 @@ -2122,6 +2192,29 @@ static int sp_map_spa_to_mm(struct mm_struct *mm, struct sp_area *spa, } mmap_write_unlock(mm); } + +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + if (ret) + goto out; + + if (!ascend_pt_multi_view_enabled()) + goto out; + + /* bypass dvpp allocation */ + if (spa->flags & SP_DVPP) + goto out; + + if (spa->type != SPA_TYPE_ALLOC) + goto out; + + ret = sp_nc_mmap(mm, spa, prot); + if (ret) { + down_write(&mm->mmap_lock); + do_munmap(mm, mmap_addr, spa_size(spa), NULL); + up_write(&mm->mmap_lock); + } +out: +#endif mmput_async(mm); return ret; @@ -3001,8 +3094,22 @@ bool mg_sp_config_dvpp_range(size_t start, size_t size, int device_id, int tgid) } EXPORT_SYMBOL_GPL(mg_sp_config_dvpp_range); +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW +static bool is_spnc_reserve_addr(unsigned long addr) +{ + if (!ascend_pt_multi_view_enabled()) + return false; + + return addr >= MMAP_SHARE_POOL_NC_START && addr < MMAP_SHARE_POOL_NC_END; +} +#endif + static bool is_sp_reserve_addr(unsigned long addr) { +#ifdef CONFIG_ASCEND_PAGE_TABLE_MULTI_VIEW + if (is_spnc_reserve_addr(addr)) + return true; +#endif return addr >= MMAP_SHARE_POOL_START && addr < MMAP_SHARE_POOL_END; } -- 2.43.0