hulk inclusion category: bugfix bugzilla: https://gitee.com/src-openeuler/kernel/issues/IDBFPR -------------------------------- Fix invalid paging request while many processes attach transhugepage to same addr. Return -EAGAIN, while dst addr already has mappings. Do not pin memory to avoid the src pgtable to be modified. User should ensure the memory pinned. Unable to handle kernel paging request at virtual address ffffff0562edc000 Mem abort info: ESR = 0x96000006 EC = 0x25: DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000006 CM = 0, WnR = 0 swapper pgtable: 4k pages, 48-bit VAs, pgdp=0000002c39a58000 [ffffff0562edc000] pgd=0000002fffff2003, p4d=0000002fffff2003, pud=0000002fffff1003, pmd=0000000000000000 Internal error: Oops: 0000000096000006 [#1] SMP CPU: 46 PID: 20703 Comm: ioctl_zcopy_too Kdump: loaded Not tainted 5.10.0-06a8b5208512 #1 Hardware name: Huawei Taishan 2280 V2/BC82AMDD, BIOS 6.57 05/17/2023 pstate: 20400009 (nzCv daif +PAN -UAO -TCO BTYPE=--) pc : attach_huge_pmd+0x1c0/0x1330 [zcopy] lr : attach_huge_pmd+0x12a8/0x1330 [zcopy] sp : ffff800052833b70 x29: ffff800052833b70 x28: 0000000303700078 x27: ffff4176c55666e8 x26: 0000ffff5ba00000 x25: ffffff06180159c0 x24: ffff4176cc695878 x23: ffff4186000e1058 x22: ffffff05daf55980 x21: ffff6185c1aa2640 x20: ffffb3cdde249880 x19: ffffb3cd7e7cb7c0 x18: ffffffffffffffbc x17: 0000000000000000 x16: ffffb3cddb8cd028 x15: 00000000ffffffc2 x14: 00000000000e89c3 x13: 000000000000003d x12: 0000000000000000 x11: 0000000000bc0000 x10: ffff4185bfdc8640 x9 : 0000000000000000 x8 : ffff4195bfd84640 x7 : fffffdffffe00000 x6 : ffffb3cddda71640 x5 : ffffff0617e03868 x4 : ffffb3cddd2cb000 x3 : ffffff0562edc000 x2 : ffff8dc7e2313000 x1 : 0000000000303700 x0 : 00000105630dc000 Call trace: attach_huge_pmd+0x1c0/0x1330 [zcopy] attach_page_range+0x404/0x68c [zcopy] attach_pages+0x2c0/0xa24 [zcopy] zcopy_ioctl+0x120/0x22c [zcopy] vfs_ioctl+0x3c/0xb0 __se_sys_ioctl+0x12c/0x170 __arm64_sys_ioctl+0x40/0x70 invoke_syscall+0xa0/0x260 el0_svc_common.constprop.0+0xdc/0x2d0 do_el0_svc+0xd4/0x290 el0_svc+0x44/0x60 el0_sync_handler+0x26c/0x280 Fixes: 975ef0f7a508 ("zcopy: Introduce the pageattach interface") Fixes: 2125c55fc876 ("zcopy: Extend PMD trans hugepage mapping ability") Signed-off-by: Liu Mingrui <liumingrui@huawei.com> --- drivers/misc/zcopy/zcopy.c | 96 ++++++++++++++------------------------ 1 file changed, 34 insertions(+), 62 deletions(-) diff --git a/drivers/misc/zcopy/zcopy.c b/drivers/misc/zcopy/zcopy.c index a20269a3d843..c17ef99a2ccc 100644 --- a/drivers/misc/zcopy/zcopy.c +++ b/drivers/misc/zcopy/zcopy.c @@ -177,11 +177,12 @@ static pud_t *zcopy_alloc_new_pud(struct mm_struct *mm, unsigned long addr) return zcopy_pud_alloc(mm, p4d, addr); } -static pmd_t *zcopy_alloc_pmd(struct mm_struct *mm, unsigned long addr) +static pmd_t *zcopy_alloc_pmd(struct mm_struct *mm, unsigned long addr, int *rc) { pud_t *pud; pmd_t *pmd; + *rc = -ENOMEM; pud = zcopy_alloc_new_pud(mm, addr); if (!pud) return NULL; @@ -190,6 +191,12 @@ static pmd_t *zcopy_alloc_pmd(struct mm_struct *mm, unsigned long addr) if (!pmd) return NULL; + if (pmd_trans_huge(*pmd)) { + pr_warn_once("va mapped to hugepage, please free it and realloc va\n"); + *rc = -EAGAIN; + return NULL; + } + return pmd; } @@ -264,10 +271,10 @@ int attach_huge_pmd(struct vm_area_struct *dst_vma, struct vm_area_struct *src_v { struct mm_struct *dst_mm, *src_mm; spinlock_t *src_ptl, *dst_ptl; - struct page *src_thp_page, *orig_thp_page; + struct page *src_thp_page; pmd_t pmd, orig_pmd; pgtable_t pgtable; - + int ret = 0; if (!vma_is_anonymous(dst_vma)) return -EINVAL; @@ -280,9 +287,18 @@ int attach_huge_pmd(struct vm_area_struct *dst_vma, struct vm_area_struct *src_v if (unlikely(!pgtable)) return -ENOMEM; - src_ptl = pmd_lockptr(src_mm, src_pmdp); dst_ptl = pmd_lockptr(dst_mm, dst_pmdp); + spin_lock_nested(dst_ptl, SINGLE_DEPTH_NESTING); + orig_pmd = *dst_pmdp; + /* check if exists old mappings */ + if (!pmd_none(orig_pmd)) { + pte_free(dst_mm, pgtable); + spin_unlock(dst_ptl); + return -EAGAIN; + } + spin_unlock(dst_ptl); + src_ptl = pmd_lockptr(src_mm, src_pmdp); spin_lock(src_ptl); pmd = *src_pmdp; src_thp_page = pmd_page(pmd); @@ -297,24 +313,13 @@ int attach_huge_pmd(struct vm_area_struct *dst_vma, struct vm_area_struct *src_v spin_unlock(src_ptl); spin_lock_nested(dst_ptl, SINGLE_DEPTH_NESTING); - orig_pmd = *dst_pmdp; - /* umap the old page mappings */ - if (!pmd_none(orig_pmd)) { - orig_thp_page = pmd_page(orig_pmd); - put_page(orig_thp_page); - atomic_dec(compound_mapcount_ptr(orig_thp_page)); - zcopy_add_mm_counter(dst_mm, MM_ANONPAGES, -HPAGE_PMD_NR); - mm_dec_nr_ptes(dst_mm); - } - zcopy_add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); mm_inc_nr_ptes(dst_mm); zcopy_pgtable_trans_huge_deposit(dst_mm, dst_pmdp, pgtable); zcopy_set_pmd_at(dst_mm, dst_addr, dst_pmdp, pmd); flush_tlb_range(dst_vma, dst_addr, dst_addr + HPAGE_PMD_SIZE); spin_unlock(dst_ptl); - - return 0; + return ret; } @@ -324,10 +329,11 @@ static int attach_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *sr { struct mm_struct *dst_mm = dst_vma->vm_mm; pte_t *src_ptep, *dst_ptep, pte, orig_pte; - struct page *src_page, *orig_page; + struct page *src_page; spinlock_t *dst_ptl; int rss[NR_MM_COUNTERS]; unsigned long src_addr_end = src_addr + len; + int ret = 0; memset(rss, 0, sizeof(int) * NR_MM_COUNTERS); @@ -346,30 +352,26 @@ static int attach_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *sr if (pte_none(*src_ptep) || pte_special(*src_ptep) || !pte_present(pte)) continue; + /* check if exists old mappings */ + orig_pte = *dst_ptep; + if (!pte_none(orig_pte)) { + ret = -EAGAIN; + goto out; + } + src_page = pte_page(pte); get_page(src_page); page_dup_rmap(src_page, false); rss[MM_ANONPAGES]++; - - /* - * If dst virtual addr has page mapping, before setup the new mapping. - * we should decrease the orig page mapcount and refcount. - */ - orig_pte = *dst_ptep; - if (!pte_none(orig_pte)) { - orig_page = pte_page(orig_pte); - put_page(orig_page); - zcopy_page_remove_rmap(orig_page, false); - rss[MM_ANONPAGES]--; - } zcopy_set_pte_at(dst_mm, dst_addr, dst_ptep, pte); } flush_tlb_range(dst_vma, dst_addr, dst_addr + len); +out: zcopy_add_mm_rss_vec(dst_mm, rss); spin_unlock(dst_ptl); - return 0; + return ret; } static int attach_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, @@ -394,11 +396,9 @@ static int attach_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, src_pmd = zcopy_get_pmd(src_mm, src_addr); if (!src_pmd) continue; - dst_pmd = zcopy_alloc_pmd(dst_mm, dst_addr); - if (!dst_pmd) { - ret = -ENOMEM; + dst_pmd = zcopy_alloc_pmd(dst_mm, dst_addr, &ret); + if (!dst_pmd) break; - } if (pmd_trans_huge(*src_pmd)) { if (extent == HPAGE_PMD_SIZE) { @@ -443,11 +443,6 @@ static int attach_pages(unsigned long dst_addr, unsigned long src_addr, { struct mm_struct *dst_mm, *src_mm; struct task_struct *src_task, *dst_task; - struct page **process_pages; - unsigned long nr_pages; - unsigned int flags = 0; - int pinned_pages; - int locked = 1; int ret; ret = -EINVAL; @@ -494,33 +489,10 @@ static int attach_pages(unsigned long dst_addr, unsigned long src_addr, goto put_dst_mm; } - nr_pages = (src_addr + size - 1) / PAGE_SIZE - src_addr / PAGE_SIZE + 1; - process_pages = kvmalloc_array(nr_pages, sizeof(struct pages *), GFP_KERNEL); - if (!process_pages) { - ret = -ENOMEM; - goto put_dst_mm; - } - - mmap_read_lock(src_mm); - pinned_pages = pin_user_pages_remote(src_mm, src_addr, nr_pages, - flags, process_pages, - NULL, &locked); - if (locked) - mmap_read_unlock(src_mm); - - if (pinned_pages <= 0) { - ret = -EFAULT; - goto free_pages_array; - } - trace_attach_page_range_start(dst_mm, src_mm, dst_addr, src_addr, size); ret = attach_page_range(dst_mm, src_mm, dst_addr, src_addr, size); trace_attach_page_range_end(dst_mm, src_mm, dst_addr, src_addr, ret); - unpin_user_pages_dirty_lock(process_pages, pinned_pages, 0); - -free_pages_array: - kvfree(process_pages); put_dst_mm: mmput(dst_mm); put_dst_task: -- 2.25.1