- Kernel - mailweb.openeuler.org

[PATCH OLK-5.10 1/1] ARM: mm: non-LPAE systems HugeTLB support for hulk
by Chen Jun 21 Dec '20

21 Dec '20

From: Linxu Fang <fanglinxu(a)huawei.com> maillist inclusion category: feature bugzilla: 46792 CVE: NA Reference: https://lists.linaro.org/pipermail/linaro-kernel/2013-October/008031.html ---------------------------------------- It's a improved version of Steve Capper's RFC patch, see: https://lists.linaro.org/pipermail/linaro-kernel/2013-October/008031.html Signed-off-by: Linxu Fang <fanglinxu(a)huawei.com> Reviewed-by: Hanjun Guo <guohanjun(a)huawei.com> Signed-off-by: zhangyi (F) <yi.zhang(a)huawei.com> Signed-off-by: Jing Xiangfeng <jingxiangfeng(a)huawei.com> --- arch/arm/Kconfig | 2 +- arch/arm/include/asm/hugetlb-2level.h | 148 +++++++++++++++++++ arch/arm/include/asm/hugetlb.h | 6 +- arch/arm/include/asm/pgtable-2level-hwdef.h | 2 + arch/arm/include/asm/pgtable-2level.h | 154 +++++++++++++++++++- arch/arm/include/asm/pgtable-3level.h | 2 +- arch/arm/include/asm/pgtable.h | 14 ++ arch/arm/kernel/head.S | 4 + arch/arm/mm/fault.c | 26 ++-- arch/arm/mm/fsr-2level.c | 4 +- arch/arm/mm/hugetlbpage.c | 2 +- arch/arm/mm/mmu.c | 25 ++++ 12 files changed, 369 insertions(+), 20 deletions(-) create mode 100644 arch/arm/include/asm/hugetlb-2level.h diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 002e0cf025f5..3efce886d7a0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1545,7 +1545,7 @@ config HW_PERF_EVENTS config SYS_SUPPORTS_HUGETLBFS def_bool y - depends on ARM_LPAE + depends on ARM_LPAE || (!CPU_USE_DOMAINS && !MEMORY_FAILURE && !CPU_SW_DOMAIN_PAN) config HAVE_ARCH_TRANSPARENT_HUGEPAGE def_bool y diff --git a/arch/arm/include/asm/hugetlb-2level.h b/arch/arm/include/asm/hugetlb-2level.h new file mode 100644 index 000000000000..ce746dae2497 --- /dev/null +++ b/arch/arm/include/asm/hugetlb-2level.h @@ -0,0 +1,148 @@ +/* + * arch/arm/include/asm/hugetlb-2level.h + * + * Copyright (C) 2014 Linaro Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _ASM_ARM_HUGETLB_2LEVEL_H +#define _ASM_ARM_HUGETLB_2LEVEL_H + +#define __HAVE_ARCH_HUGE_PTEP_GET +static inline pte_t huge_ptep_get(pte_t *ptep) +{ + pmd_t pmd = *((pmd_t *)ptep); + pte_t retval; + + if (!pmd_val(pmd)) + return __pte(0); + + retval = __pte((pteval_t) (pmd_val(pmd) & HPAGE_MASK) + | arm_hugepteprotval); + + if (pmd_exec(pmd)) + retval = pte_mkexec(retval); + else + retval = pte_mknexec(retval); + + if (pmd_young(pmd)) + retval = pte_mkyoung(retval); + else + retval = pte_mkold(retval); + + if (pmd_dirty(pmd)) + retval = pte_mkdirty(retval); + else + retval = pte_mkclean(retval); + + if (pmd_write(pmd)) + retval = pte_mkwrite(retval); + else + retval = pte_wrprotect(retval); + + if (pmd & PMD_SECT_BUFFERABLE) + retval |= PMD_SECT_BUFFERABLE; + else + retval &= ~PMD_SECT_BUFFERABLE; + + if (pmd & PMD_SECT_CACHEABLE) + retval |= PMD_SECT_CACHEABLE; + else + retval &= ~PMD_SECT_CACHEABLE; + + if (pmd & PMD_SECT_TEX(1)) + retval |= L_PTE_MT_DEV_SHARED; + else + retval &= ~L_PTE_MT_DEV_SHARED; + + if (pmd & PMD_SECT_S) + retval |= L_PTE_SHARED; + else + retval &= ~(L_PTE_SHARED); + + if (pmd_protnone(pmd)) + retval = pte_mkprotnone(retval); + else + retval = pte_rmprotnone(retval); + + return retval; +} + +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + pmdval_t pmdval = (pmdval_t) pte_val(pte); + pmd_t *pmdp = (pmd_t *) ptep; + + /* take the target address bits from the pte only */ + pmdval &= HPAGE_MASK; + + /* + * now use pmd_modify to translate the permission bits from the pte + * and set the memory type information. + */ + pmdval = pmd_val(pmd_modify(__pmd(pmdval), __pgprot(pte_val(pte)))); + + __sync_icache_dcache(pte); + + set_pmd_at(mm, addr, pmdp, __pmd(pmdval)); +} + +static inline pte_t pte_mkhuge(pte_t pte) { return pte; } + +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *)ptep; + + pmd_clear(pmdp); + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *) ptep; + + set_pmd_at(mm, addr, pmdp, pmd_wrprotect(*pmdp)); +} + +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR +static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *)ptep; + pte_t pte = huge_ptep_get(ptep); + + pmd_clear(pmdp); + + return pte; +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + int changed = !pte_same(huge_ptep_get(ptep), pte); + + if (changed) { + set_huge_pte_at(vma->vm_mm, addr, ptep, pte); + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); + } + + return changed; +} + +#endif /* _ASM_ARM_HUGETLB_2LEVEL_H */ diff --git a/arch/arm/include/asm/hugetlb.h b/arch/arm/include/asm/hugetlb.h index d02d6ca88e92..44fd1a5f3a32 100644 --- a/arch/arm/include/asm/hugetlb.h +++ b/arch/arm/include/asm/hugetlb.h @@ -11,7 +11,11 @@ #define _ASM_ARM_HUGETLB_H #include <asm/page.h> -#include <asm/hugetlb-3level.h> +#ifdef CONFIG_ARM_LPAE + #include <asm/hugetlb-3level.h> +#else +#include <asm/hugetlb-2level.h> +#endif #include <asm-generic/hugetlb.h> static inline void arch_clear_hugepage_flags(struct page *page) diff --git a/arch/arm/include/asm/pgtable-2level-hwdef.h b/arch/arm/include/asm/pgtable-2level-hwdef.h index 556937e1790e..b3ea5d00b5c7 100644 --- a/arch/arm/include/asm/pgtable-2level-hwdef.h +++ b/arch/arm/include/asm/pgtable-2level-hwdef.h @@ -20,7 +20,9 @@ #define PMD_PXNTABLE (_AT(pmdval_t, 1) << 2) /* v7 */ #define PMD_BIT4 (_AT(pmdval_t, 1) << 4) #define PMD_DOMAIN(x) (_AT(pmdval_t, (x)) << 5) +#ifndef PMD_DOMAIN_MASK #define PMD_DOMAIN_MASK PMD_DOMAIN(0x0f) +#endif #define PMD_PROTECTION (_AT(pmdval_t, 1) << 9) /* v5 */ /* * - section diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index baf7d0204eb5..000feaf88248 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -7,6 +7,8 @@ #ifndef _ASM_PGTABLE_2LEVEL_H #define _ASM_PGTABLE_2LEVEL_H +#include <asm/tlbflush.h> + #define __PAGETABLE_PMD_FOLDED 1 /* @@ -185,13 +187,27 @@ #define pud_clear(pudp) do { } while (0) #define set_pud(pud,pudp) do { } while (0) +static inline int pmd_large(pmd_t pmd) +{ + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_FAULT) + return pmd_val(pmd); + + return ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT); +} + +static inline int pte_huge(pte_t pte) +{ + pmd_t pmd = (pmd_t)pte; + + return pmd_large(pmd); +} + static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) { return (pmd_t *)pud; } #define pmd_offset pmd_offset -#define pmd_large(pmd) (pmd_val(pmd) & 2) #define pmd_leaf(pmd) (pmd_val(pmd) & 2) #define pmd_bad(pmd) (pmd_val(pmd) & 2) #define pmd_present(pmd) (pmd_val(pmd)) @@ -215,6 +231,142 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) #define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext) +/* + * now follows some of the definitions to allow huge page support, we can't put + * these in the hugetlb source files as they are also required for transparent + * hugepage support. + */ + +#define HPAGE_SHIFT PMD_SHIFT +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +#define HUGE_LINUX_PTE_COUNT (PAGE_OFFSET >> HPAGE_SHIFT) +#define HUGE_LINUX_PTE_SIZE (HUGE_LINUX_PTE_COUNT * sizeof(pte_t *)) +#define HUGE_LINUX_PTE_INDEX(addr) (addr >> HPAGE_SHIFT) + +/* + * We re-purpose the following domain bits in the section descriptor + */ +#ifndef PMD_DOMAIN_MASK +#define PMD_DOMAIN_MASK (_AT(pmdval_t, 0xF) << 5) +#endif +#define PMD_DSECT_PROT_NONE (_AT(pmdval_t, 1) << 5) +#define PMD_DSECT_DIRTY (_AT(pmdval_t, 1) << 6) +#define PMD_DSECT_AF (_AT(pmdval_t, 1) << 7) + +#define PMD_BIT_FUNC(fn, op) \ +static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } + +extern pmdval_t arm_hugepmdprotval; +extern pteval_t arm_hugepteprotval; + +#define pmd_mkhuge(pmd) (__pmd((pmd_val(pmd) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT)) + +PMD_BIT_FUNC(mkold, &= ~PMD_DSECT_AF); +PMD_BIT_FUNC(mkdirty, |= PMD_DSECT_DIRTY); +PMD_BIT_FUNC(mkclean, &= ~PMD_DSECT_DIRTY); +PMD_BIT_FUNC(mkyoung, |= PMD_DSECT_AF); +PMD_BIT_FUNC(mkwrite, |= PMD_SECT_AP_WRITE); +PMD_BIT_FUNC(wrprotect, &= ~PMD_SECT_AP_WRITE); +PMD_BIT_FUNC(mknotpresent, &= ~PMD_TYPE_MASK); +PMD_BIT_FUNC(mkexec, &= ~PMD_SECT_XN); +PMD_BIT_FUNC(mknexec, |= PMD_SECT_XN); +PMD_BIT_FUNC(mkprotnone, |= PMD_DSECT_PROT_NONE); +PMD_BIT_FUNC(rmprotnone, &= ~PMD_DSECT_PROT_NONE); + +#ifdef CONFIG_NUMA_BALANCING +#define pmd_protnone(pmd) (pmd_val(pmd) & PMD_DSECT_PROT_NONE) +#else +static inline int pmd_protnone(pmd_t pmd); +#endif +#define pmd_young(pmd) (pmd_val(pmd) & PMD_DSECT_AF) +#define pmd_write(pmd) (pmd_val(pmd) & PMD_SECT_AP_WRITE) +#define pmd_exec(pmd) (!(pmd_val(pmd) & PMD_SECT_XN)) +#define pmd_dirty(pmd) (pmd_val(pmd) & PMD_DSECT_DIRTY) + +#define __HAVE_ARCH_PMD_WRITE + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + /* + * we can sometimes be passed a pmd pointing to a level 2 descriptor + * from collapse_huge_page. + */ + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_TABLE) { + pmdp[0] = __pmd(pmd_val(pmd)); + pmdp[1] = __pmd(pmd_val(pmd) + 256 * sizeof(pte_t)); + } else { + if (pmd_protnone(pmd)) + pmd_val(pmd) &= ~PMD_TYPE_MASK; + else + pmd_val(pmd) |= PMD_TYPE_SECT; + + pmdp[0] = __pmd(pmd_val(pmd)); + pmdp[1] = __pmd(pmd_val(pmd) + SECTION_SIZE); + } + + flush_pmd_entry(pmdp); +} + +#define pmd_modify(pmd, prot) \ +({ \ + pmd_t pmdret = __pmd((pmd_val(pmd) \ + & (PMD_MASK | PMD_DOMAIN_MASK)) \ + | arm_hugepmdprotval); \ + pgprot_t inprot = prot; \ + pte_t newprot = __pte(pgprot_val(inprot)); \ + \ + if (pte_dirty(newprot)) \ + pmdret = pmd_mkdirty(pmdret); \ + else \ + pmdret = pmd_mkclean(pmdret); \ + \ + if (pte_exec(newprot)) \ + pmdret = pmd_mkexec(pmdret); \ + else \ + pmdret = pmd_mknexec(pmdret); \ + \ + if (pte_write(newprot)) \ + pmdret = pmd_mkwrite(pmdret); \ + else \ + pmdret = pmd_wrprotect(pmdret); \ + \ + if (pte_young(newprot)) \ + pmdret = pmd_mkyoung(pmdret); \ + else \ + pmdret = pmd_mkold(pmdret); \ + \ + if (pte_protnone(newprot)) \ + pmdret = pmd_mkprotnone(pmdret); \ + else \ + pmdret = pmd_rmprotnone(pmdret); \ + \ + if (pte_val(newprot) & PMD_SECT_BUFFERABLE) \ + pmdret |= PMD_SECT_BUFFERABLE; \ + else \ + pmdret &= ~PMD_SECT_BUFFERABLE; \ + \ + if (pte_val(newprot) & PMD_SECT_CACHEABLE) \ + pmdret |= PMD_SECT_CACHEABLE; \ + else \ + pmdret &= ~PMD_SECT_CACHEABLE; \ + \ + if (pte_val(newprot) & L_PTE_MT_DEV_SHARED) \ + pmdret |= PMD_SECT_TEX(1); \ + else \ + pmdret &= ~(PMD_SECT_TEX(1)); \ + \ + if (pte_val(newprot) & L_PTE_SHARED) \ + pmdret |= PMD_SECT_S; \ + else \ + pmdret &= ~PMD_SECT_S; \ + \ + pmdret; \ +}) + /* * We don't have huge page support for short descriptors, for the moment * define empty stubs for use by pin_page_for_write. diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index 2b85d175e999..af53b40d1627 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -55,7 +55,7 @@ * Hugetlb definitions. */ #define HPAGE_SHIFT PMD_SHIFT -#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_SIZE (_AC(1, ULL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index c02f24400369..e3a6ebcfe415 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -196,6 +196,10 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pte_young(pte) (pte_isset((pte), L_PTE_YOUNG)) #define pte_exec(pte) (pte_isclear((pte), L_PTE_XN)) +#ifdef CONFIG_NUMA_BALANCING +#define pte_protnone(pte) (pte_val(pte) & L_PTE_NONE) +#endif + #define pte_valid_user(pte) \ (pte_valid(pte) && pte_isset((pte), L_PTE_USER) && pte_young(pte)) @@ -274,6 +278,16 @@ static inline pte_t pte_mknexec(pte_t pte) return set_pte_bit(pte, __pgprot(L_PTE_XN)); } +static inline pte_t pte_mkprotnone(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(L_PTE_NONE)); +} + +static inline pte_t pte_rmprotnone(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(L_PTE_NONE)); +} + static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const pteval_t mask = L_PTE_XN | L_PTE_RDONLY | L_PTE_USER | diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index f8904227e7fd..dfc95659bb89 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -460,8 +460,12 @@ __enable_mmu: #endif #ifdef CONFIG_ARM_LPAE mcrr p15, 0, r4, r5, c2 @ load TTBR0 +#else +#ifdef CONFIG_SYS_SUPPORTS_HUGETLBFS + ldr r5, =0x55555555 #else mov r5, #DACR_INIT +#endif mcr p15, 0, r5, c3, c0, 0 @ load domain access register mcr p15, 0, r4, c2, c0, 0 @ load page table pointer #endif diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index efa402025031..9985bab5ee66 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -26,6 +26,10 @@ #ifdef CONFIG_MMU +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE) +#include <linux/hugetlb.h> +#endif + /* * This is useful to dump out the page tables associated with * 'addr' in mm 'mm'. @@ -76,6 +80,15 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) break; if (pmd_bad(*pmd)) { +#if !defined(CONFIG_ARM_LPAE) && defined(CONFIG_HUGETLBFS) + if (pte_huge((pte_t)*pgd)) { + pte_t huge_pte = huge_ptep_get((pte_t *)pgd); + + pr_alert("[%08lx] *ppgd=%08llx", addr, + (long long)pmd_val(huge_pte)); + break; + } +#endif pr_cont("(bad)"); break; } @@ -462,19 +475,6 @@ do_translation_fault(unsigned long addr, unsigned int fsr, } #endif /* CONFIG_MMU */ -/* - * Some section permission faults need to be handled gracefully. - * They can happen due to a __{get,put}_user during an oops. - */ -#ifndef CONFIG_ARM_LPAE -static int -do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) -{ - do_bad_area(addr, fsr, regs); - return 0; -} -#endif /* CONFIG_ARM_LPAE */ - /* * This abort handler always returns "fault". */ diff --git a/arch/arm/mm/fsr-2level.c b/arch/arm/mm/fsr-2level.c index f2be95197265..f22e33ab0dc6 100644 --- a/arch/arm/mm/fsr-2level.c +++ b/arch/arm/mm/fsr-2level.c @@ -17,7 +17,7 @@ static struct fsr_info fsr_info[] = { { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, /* @@ -57,7 +57,7 @@ static struct fsr_info ifsr_info[] = { { do_bad, SIGBUS, 0, "unknown 10" }, { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, { do_bad, SIGBUS, 0, "unknown 16" }, diff --git a/arch/arm/mm/hugetlbpage.c b/arch/arm/mm/hugetlbpage.c index dd7a0277c5c0..5ede7ef5e531 100644 --- a/arch/arm/mm/hugetlbpage.c +++ b/arch/arm/mm/hugetlbpage.c @@ -30,5 +30,5 @@ int pud_huge(pud_t pud) int pmd_huge(pmd_t pmd) { - return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); + return pmd_large(pmd); } diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index ab69250a86bc..911ce69f04a0 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -402,6 +402,20 @@ void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) local_flush_tlb_kernel_range(vaddr, vaddr + PAGE_SIZE); } +/* + * If the system supports huge pages and we are running with short descriptors, + * then compute the pmd and linux pte prot values for a huge page. + * + * These values are used by both the HugeTLB and THP code. + */ +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE) +pmdval_t arm_hugepmdprotval; +EXPORT_SYMBOL(arm_hugepmdprotval); + +pteval_t arm_hugepteprotval; +EXPORT_SYMBOL(arm_hugepteprotval); +#endif + /* * Adjust the PMD section entries according to the CPU in use. */ @@ -667,6 +681,17 @@ static void __init build_mem_type_table(void) if (t->prot_sect) t->prot_sect |= PMD_DOMAIN(t->domain); } + +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE) + /* + * we assume all huge pages are user pages and that hardware access + * flag updates are disabled (i.e. SCTLR.AFE == 0b). + */ + arm_hugepteprotval = mem_types[MT_MEMORY_RWX].prot_pte | L_PTE_USER | L_PTE_VALID; + + arm_hugepmdprotval = mem_types[MT_MEMORY_RWX].prot_sect | PMD_SECT_AP_READ + | PMD_SECT_nG; +#endif } #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE -- 2.25.0

2 1

[PATCH OLK-5.10 1/1] ARM: mm: non-LPAE systems HugeTLB support for hulk
by Chen Jun 21 Dec '20

21 Dec '20

From: f00440829 <fanglinxu(a)huawei.com> euler inclusion category: feature bugzilla: 46792 CVE: NA ---------------------------------------- It's a improved version of Steve Capper's RFC patch, see: https://lists.linaro.org/pipermail/linaro-kernel/2013-October/008031.html Signed-off-by: Linxu Fang <fanglinxu(a)huawei.com> Reviewed-by: Hanjun Guo <guohanjun(a)huawei.com> Signed-off-by: zhangyi (F) <yi.zhang(a)huawei.com> Signed-off-by: Jing Xiangfeng <jingxiangfeng(a)huawei.com> Signed-off-by: Chen Jun <chenjun102(a)huawei.com> --- arch/arm/Kconfig | 2 +- arch/arm/include/asm/hugetlb-2level.h | 148 +++++++++++++++++++ arch/arm/include/asm/hugetlb.h | 6 +- arch/arm/include/asm/pgtable-2level-hwdef.h | 2 + arch/arm/include/asm/pgtable-2level.h | 154 +++++++++++++++++++- arch/arm/include/asm/pgtable-3level.h | 2 +- arch/arm/include/asm/pgtable.h | 14 ++ arch/arm/kernel/head.S | 4 + arch/arm/mm/fault.c | 26 ++-- arch/arm/mm/fsr-2level.c | 4 +- arch/arm/mm/hugetlbpage.c | 2 +- arch/arm/mm/mmu.c | 25 ++++ 12 files changed, 369 insertions(+), 20 deletions(-) create mode 100644 arch/arm/include/asm/hugetlb-2level.h diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 002e0cf025f5..3efce886d7a0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1545,7 +1545,7 @@ config HW_PERF_EVENTS config SYS_SUPPORTS_HUGETLBFS def_bool y - depends on ARM_LPAE + depends on ARM_LPAE || (!CPU_USE_DOMAINS && !MEMORY_FAILURE && !CPU_SW_DOMAIN_PAN) config HAVE_ARCH_TRANSPARENT_HUGEPAGE def_bool y diff --git a/arch/arm/include/asm/hugetlb-2level.h b/arch/arm/include/asm/hugetlb-2level.h new file mode 100644 index 000000000000..ce746dae2497 --- /dev/null +++ b/arch/arm/include/asm/hugetlb-2level.h @@ -0,0 +1,148 @@ +/* + * arch/arm/include/asm/hugetlb-2level.h + * + * Copyright (C) 2014 Linaro Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _ASM_ARM_HUGETLB_2LEVEL_H +#define _ASM_ARM_HUGETLB_2LEVEL_H + +#define __HAVE_ARCH_HUGE_PTEP_GET +static inline pte_t huge_ptep_get(pte_t *ptep) +{ + pmd_t pmd = *((pmd_t *)ptep); + pte_t retval; + + if (!pmd_val(pmd)) + return __pte(0); + + retval = __pte((pteval_t) (pmd_val(pmd) & HPAGE_MASK) + | arm_hugepteprotval); + + if (pmd_exec(pmd)) + retval = pte_mkexec(retval); + else + retval = pte_mknexec(retval); + + if (pmd_young(pmd)) + retval = pte_mkyoung(retval); + else + retval = pte_mkold(retval); + + if (pmd_dirty(pmd)) + retval = pte_mkdirty(retval); + else + retval = pte_mkclean(retval); + + if (pmd_write(pmd)) + retval = pte_mkwrite(retval); + else + retval = pte_wrprotect(retval); + + if (pmd & PMD_SECT_BUFFERABLE) + retval |= PMD_SECT_BUFFERABLE; + else + retval &= ~PMD_SECT_BUFFERABLE; + + if (pmd & PMD_SECT_CACHEABLE) + retval |= PMD_SECT_CACHEABLE; + else + retval &= ~PMD_SECT_CACHEABLE; + + if (pmd & PMD_SECT_TEX(1)) + retval |= L_PTE_MT_DEV_SHARED; + else + retval &= ~L_PTE_MT_DEV_SHARED; + + if (pmd & PMD_SECT_S) + retval |= L_PTE_SHARED; + else + retval &= ~(L_PTE_SHARED); + + if (pmd_protnone(pmd)) + retval = pte_mkprotnone(retval); + else + retval = pte_rmprotnone(retval); + + return retval; +} + +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + pmdval_t pmdval = (pmdval_t) pte_val(pte); + pmd_t *pmdp = (pmd_t *) ptep; + + /* take the target address bits from the pte only */ + pmdval &= HPAGE_MASK; + + /* + * now use pmd_modify to translate the permission bits from the pte + * and set the memory type information. + */ + pmdval = pmd_val(pmd_modify(__pmd(pmdval), __pgprot(pte_val(pte)))); + + __sync_icache_dcache(pte); + + set_pmd_at(mm, addr, pmdp, __pmd(pmdval)); +} + +static inline pte_t pte_mkhuge(pte_t pte) { return pte; } + +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *)ptep; + + pmd_clear(pmdp); + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *) ptep; + + set_pmd_at(mm, addr, pmdp, pmd_wrprotect(*pmdp)); +} + +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR +static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *)ptep; + pte_t pte = huge_ptep_get(ptep); + + pmd_clear(pmdp); + + return pte; +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + int changed = !pte_same(huge_ptep_get(ptep), pte); + + if (changed) { + set_huge_pte_at(vma->vm_mm, addr, ptep, pte); + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); + } + + return changed; +} + +#endif /* _ASM_ARM_HUGETLB_2LEVEL_H */ diff --git a/arch/arm/include/asm/hugetlb.h b/arch/arm/include/asm/hugetlb.h index d02d6ca88e92..44fd1a5f3a32 100644 --- a/arch/arm/include/asm/hugetlb.h +++ b/arch/arm/include/asm/hugetlb.h @@ -11,7 +11,11 @@ #define _ASM_ARM_HUGETLB_H #include <asm/page.h> -#include <asm/hugetlb-3level.h> +#ifdef CONFIG_ARM_LPAE + #include <asm/hugetlb-3level.h> +#else +#include <asm/hugetlb-2level.h> +#endif #include <asm-generic/hugetlb.h> static inline void arch_clear_hugepage_flags(struct page *page) diff --git a/arch/arm/include/asm/pgtable-2level-hwdef.h b/arch/arm/include/asm/pgtable-2level-hwdef.h index 556937e1790e..b3ea5d00b5c7 100644 --- a/arch/arm/include/asm/pgtable-2level-hwdef.h +++ b/arch/arm/include/asm/pgtable-2level-hwdef.h @@ -20,7 +20,9 @@ #define PMD_PXNTABLE (_AT(pmdval_t, 1) << 2) /* v7 */ #define PMD_BIT4 (_AT(pmdval_t, 1) << 4) #define PMD_DOMAIN(x) (_AT(pmdval_t, (x)) << 5) +#ifndef PMD_DOMAIN_MASK #define PMD_DOMAIN_MASK PMD_DOMAIN(0x0f) +#endif #define PMD_PROTECTION (_AT(pmdval_t, 1) << 9) /* v5 */ /* * - section diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index baf7d0204eb5..000feaf88248 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -7,6 +7,8 @@ #ifndef _ASM_PGTABLE_2LEVEL_H #define _ASM_PGTABLE_2LEVEL_H +#include <asm/tlbflush.h> + #define __PAGETABLE_PMD_FOLDED 1 /* @@ -185,13 +187,27 @@ #define pud_clear(pudp) do { } while (0) #define set_pud(pud,pudp) do { } while (0) +static inline int pmd_large(pmd_t pmd) +{ + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_FAULT) + return pmd_val(pmd); + + return ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_SECT); +} + +static inline int pte_huge(pte_t pte) +{ + pmd_t pmd = (pmd_t)pte; + + return pmd_large(pmd); +} + static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) { return (pmd_t *)pud; } #define pmd_offset pmd_offset -#define pmd_large(pmd) (pmd_val(pmd) & 2) #define pmd_leaf(pmd) (pmd_val(pmd) & 2) #define pmd_bad(pmd) (pmd_val(pmd) & 2) #define pmd_present(pmd) (pmd_val(pmd)) @@ -215,6 +231,142 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) #define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext) +/* + * now follows some of the definitions to allow huge page support, we can't put + * these in the hugetlb source files as they are also required for transparent + * hugepage support. + */ + +#define HPAGE_SHIFT PMD_SHIFT +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +#define HUGE_LINUX_PTE_COUNT (PAGE_OFFSET >> HPAGE_SHIFT) +#define HUGE_LINUX_PTE_SIZE (HUGE_LINUX_PTE_COUNT * sizeof(pte_t *)) +#define HUGE_LINUX_PTE_INDEX(addr) (addr >> HPAGE_SHIFT) + +/* + * We re-purpose the following domain bits in the section descriptor + */ +#ifndef PMD_DOMAIN_MASK +#define PMD_DOMAIN_MASK (_AT(pmdval_t, 0xF) << 5) +#endif +#define PMD_DSECT_PROT_NONE (_AT(pmdval_t, 1) << 5) +#define PMD_DSECT_DIRTY (_AT(pmdval_t, 1) << 6) +#define PMD_DSECT_AF (_AT(pmdval_t, 1) << 7) + +#define PMD_BIT_FUNC(fn, op) \ +static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } + +extern pmdval_t arm_hugepmdprotval; +extern pteval_t arm_hugepteprotval; + +#define pmd_mkhuge(pmd) (__pmd((pmd_val(pmd) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT)) + +PMD_BIT_FUNC(mkold, &= ~PMD_DSECT_AF); +PMD_BIT_FUNC(mkdirty, |= PMD_DSECT_DIRTY); +PMD_BIT_FUNC(mkclean, &= ~PMD_DSECT_DIRTY); +PMD_BIT_FUNC(mkyoung, |= PMD_DSECT_AF); +PMD_BIT_FUNC(mkwrite, |= PMD_SECT_AP_WRITE); +PMD_BIT_FUNC(wrprotect, &= ~PMD_SECT_AP_WRITE); +PMD_BIT_FUNC(mknotpresent, &= ~PMD_TYPE_MASK); +PMD_BIT_FUNC(mkexec, &= ~PMD_SECT_XN); +PMD_BIT_FUNC(mknexec, |= PMD_SECT_XN); +PMD_BIT_FUNC(mkprotnone, |= PMD_DSECT_PROT_NONE); +PMD_BIT_FUNC(rmprotnone, &= ~PMD_DSECT_PROT_NONE); + +#ifdef CONFIG_NUMA_BALANCING +#define pmd_protnone(pmd) (pmd_val(pmd) & PMD_DSECT_PROT_NONE) +#else +static inline int pmd_protnone(pmd_t pmd); +#endif +#define pmd_young(pmd) (pmd_val(pmd) & PMD_DSECT_AF) +#define pmd_write(pmd) (pmd_val(pmd) & PMD_SECT_AP_WRITE) +#define pmd_exec(pmd) (!(pmd_val(pmd) & PMD_SECT_XN)) +#define pmd_dirty(pmd) (pmd_val(pmd) & PMD_DSECT_DIRTY) + +#define __HAVE_ARCH_PMD_WRITE + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + /* + * we can sometimes be passed a pmd pointing to a level 2 descriptor + * from collapse_huge_page. + */ + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_TABLE) { + pmdp[0] = __pmd(pmd_val(pmd)); + pmdp[1] = __pmd(pmd_val(pmd) + 256 * sizeof(pte_t)); + } else { + if (pmd_protnone(pmd)) + pmd_val(pmd) &= ~PMD_TYPE_MASK; + else + pmd_val(pmd) |= PMD_TYPE_SECT; + + pmdp[0] = __pmd(pmd_val(pmd)); + pmdp[1] = __pmd(pmd_val(pmd) + SECTION_SIZE); + } + + flush_pmd_entry(pmdp); +} + +#define pmd_modify(pmd, prot) \ +({ \ + pmd_t pmdret = __pmd((pmd_val(pmd) \ + & (PMD_MASK | PMD_DOMAIN_MASK)) \ + | arm_hugepmdprotval); \ + pgprot_t inprot = prot; \ + pte_t newprot = __pte(pgprot_val(inprot)); \ + \ + if (pte_dirty(newprot)) \ + pmdret = pmd_mkdirty(pmdret); \ + else \ + pmdret = pmd_mkclean(pmdret); \ + \ + if (pte_exec(newprot)) \ + pmdret = pmd_mkexec(pmdret); \ + else \ + pmdret = pmd_mknexec(pmdret); \ + \ + if (pte_write(newprot)) \ + pmdret = pmd_mkwrite(pmdret); \ + else \ + pmdret = pmd_wrprotect(pmdret); \ + \ + if (pte_young(newprot)) \ + pmdret = pmd_mkyoung(pmdret); \ + else \ + pmdret = pmd_mkold(pmdret); \ + \ + if (pte_protnone(newprot)) \ + pmdret = pmd_mkprotnone(pmdret); \ + else \ + pmdret = pmd_rmprotnone(pmdret); \ + \ + if (pte_val(newprot) & PMD_SECT_BUFFERABLE) \ + pmdret |= PMD_SECT_BUFFERABLE; \ + else \ + pmdret &= ~PMD_SECT_BUFFERABLE; \ + \ + if (pte_val(newprot) & PMD_SECT_CACHEABLE) \ + pmdret |= PMD_SECT_CACHEABLE; \ + else \ + pmdret &= ~PMD_SECT_CACHEABLE; \ + \ + if (pte_val(newprot) & L_PTE_MT_DEV_SHARED) \ + pmdret |= PMD_SECT_TEX(1); \ + else \ + pmdret &= ~(PMD_SECT_TEX(1)); \ + \ + if (pte_val(newprot) & L_PTE_SHARED) \ + pmdret |= PMD_SECT_S; \ + else \ + pmdret &= ~PMD_SECT_S; \ + \ + pmdret; \ +}) + /* * We don't have huge page support for short descriptors, for the moment * define empty stubs for use by pin_page_for_write. diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index 2b85d175e999..af53b40d1627 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -55,7 +55,7 @@ * Hugetlb definitions. */ #define HPAGE_SHIFT PMD_SHIFT -#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_SIZE (_AC(1, ULL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index c02f24400369..e3a6ebcfe415 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -196,6 +196,10 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pte_young(pte) (pte_isset((pte), L_PTE_YOUNG)) #define pte_exec(pte) (pte_isclear((pte), L_PTE_XN)) +#ifdef CONFIG_NUMA_BALANCING +#define pte_protnone(pte) (pte_val(pte) & L_PTE_NONE) +#endif + #define pte_valid_user(pte) \ (pte_valid(pte) && pte_isset((pte), L_PTE_USER) && pte_young(pte)) @@ -274,6 +278,16 @@ static inline pte_t pte_mknexec(pte_t pte) return set_pte_bit(pte, __pgprot(L_PTE_XN)); } +static inline pte_t pte_mkprotnone(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(L_PTE_NONE)); +} + +static inline pte_t pte_rmprotnone(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(L_PTE_NONE)); +} + static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const pteval_t mask = L_PTE_XN | L_PTE_RDONLY | L_PTE_USER | diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index f8904227e7fd..dfc95659bb89 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -460,8 +460,12 @@ __enable_mmu: #endif #ifdef CONFIG_ARM_LPAE mcrr p15, 0, r4, r5, c2 @ load TTBR0 +#else +#ifdef CONFIG_SYS_SUPPORTS_HUGETLBFS + ldr r5, =0x55555555 #else mov r5, #DACR_INIT +#endif mcr p15, 0, r5, c3, c0, 0 @ load domain access register mcr p15, 0, r4, c2, c0, 0 @ load page table pointer #endif diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index efa402025031..9985bab5ee66 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -26,6 +26,10 @@ #ifdef CONFIG_MMU +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE) +#include <linux/hugetlb.h> +#endif + /* * This is useful to dump out the page tables associated with * 'addr' in mm 'mm'. @@ -76,6 +80,15 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) break; if (pmd_bad(*pmd)) { +#if !defined(CONFIG_ARM_LPAE) && defined(CONFIG_HUGETLBFS) + if (pte_huge((pte_t)*pgd)) { + pte_t huge_pte = huge_ptep_get((pte_t *)pgd); + + pr_alert("[%08lx] *ppgd=%08llx", addr, + (long long)pmd_val(huge_pte)); + break; + } +#endif pr_cont("(bad)"); break; } @@ -462,19 +475,6 @@ do_translation_fault(unsigned long addr, unsigned int fsr, } #endif /* CONFIG_MMU */ -/* - * Some section permission faults need to be handled gracefully. - * They can happen due to a __{get,put}_user during an oops. - */ -#ifndef CONFIG_ARM_LPAE -static int -do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) -{ - do_bad_area(addr, fsr, regs); - return 0; -} -#endif /* CONFIG_ARM_LPAE */ - /* * This abort handler always returns "fault". */ diff --git a/arch/arm/mm/fsr-2level.c b/arch/arm/mm/fsr-2level.c index f2be95197265..f22e33ab0dc6 100644 --- a/arch/arm/mm/fsr-2level.c +++ b/arch/arm/mm/fsr-2level.c @@ -17,7 +17,7 @@ static struct fsr_info fsr_info[] = { { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, /* @@ -57,7 +57,7 @@ static struct fsr_info ifsr_info[] = { { do_bad, SIGBUS, 0, "unknown 10" }, { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, { do_bad, SIGBUS, 0, "unknown 16" }, diff --git a/arch/arm/mm/hugetlbpage.c b/arch/arm/mm/hugetlbpage.c index dd7a0277c5c0..5ede7ef5e531 100644 --- a/arch/arm/mm/hugetlbpage.c +++ b/arch/arm/mm/hugetlbpage.c @@ -30,5 +30,5 @@ int pud_huge(pud_t pud) int pmd_huge(pmd_t pmd) { - return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); + return pmd_large(pmd); } diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index ab69250a86bc..911ce69f04a0 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -402,6 +402,20 @@ void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) local_flush_tlb_kernel_range(vaddr, vaddr + PAGE_SIZE); } +/* + * If the system supports huge pages and we are running with short descriptors, + * then compute the pmd and linux pte prot values for a huge page. + * + * These values are used by both the HugeTLB and THP code. + */ +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE) +pmdval_t arm_hugepmdprotval; +EXPORT_SYMBOL(arm_hugepmdprotval); + +pteval_t arm_hugepteprotval; +EXPORT_SYMBOL(arm_hugepteprotval); +#endif + /* * Adjust the PMD section entries according to the CPU in use. */ @@ -667,6 +681,17 @@ static void __init build_mem_type_table(void) if (t->prot_sect) t->prot_sect |= PMD_DOMAIN(t->domain); } + +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE) + /* + * we assume all huge pages are user pages and that hardware access + * flag updates are disabled (i.e. SCTLR.AFE == 0b). + */ + arm_hugepteprotval = mem_types[MT_MEMORY_RWX].prot_pte | L_PTE_USER | L_PTE_VALID; + + arm_hugepmdprotval = mem_types[MT_MEMORY_RWX].prot_sect | PMD_SECT_AP_READ + | PMD_SECT_nG; +#endif } #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE -- 2.25.0

1 0

[PATCH v3 OLK-5.10 0/2] Backport 5.10.1 LTS
by Chen Jun 21 Dec '20

21 Dec '20

v2->v3: remove the patch "Linux-5.10.1.patch" v1->v2: modifty the commit message: linux-5.10.1 -> stable-5.10.1 *** BLURB HERE *** Greg Kroah-Hartman (2): Revert "md: change mddev 'chunk_sectors' from int to unsigned" Revert "dm raid: fix discard limits for raid1 and raid10" drivers/md/dm-raid.c | 12 +++++------- drivers/md/md.h | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) -- 2.25.0

1 2

[PATCH openEuler-1.0-LTS] defconfig: update the defconfigs to support NVDIMM
by Xie XiuQi 21 Dec '20

21 Dec '20

From: Hanjun Guo <guohanjun(a)huawei.com> hulk inclusion category: feature feature: ACPI NVDIMM Firmware Interface Table (NFIT) bugzilla: 46783 CVE: NA Reference: https://gitee.com/open_euler/dashboard/issues?id=I26KWQ Backport this patch from kernel-4.19 to openEuler 20.03 LTS SP1. 564e83bbdcae defconfig: update the defconfigs to support NVDIMM --------------------------- openEuler wants NVDIMM support, update the defconfig. Signed-off-by: Hanjun Guo <guohanjun(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> Signed-off-by: Xie XiuQi <xiexiuqi(a)huawei.com> --- arch/arm64/configs/hulk_defconfig | 2 +- arch/arm64/configs/openeuler_defconfig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/configs/hulk_defconfig b/arch/arm64/configs/hulk_defconfig index 20c7c527dc33..090ff6c93730 100644 --- a/arch/arm64/configs/hulk_defconfig +++ b/arch/arm64/configs/hulk_defconfig @@ -635,7 +635,7 @@ CONFIG_ACPI_HED=y # CONFIG_ACPI_CUSTOM_METHOD is not set # CONFIG_ACPI_BGRT is not set CONFIG_ACPI_REDUCED_HARDWARE_ONLY=y -# CONFIG_ACPI_NFIT is not set +CONFIG_ACPI_NFIT=m CONFIG_HAVE_ACPI_APEI=y CONFIG_ACPI_APEI=y CONFIG_ACPI_APEI_GHES=y diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index c68f4c9ee65c..5561751ad6eb 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -627,7 +627,7 @@ CONFIG_ACPI_HED=y # CONFIG_ACPI_CUSTOM_METHOD is not set # CONFIG_ACPI_BGRT is not set CONFIG_ACPI_REDUCED_HARDWARE_ONLY=y -# CONFIG_ACPI_NFIT is not set +CONFIG_ACPI_NFIT=m CONFIG_HAVE_ACPI_APEI=y CONFIG_ACPI_APEI=y CONFIG_ACPI_APEI_GHES=y -- 2.20.1

1 0

TCM(Nationz Tech) on SPI driver for openEuler
by Li ZhiGang 21 Dec '20

21 Dec '20

hi openEuler kernel team: 我是麒麟软件共性支撑部李志刚, 公司安排我将 TCM on SPI 的驱动移植到您的社区源码; 代码我已经核外编译验证过, 下一步请你们指导我将其放入你们社区(请告诉我最好放入哪个目录). 不好意思我没有社区开发经验, 请您告诉我如何开始好吧? 譬如是不需要我注册个账户, 最好有个简介性的文档. TCM 是国民技术公司的类米国 TPM 芯片/产品, 用于安全和可信计算等. 麒麟软件共性支撑部李志刚 2020-12-18

2 2

[PATCH OLK-5.10 00/26] KVM RISC-V Support
by MingWang Li 19 Dec '20

19 Dec '20

From: Mingwang Li <limingwang(a)huawei.com> euleros inclusion category: feature bugzilla: NA CVE: NA This series adds initial KVM RISC-V support. Currently, we are able to boot Linux on RV64/RV32 Guest with multiple VCPUs. This series can be found in riscv_kvm_v15 branch at: https//github.com/avpatel/linux.git Link: https://gitee.com/openeuler/kernel/issues/I26X9V Alistair Francis (1): Revert "riscv: Use latest system call ABI" Anup Patel (19): RISC-V: Add fragmented config for debug options RISC-V: Enable CPU Hotplug in defconfigs RISC-V: Add Microchip PolarFire kconfig option RISC-V: Initial DTS for Microchip ICICLE board RISC-V: Enable drivers for Microchip PolarFire ICICLE board RISC-V: Add hypervisor extension related CSR defines RISC-V: Add initial skeletal KVM support RISC-V: KVM: Implement VCPU create, init and destroy functions RISC-V: KVM: Implement VCPU interrupts and requests handling RISC-V: KVM: Implement KVM_GET_ONE_REG/KVM_SET_ONE_REG ioctls RISC-V: KVM: Implement VCPU world-switch RISC-V: KVM: Handle MMIO exits for VCPU RISC-V: KVM: Handle WFI exits for VCPU RISC-V: KVM: Implement VMID allocator RISC-V: KVM: Implement stage2 page table programming RISC-V: KVM: Implement MMU notifiers RISC-V: KVM: Document RISC-V specific parts of KVM API RISC-V: KVM: Add MAINTAINERS entry RISC-V: Enable KVM for RV64 and RV32 Atish Patra (4): RISC-V: KVM: Add timer functionality RISC-V: KVM: FP lazy save/restore RISC-V: KVM: Implement ONE REG interface for FP registers RISC-V: KVM: Add SBI v0.1 support Mingwang Li (1): RISCV: KVM: fix bug in migration m34782 (1): Microchip Polarfire SoC Clock Driver Documentation/virt/kvm/api.rst | 193 +++- MAINTAINERS | 11 + arch/riscv/Kconfig | 2 + arch/riscv/Kconfig.socs | 7 + arch/riscv/Makefile | 2 + arch/riscv/boot/dts/Makefile | 1 + arch/riscv/boot/dts/microchip/Makefile | 2 + .../boot/dts/microchip/icicle-kit-es.dts | 307 +++++ arch/riscv/configs/defconfig | 30 +- arch/riscv/configs/extra_debug.config | 21 + arch/riscv/configs/rv32_defconfig | 26 +- arch/riscv/include/asm/csr.h | 89 ++ arch/riscv/include/asm/kvm_host.h | 278 +++++ arch/riscv/include/asm/kvm_types.h | 7 + arch/riscv/include/asm/kvm_vcpu_timer.h | 44 + arch/riscv/include/asm/pgtable-bits.h | 1 + arch/riscv/include/uapi/asm/kvm.h | 128 +++ arch/riscv/include/uapi/asm/unistd.h | 5 +- arch/riscv/kernel/asm-offsets.c | 156 +++ arch/riscv/kvm/Kconfig | 36 + arch/riscv/kvm/Makefile | 15 + arch/riscv/kvm/main.c | 118 ++ arch/riscv/kvm/mmu.c | 860 ++++++++++++++ arch/riscv/kvm/tlb.S | 74 ++ arch/riscv/kvm/vcpu.c | 1012 +++++++++++++++++ arch/riscv/kvm/vcpu_exit.c | 701 ++++++++++++ arch/riscv/kvm/vcpu_sbi.c | 173 +++ arch/riscv/kvm/vcpu_switch.S | 400 +++++++ arch/riscv/kvm/vcpu_timer.c | 225 ++++ arch/riscv/kvm/vm.c | 81 ++ arch/riscv/kvm/vmid.c | 120 ++ drivers/clk/Kconfig | 5 + drivers/clk/Makefile | 1 + drivers/clk/microchip/Makefile | 2 + drivers/clk/microchip/clk-pfsoc.c | 508 +++++++++ drivers/clocksource/timer-riscv.c | 8 + include/clocksource/timer-riscv.h | 16 + include/uapi/linux/kvm.h | 8 + 38 files changed, 5619 insertions(+), 54 deletions(-) create mode 100644 arch/riscv/boot/dts/microchip/Makefile create mode 100644 arch/riscv/boot/dts/microchip/icicle-kit-es.dts create mode 100644 arch/riscv/configs/extra_debug.config create mode 100644 arch/riscv/include/asm/kvm_host.h create mode 100644 arch/riscv/include/asm/kvm_types.h create mode 100644 arch/riscv/include/asm/kvm_vcpu_timer.h create mode 100644 arch/riscv/include/uapi/asm/kvm.h create mode 100644 arch/riscv/kvm/Kconfig create mode 100644 arch/riscv/kvm/Makefile create mode 100644 arch/riscv/kvm/main.c create mode 100644 arch/riscv/kvm/mmu.c create mode 100644 arch/riscv/kvm/tlb.S create mode 100644 arch/riscv/kvm/vcpu.c create mode 100644 arch/riscv/kvm/vcpu_exit.c create mode 100644 arch/riscv/kvm/vcpu_sbi.c create mode 100644 arch/riscv/kvm/vcpu_switch.S create mode 100644 arch/riscv/kvm/vcpu_timer.c create mode 100644 arch/riscv/kvm/vm.c create mode 100644 arch/riscv/kvm/vmid.c create mode 100644 drivers/clk/microchip/clk-pfsoc.c create mode 100644 include/clocksource/timer-riscv.h -- 2.19.1

2 27

[PATCH 1/4] net/hinic: Set default features when probe netdev
by Yang Yingliang 17 Dec '20

17 Dec '20

From: Chiqijun <chiqijun(a)huawei.com> driver inclusion category: bugfix bugzilla: 4472 ----------------------------------------------------------------------- In the x86 system, the hinic device turns off LRO by default, and the LRO state will not be set when the driver is loaded. If the LRO of the chip is turned on before the driver is loaded, it will cause the chip to still turn on LRO after reloading the driver, but the LRO that the user sees is turned off. Therefore, the driver is forced to set the default features of the device when probe. Signed-off-by: Chiqijun <chiqijun(a)huawei.com> Reviewed-by: Wangxiaoyun <cloud.wangxiaoyun(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- drivers/net/ethernet/huawei/hinic/hinic_main.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c index cf7b6ceef060..26cd2929f60a 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_main.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c @@ -1491,7 +1491,7 @@ static int set_feature_lro(struct hinic_nic_dev *nic_dev, static int set_features(struct hinic_nic_dev *nic_dev, netdev_features_t pre_features, - netdev_features_t features, bool force_change) + netdev_features_t features) { netdev_features_t failed_features = 0; u32 err; @@ -1518,7 +1518,7 @@ static int hinic_set_features(struct net_device *netdev, struct hinic_nic_dev *nic_dev = netdev_priv(netdev); return set_features(nic_dev, nic_dev->netdev->features, - features, false); + features); } static netdev_features_t hinic_fix_features(struct net_device *netdev, @@ -1563,7 +1563,8 @@ static int hinic_set_default_hw_feature(struct hinic_nic_dev *nic_dev) } /* enable all hw features in netdev->features */ - return set_features(nic_dev, 0, nic_dev->netdev->features, true); + return set_features(nic_dev, ~nic_dev->netdev->features, + nic_dev->netdev->features); } static int hinic_setup_tc_mqprio(struct net_device *dev, -- 2.25.1

1 3

[PATCH 1/2] ext4: Protect superblock modifications with a buffer lock
by Yang Yingliang 17 Dec '20

17 Dec '20

From: Jan Kara <jack(a)suse.cz> hulk inclusion category: bugfix bugzilla: 46758 CVE: NA --------------------------- Protect all superblock modifications (including checksum computation) with a superblock buffer lock. That way we are sure computed checksum matches current superblock contents (a mismatch could cause checksum failures in nojournal mode or if an unjournalled superblock update races with a journalled one). Also we avoid modifying superblock contents while it is being written out (which can cause DIF/DIX failures if we are running in nojournal mode). Signed-off-by: Jan Kara <jack(a)suse.cz> [backport the 10th patch: https://www.spinics.net/lists/linux-ext4/msg75423.html drop the other lock_buffer besides operation for orphan] Signed-off-by: yangerkun <yangerkun(a)huawei.com> Reviewed-by: zhangyi (F) <yi.zhang(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- fs/ext4/ext4_jbd2.c | 1 - fs/ext4/file.c | 1 + fs/ext4/inode.c | 1 + fs/ext4/namei.c | 6 ++++++ fs/ext4/resize.c | 4 ++++ fs/ext4/xattr.c | 1 + 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c index a589b7f79558..f9ac7dfd93bf 100644 --- a/fs/ext4/ext4_jbd2.c +++ b/fs/ext4/ext4_jbd2.c @@ -361,7 +361,6 @@ int __ext4_handle_dirty_super(const char *where, unsigned int line, struct buffer_head *bh = EXT4_SB(sb)->s_sbh; int err = 0; - ext4_superblock_csum_set(sb); if (ext4_handle_valid(handle)) { err = jbd2_journal_dirty_metadata(handle, bh); if (err) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 52d155b4e733..1703871fa2d0 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -434,6 +434,7 @@ static int ext4_sample_last_mounted(struct super_block *sb, goto out_journal; strlcpy(sbi->s_es->s_last_mounted, cp, sizeof(sbi->s_es->s_last_mounted)); + ext4_superblock_csum_set(sb); ext4_handle_dirty_super(handle, sb); out_journal: ext4_journal_stop(handle); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 8d7e2cd9ae37..6ee89cd7a408 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5398,6 +5398,7 @@ static int ext4_do_update_inode(handle_t *handle, if (err) goto out_brelse; ext4_set_feature_large_file(sb); + ext4_superblock_csum_set(sb); ext4_handle_sync(handle); err = ext4_handle_dirty_super(handle, sb); } diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index ffc3695eb153..762eb6913240 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -2889,7 +2889,10 @@ int ext4_orphan_add(handle_t *handle, struct inode *inode) (le32_to_cpu(sbi->s_es->s_inodes_count))) { /* Insert this inode at the head of the on-disk orphan list */ NEXT_ORPHAN(inode) = le32_to_cpu(sbi->s_es->s_last_orphan); + lock_buffer(sbi->s_sbh); sbi->s_es->s_last_orphan = cpu_to_le32(inode->i_ino); + ext4_superblock_csum_set(sb); + unlock_buffer(sbi->s_sbh); dirty = true; } list_add(&EXT4_I(inode)->i_orphan, &sbi->s_orphan); @@ -2972,7 +2975,10 @@ int ext4_orphan_del(handle_t *handle, struct inode *inode) mutex_unlock(&sbi->s_orphan_lock); goto out_brelse; } + lock_buffer(sbi->s_sbh); sbi->s_es->s_last_orphan = cpu_to_le32(ino_next); + ext4_superblock_csum_set(inode->i_sb); + unlock_buffer(sbi->s_sbh); mutex_unlock(&sbi->s_orphan_lock); err = ext4_handle_dirty_super(handle, inode->i_sb); } else { diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c index 6a0c5c880354..c2e007d836e4 100644 --- a/fs/ext4/resize.c +++ b/fs/ext4/resize.c @@ -901,6 +901,7 @@ static int add_new_gdb(handle_t *handle, struct inode *inode, ext4_kvfree_array_rcu(o_group_desc); le16_add_cpu(&es->s_reserved_gdt_blocks, -1); + ext4_superblock_csum_set(sb); err = ext4_handle_dirty_super(handle, sb); if (err) ext4_std_error(sb, err); @@ -1423,6 +1424,7 @@ static void ext4_update_super(struct super_block *sb, * active. */ ext4_r_blocks_count_set(es, ext4_r_blocks_count(es) + reserved_blocks); + ext4_superblock_csum_set(sb); /* Update the free space counts */ percpu_counter_add(&sbi->s_freeclusters_counter, @@ -1721,6 +1723,7 @@ static int ext4_group_extend_no_check(struct super_block *sb, ext4_blocks_count_set(es, o_blocks_count + add); ext4_free_blocks_count_set(es, ext4_free_blocks_count(es) + add); + ext4_superblock_csum_set(sb); ext4_debug("freeing blocks %llu through %llu\n", o_blocks_count, o_blocks_count + add); /* We add the blocks to the bitmap and set the group need init bit */ @@ -1882,6 +1885,7 @@ static int ext4_convert_meta_bg(struct super_block *sb, struct inode *inode) ext4_set_feature_meta_bg(sb); sbi->s_es->s_first_meta_bg = cpu_to_le32(num_desc_blocks(sb, sbi->s_groups_count)); + ext4_superblock_csum_set(sb); err = ext4_handle_dirty_super(handle, sb); if (err) { diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 24cf730ba6b0..ae029dccebc1 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -791,6 +791,7 @@ static void ext4_xattr_update_super_block(handle_t *handle, BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get_write_access"); if (ext4_journal_get_write_access(handle, EXT4_SB(sb)->s_sbh) == 0) { ext4_set_feature_xattr(sb); + ext4_superblock_csum_set(sb); ext4_handle_dirty_super(handle, sb); } } -- 2.25.1

1 1

[PATCH 01/15] arm64: KVM: Remove some extra semicolon in kvm_target_cpu
by Yang Yingliang 17 Dec '20

17 Dec '20

From: zhong jiang <zhongjiang(a)huawei.com> mainline inclusion from mainline-4.20 commit f0725345e3e127032376e4fcb6b0fc893237fcef category: bugfix bugzilla: NA CVE: NA ------------------------------------------------- There are some extra semicolon in kvm_target_cpu, remove it. Signed-off-by: zhong jiang <zhongjiang(a)huawei.com> Signed-off-by: Marc Zyngier <marc.zyngier(a)arm.com> Signed-off-by: Xiangyou Xie <xiexiangyou(a)huawei.com> Reviewed-by: Ying Fang <fangying1(a)huawei.com> Reviewed-by: Zenghui Yu <yuzenghui(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- arch/arm64/kvm/guest.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index ff9908c5d496..ff00e1c8023a 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -432,15 +432,15 @@ int __attribute_const__ kvm_target_cpu(void) return KVM_ARM_TARGET_CORTEX_A53; case ARM_CPU_PART_CORTEX_A57: return KVM_ARM_TARGET_CORTEX_A57; - }; + } break; case ARM_CPU_IMP_APM: switch (part_number) { case APM_CPU_PART_POTENZA: return KVM_ARM_TARGET_XGENE_POTENZA; - }; + } break; - }; + } /* Return a default generic target */ return KVM_ARM_TARGET_GENERIC_V8; -- 2.25.1

1 14

[PATCH] mm: thp: make the THP mapcount atomic against __split_huge_pmd_locked()
by Yang Yingliang 17 Dec '20

17 Dec '20

From: Andrea Arcangeli <aarcange(a)redhat.com> stable inclusion from linux-4.19.129 commit 453d8a481b127edfa7fb76af1bc7586b7a63bdd2 CVE: CVE-2020-29368 -------------------------------- commit c444eb564fb16645c172d550359cb3d75fe8a040 upstream. Write protect anon page faults require an accurate mapcount to decide if to break the COW or not. This is implemented in the THP path with reuse_swap_page() -> page_trans_huge_map_swapcount()/page_trans_huge_mapcount(). If the COW triggers while the other processes sharing the page are under a huge pmd split, to do an accurate reading, we must ensure the mapcount isn't computed while it's being transferred from the head page to the tail pages. reuse_swap_cache() already runs serialized by the page lock, so it's enough to add the page lock around __split_huge_pmd_locked too, in order to add the missing serialization. Note: the commit in "Fixes" is just to facilitate the backporting, because the code before such commit didn't try to do an accurate THP mapcount calculation and it instead used the page_count() to decide if to COW or not. Both the page_count and the pin_count are THP-wide refcounts, so they're inaccurate if used in reuse_swap_page(). Reverting such commit (besides the unrelated fix to the local anon_vma assignment) would have also opened the window for memory corruption side effects to certain workloads as documented in such commit header. Signed-off-by: Andrea Arcangeli <aarcange(a)redhat.com> Suggested-by: Jann Horn <jannh(a)google.com> Reported-by: Jann Horn <jannh(a)google.com> Acked-by: Kirill A. Shutemov <kirill.shutemov(a)linux.intel.com> Fixes: 6d0a07edd17c ("mm: thp: calculate the mapcount correctly for THP pages during WP faults") Cc: stable(a)vger.kernel.org Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org> Reviewed-by: Jason Yan <yanaijie(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- mm/huge_memory.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b60cbf77e902..816a7fd3c6ff 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2286,6 +2286,8 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, spinlock_t *ptl; struct mm_struct *mm = vma->vm_mm; unsigned long haddr = address & HPAGE_PMD_MASK; + bool was_locked = false; + pmd_t _pmd; mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE); ptl = pmd_lock(mm, pmd); @@ -2295,11 +2297,32 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, * pmd against. Otherwise we can end up replacing wrong page. */ VM_BUG_ON(freeze && !page); - if (page && page != pmd_page(*pmd)) - goto out; + if (page) { + VM_WARN_ON_ONCE(!PageLocked(page)); + was_locked = true; + if (page != pmd_page(*pmd)) + goto out; + } +repeat: if (pmd_trans_huge(*pmd)) { - page = pmd_page(*pmd); + if (!page) { + page = pmd_page(*pmd); + if (unlikely(!trylock_page(page))) { + get_page(page); + _pmd = *pmd; + spin_unlock(ptl); + lock_page(page); + spin_lock(ptl); + if (unlikely(!pmd_same(*pmd, _pmd))) { + unlock_page(page); + put_page(page); + page = NULL; + goto repeat; + } + put_page(page); + } + } if (PageMlocked(page)) clear_page_mlock(page); } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd))) @@ -2307,6 +2330,8 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, __split_huge_pmd_locked(vma, pmd, haddr, freeze); out: spin_unlock(ptl); + if (!was_locked && page) + unlock_page(page); /* * No need to double call mmu_notifier->invalidate_range() callback. * They are 3 cases to consider inside __split_huge_pmd_locked(): -- 2.25.1

1 0