From: Zeng Heng <zengheng4@huawei.com> hulk inclusion category: feature bugzilla: https://gitcode.com/openeuler/kernel/issues/10066 ---------------------------------------- flush_tlb_page() currently broadcasts the invalidation with TLBI VALE1IS regardless of where the mm has run. As with the range flush path, restrict it to the TLBI Domain covering the CPUs the mm has run on when such a domain exists, issuing TLBIP VALE1IS instead. The non-range TLBIP instructions take the VA in the upper register of the pair and the ASID, TTL, TTL64 and TLBID fields in the lower one. No TTL hint is passed, keeping the behaviour identical to the existing __tlbi(vale1is). The page flush helpers are moved after the TLBIP machinery in the header so that they can use it. Signed-off-by: Zeng Heng <zengheng4@huawei.com> --- arch/arm64/include/asm/tlbflush.h | 71 ++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 615e35c1968f..b43c12086b4b 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -309,32 +309,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm) mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); } -static inline void __flush_tlb_page_nosync(struct mm_struct *mm, - unsigned long uaddr) -{ - unsigned long addr; - - dsb(ishst); - addr = __TLBI_VADDR(uaddr, ASID(mm)); - __tlbi(vale1is, addr); - __tlbi_user(vale1is, addr); - mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, - (uaddr & PAGE_MASK) + PAGE_SIZE); -} - -static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, - unsigned long uaddr) -{ - return __flush_tlb_page_nosync(vma->vm_mm, uaddr); -} - -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long uaddr) -{ - flush_tlb_page_nosync(vma, uaddr); - dsb(ish); -} - static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) { #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI @@ -687,6 +661,51 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, __flush_tlb_range(vma, start, end, PAGE_SIZE, false, 0); } +static inline void __flush_tlb_page_nosync(struct mm_struct *mm, + unsigned long uaddr) +{ + unsigned long asid = ASID(mm); + int domain, scope; + unsigned long lo; + + dsb(ishst); + + scope = flush_tlb_user_scope(mm); + if (scope == TLB_FLUSH_SCOPE_DOMAIN) { + domain = pick_best_domain(mm_cpumask(mm)); + + lo = __tlbip_va_lo(asid, domain, 0); + __tlbip(vale1is, lo, __tlbip_addr_hi(uaddr)); + __tlbip_user(vale1is, lo, __tlbip_addr_hi(uaddr)); + } else { + unsigned long addr = __TLBI_VADDR(uaddr, asid); + + if (scope == TLB_FLUSH_SCOPE_BROADCAST) { + __tlbi(vale1is, addr); + __tlbi_user(vale1is, addr); + } else { + __tlbi(vale1, addr); + __tlbi_user(vale1, addr); + } + } + + mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, + (uaddr & PAGE_MASK) + PAGE_SIZE); +} + +static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, + unsigned long uaddr) +{ + return __flush_tlb_page_nosync(vma->vm_mm, uaddr); +} + +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long uaddr) +{ + flush_tlb_page_nosync(vma, uaddr); + dsb(ish); +} + static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) { const unsigned long stride = PAGE_SIZE; -- 2.33.0