From: Cheng Jian cj.chengjian@huawei.com
hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4BLL0 CVE: NA
---------------------------
add CONFIG_ARM64_TLBI_IPI to isolate code.
Signed-off-by: Cheng Jian cj.chengjian@huawei.com Reviewed-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- arch/arm64/Kconfig | 21 +++++++++++++++++++++ arch/arm64/include/asm/mmu_context.h | 4 +++- arch/arm64/kernel/smp.c | 4 ++++ arch/arm64/kernel/tlbflush.c | 25 ++++++++++++++++++++++--- arch/arm64/mm/context.c | 4 ++++ 5 files changed, 54 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 2f34aef79179e..f984280b0d577 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1607,6 +1607,27 @@ config UCE_KERNEL_RECOVERY
endmenu
+menu "TLB options" + +config ARM64_TLBI_IPI + bool "IPI based ARM64 TLB invalidation" + depends on ARM64 + default n + help + adds new boot parameter 'disable_tlbflush_is' to disable TLB flush + within the same inner shareable domain for performance tuning. + + When this new parameter is specified, TLB entry is invalidated by + __tlbi(aside1, asid) only on the CPUs specified by mm_cpumask(mm). + + By using TLB.IS, all CPUs within the same inner shareable domain + check if there are TLB entries which have this ASID, this causes + performance noise, especially at large-scale HPC environment, which + has more than thousand nodes with low latency interconnect. + + If unsure, say N. +endmenu + menu "CPU Power Management"
source "drivers/cpuidle/Kconfig" diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 005345d77349c..e319ce86fa708 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -229,12 +229,14 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { - unsigned int cpu = smp_processor_id(); + unsigned int __maybe_unused cpu = smp_processor_id();
if (prev != next) { __switch_mm(next); +#ifdef CONFIG_ARM64_TLBI_IPI cpumask_clear_cpu(cpu, mm_cpumask(prev)); local_flush_tlb_mm(prev); +#endif }
/* diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index e86940d353a3e..f09c10863867b 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -387,7 +387,9 @@ asmlinkage notrace void secondary_start_kernel(void) */ mmgrab(mm); current->active_mm = mm; +#ifdef CONFIG_ARM64_TLBI_IPI cpumask_set_cpu(cpu, mm_cpumask(mm)); +#endif
/* * TTBR0 is only used for the identity mapping at this stage. Make it @@ -490,10 +492,12 @@ int __cpu_disable(void) */ irq_migrate_all_off_this_cpu();
+#ifdef CONFIG_ARM64_TLBI_IPI /* * Remove this CPU from the vm mask set of all processes. */ clear_tasks_mm_cpumask(cpu); +#endif
return 0; } diff --git a/arch/arm64/kernel/tlbflush.c b/arch/arm64/kernel/tlbflush.c index 52c9a237759a6..e20fbd38fd262 100644 --- a/arch/arm64/kernel/tlbflush.c +++ b/arch/arm64/kernel/tlbflush.c @@ -4,6 +4,7 @@ #include <linux/smp.h> #include <asm/tlbflush.h>
+#ifdef CONFIG_ARM64_TLBI_IPI struct tlb_args { struct vm_area_struct *ta_vma; unsigned long ta_start; @@ -21,6 +22,7 @@ static int __init disable_tlbflush_is_setup(char *str) return 0; } __setup("disable_tlbflush_is", disable_tlbflush_is_setup); +#endif
static inline void __flush_tlb_mm(struct mm_struct *mm) { @@ -32,20 +34,26 @@ static inline void __flush_tlb_mm(struct mm_struct *mm) dsb(ish); }
+#ifdef CONFIG_ARM64_TLBI_IPI static inline void ipi_flush_tlb_mm(void *arg) { struct mm_struct *mm = arg;
local_flush_tlb_mm(mm); } +#endif
void flush_tlb_mm(struct mm_struct *mm) { - if (disable_tlbflush_is) +#ifdef CONFIG_ARM64_TLBI_IPI + if (unlikely(disable_tlbflush_is)) on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, (void *)mm, true); else __flush_tlb_mm(mm); +#else + __flush_tlb_mm(mm); +#endif }
static inline void __flush_tlb_page_nosync(unsigned long addr) @@ -74,11 +82,15 @@ void flush_tlb_page_nosync(struct vm_area_struct *vma, unsigned long uaddr) { unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm));
- if (disable_tlbflush_is) +#ifdef CONFIG_ARM64_TLBI_IPI + if (unlikely(disable_tlbflush_is)) on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page_nosync, &addr, true); else __flush_tlb_page_nosync(addr); +#else + __flush_tlb_page_nosync(addr); +#endif }
static inline void ___flush_tlb_range(unsigned long start, unsigned long end, @@ -112,6 +124,7 @@ static inline void __local_flush_tlb_range(unsigned long addr, bool last_level) dsb(nsh); }
+#ifdef CONFIG_ARM64_TLBI_IPI static inline void ipi_flush_tlb_range(void *arg) { struct tlb_args *ta = (struct tlb_args *)arg; @@ -120,6 +133,7 @@ static inline void ipi_flush_tlb_range(void *arg) for (addr = ta->ta_start; addr < ta->ta_end; addr += ta->ta_stride) __local_flush_tlb_range(addr, ta->ta_last_level); } +#endif
void __flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long stride, bool last_level) @@ -140,7 +154,9 @@ void __flush_tlb_range(struct vm_area_struct *vma, unsigned long start, start = __TLBI_VADDR(start, asid); end = __TLBI_VADDR(end, asid);
- if (disable_tlbflush_is) { + +#ifdef CONFIG_ARM64_TLBI_IPI + if (unlikely(disable_tlbflush_is)) { struct tlb_args ta = { .ta_start = start, .ta_end = end, @@ -152,4 +168,7 @@ void __flush_tlb_range(struct vm_area_struct *vma, unsigned long start, &ta, true); } else ___flush_tlb_range(start, end, stride, last_level); +#else + ___flush_tlb_range(start, end, stride, last_level); +#endif } diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 27d1f3fec1cc9..35a103c7c22bf 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -207,7 +207,9 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu) set_asid: __set_bit(asid, asid_map); cur_idx = asid; +#ifdef CONFIG_ARM64_TLBI_IPI cpumask_clear(mm_cpumask(mm)); +#endif return idx2asid(asid) | generation; }
@@ -255,7 +257,9 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu) switch_mm_fastpath:
arm64_apply_bp_hardening(); +#ifdef CONFIG_ARM64_TLBI_IPI cpumask_set_cpu(cpu, mm_cpumask(mm)); +#endif
/* * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when