hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10207 ---------------------------------------- Enable mm_cpumask tracking unconditionally, so the mask is always valid even without FEAT_TLBID. Then in flush_tlb_begin(), use it to decide the flush scope: with IRQs disabled and a DSB ISHST ordering the page-table updates, if no other CPU is in the mask, do a local TLBI instead of broadcasting a DVM to the inner-shareable domain. UnixBench(3-round average): Process Creation +8.0%, Execl +7.7%, Shell Scripts +5.7%. Reported-by: Nianyao Tang <tangnianyao@huawei.com> Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- arch/arm64/include/asm/tlbflush.h | 17 +++++++---------- arch/arm64/kernel/smp.c | 3 +-- arch/arm64/mm/context.c | 6 ++---- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 5ccc0982f834..6fa909874a19 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -188,20 +188,17 @@ enum tlb_flush_scope { static inline enum tlb_flush_scope flush_tlb_begin(struct mm_struct *mm, unsigned long *flags) { - if (!system_supports_tlbid()) { - dsb(ishst); - return TLB_FLUSH_SCOPE_BROADCAST; - } - local_irq_save(*flags); + dsb(ishst); - if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) >= nr_cpu_ids) { - dsb(nshst); + if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) >= nr_cpu_ids) return TLB_FLUSH_SCOPE_LOCAL; - } else { - dsb(ishst); + + if (system_supports_tlbid()) return TLB_FLUSH_SCOPE_DOMAIN; - } + + local_irq_restore(*flags); + return TLB_FLUSH_SCOPE_BROADCAST; } static inline void flush_tlb_end(enum tlb_flush_scope scope, unsigned long flags) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 4290231e02e0..1ed686b3c0f4 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -351,8 +351,7 @@ void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) pr_debug("CPU%u: shutdown\n", cpu); - if (system_supports_tlbid()) - clear_tasks_mm_cpumask(cpu); + clear_tasks_mm_cpumask(cpu); /* * Now that the dying CPU is beyond the point of no return w.r.t. diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index ce314cb9fd09..294aa0da7726 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -206,8 +206,7 @@ static u64 new_context(struct mm_struct *mm) /* We have more ASIDs than CPUs, so this will always succeed */ asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1); - if (system_supports_tlbid()) - cpumask_clear(mm_cpumask(mm)); + cpumask_clear(mm_cpumask(mm)); set_asid: __set_bit(asid, asid_map); @@ -264,8 +263,7 @@ void check_and_switch_context(struct mm_struct *mm) arm64_apply_bp_hardening(); - if (system_supports_tlbid()) - cpumask_set_cpu(cpu, mm_cpumask(mm)); + cpumask_set_cpu(cpu, mm_cpumask(mm)); /* * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when -- 2.33.0