From: Zeng Heng <zengheng4@huawei.com> hulk inclusion category: feature bugzilla: https://gitcode.com/openeuler/kernel/issues/10066 ---------------------------------------- TLBIP is the 128-bit form of TLBI (an alias of the SYSP system instruction) and is available when FEAT_TLBID is implemented. It carries the TLBI Domain in bits [15:0] of the first register of an even-aligned register pair (Xt, Xt+1), restricting the invalidation to the PEs within that domain. This allows a range invalidation to be multicast to the CPUs of a single domain instead of being broadcast to every CPU in the system. Extend __flush_tlb_range_nosync() to issue the invalidation with TLBIP RVAE1IS/RVALE1IS (or VAE1IS/VALE1IS for a single page) carrying the domain selected by pick_best_domain() when the CPUs the mm has run on fit into a single TLBI Domain, mirroring what flush_tlb_mm() already does for the whole address space. The *E1IS variants are used, like the existing aside1is in flush_tlb_mm(), so the invalidation completes with the usual DSB ISH. Fall back to the regular TLBI broadcast when no suitable domain is found. The TLBIP mnemonic is used when the assembler supports it (CONFIG_AS_HAS_TLBIP), with the instructions emitted as raw encodings otherwise. Since the encoding only holds one register and implies its pair as Xt+1, the operand pair is fixed to x10/x11. TTL64 is set in the operand so that the TTL hint applies to the VMSAv8-64 TLB entries created by the kernel. Signed-off-by: Zeng Heng <zengheng4@huawei.com> --- arch/arm64/Kconfig | 9 ++ arch/arm64/include/asm/tlbflush.h | 191 ++++++++++++++++++++++++++++-- 2 files changed, 193 insertions(+), 7 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index fe3fa113e5cd..50146c8f8617 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2306,6 +2306,7 @@ config ARM64_TLB_RANGE config ARM64_TLBID bool "Enable support for TLBID (TLBI Domains)" default y + depends on AS_HAS_TLBIP help TLBI broadcasting all PEs introduces performance noise. By combining hardware and software, TLBID (TLBI Domain) limit TLBI @@ -2323,6 +2324,14 @@ config ARM64_TLBID The feature is detected at runtime, and will remain disabled if the system does not implement the feature. +config AS_HAS_TLBIP + def_bool $(as-instr,.arch_extension d128) + help + The TLBIP instruction mnemonic requires the FEAT_D128 extension + of the assembler. When the assembler does not support it, the + kernel falls back to emitting TLBIP instructions as raw + encodings. + config ARM64_MPAM bool "Enable support for MPAM" select ACPI_MPAM if ACPI diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 03ebfc94287f..615e35c1968f 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -445,6 +445,150 @@ do { \ #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false) +/* + * TLBIP is the 128-bit form of TLBI (an alias of the SYSP system + * instruction, FEAT_SYSINSTR128) and takes a pair of consecutive, + * even-aligned registers (Xt, Xt+1), of which only Xt is encoded in the + * instruction. With FEAT_TLBID the invalidation is restricted to the PEs + * that are within both the broadcast shareability domain and the TLBI + * Domain held in bits [15:0] of Xt. This allows a range invalidation to + * be multicast to the CPUs of a single domain instead of being broadcast + * to every CPU in the system. + * + * The operand pair of the *E1IS range instructions is laid out as: + * + * Xt : [63:48] ASID, [47:46] TG, [45:44] SCALE, [43:39] NUM, + * [38:37] TTL, [32] TTL64, [15:0] TLBID + * Xt+1: [43:0] BaseADDR[55:12] + * + * and for the non-range VAE1IS/VALE1IS instructions as: + * + * Xt : [63:48] ASID, [47:44] TTL, [32] TTL64, [15:0] TLBID + * Xt+1: [43:0] VA[55:12] + * + * TTL64 is always set so that the TTL hint applies to the VMSAv8-64 TLB + * entries, which are the only ones the kernel creates. When the assembler + * does not support the TLBIP mnemonic (CONFIG_AS_HAS_TLBIP), the + * instructions are emitted as raw encodings instead. + */ +#define __TLBI_PAIR_LO "x10" +#define __TLBI_PAIR_HI "x11" + +#define __TLBIP_PREAMBLE ".arch_extension d128\n" +#define __tlbip(op, lo, hi) \ +do { \ + register unsigned long __tlbip_lo asm (__TLBI_PAIR_LO) = (lo); \ + register unsigned long __tlbip_hi asm (__TLBI_PAIR_HI) = (hi); \ + \ + asm volatile(__TLBIP_PREAMBLE \ + "tlbip " #op ", " __TLBI_PAIR_LO ", " __TLBI_PAIR_HI "\n\t" \ + : : "r" (__tlbip_lo), "r" (__tlbip_hi)); \ +} while (0) + +#define __tlbip_user(insn, lo, hi) \ +do { \ + if (arm64_kernel_unmapped_at_el0()) \ + __tlbip(insn, (lo) | USER_ASID_FLAG, hi); \ +} while (0) + +/* Build the lower register of a TLBIP operand pair carrying @asid. */ +static __always_inline unsigned long +__tlbip_range_lo(unsigned long asid, unsigned long domain, + int scale, int num, int ttl) +{ + return FIELD_PREP(GENMASK_ULL(63, 48), asid) | + FIELD_PREP(GENMASK_ULL(47, 46), get_trans_granule()) | + FIELD_PREP(GENMASK_ULL(45, 44), scale) | + FIELD_PREP(GENMASK_ULL(43, 39), num) | + FIELD_PREP(GENMASK_ULL(38, 37), ttl) | + FIELD_PREP(BIT_ULL(32), 1) | /* TTL64 */ + FIELD_PREP(GENMASK_ULL(15, 0), domain); +} + +/* Build the upper register of a TLBIP operand pair. */ +static __always_inline unsigned long __tlbip_addr_hi(unsigned long addr) +{ + return FIELD_PREP(GENMASK_ULL(43, 0), addr >> PAGE_SHIFT); +} + +/* + * Build the lower register of the non-range TLBIP VAE1IS/VALE1IS operand + * pair. The TTL field uses the same granule+level encoding as + * __tlbi_level(). + */ +static __always_inline unsigned long +__tlbip_va_lo(unsigned long asid, unsigned long domain, int tlb_level) +{ + unsigned long lo = FIELD_PREP(GENMASK_ULL(63, 48), asid) | + FIELD_PREP(BIT_ULL(32), 1) | /* TTL64 */ + FIELD_PREP(GENMASK_ULL(15, 0), domain); + + if (cpus_have_const_cap(ARM64_HAS_ARMv8_4_TTL) && tlb_level) { + unsigned long ttl = (tlb_level & 3) | + (get_trans_granule() << 2); + + lo |= FIELD_PREP(GENMASK_ULL(47, 44), ttl); + } + + return lo; +} + +/* + * __flush_tlb_range_domain_op - Perform a TLBIP range operation restricted + * to a single TLBI Domain + * + * Same algorithm as __flush_tlb_range_op(), except that the invalidation is + * issued with the TLBIP *E1IS instructions carrying @domain, so that only + * the PEs of that domain have to process it. Like the regular TLBI + * instructions they broadcast to the Inner Shareable domain and are + * completed with the usual DSB ISH. + * + * @op: TLBIP operation name for the non-range instruction + * @rop: TLBIP operation name for the range instruction + * @start: The start address of the range + * @pages: Range as the number of pages from 'start' + * @stride: Flush granularity + * @asid: The ASID of the task + * @domain: The TLBI Domain the invalidation targets + * @tlb_level: Translation Table level hint, if known + */ +#define __flush_tlb_range_domain_op(op, rop, start, pages, stride, \ + asid, domain, tlb_level) \ +do { \ + typeof(start) __flush_start = start; \ + typeof(pages) __flush_pages = pages; \ + int num = 0; \ + int scale = 3; \ + unsigned long __lo; \ + \ + while (__flush_pages > 0) { \ + if (!system_supports_tlb_range() || \ + __flush_pages == 1) { \ + __lo = __tlbip_va_lo(asid, domain, tlb_level); \ + __tlbip(op, __lo, \ + __tlbip_addr_hi(__flush_start)); \ + __tlbip_user(op, __lo, \ + __tlbip_addr_hi(__flush_start)); \ + __flush_start += stride; \ + __flush_pages -= stride >> PAGE_SHIFT; \ + continue; \ + } \ + \ + num = __TLBI_RANGE_NUM(__flush_pages, scale); \ + if (num >= 0) { \ + __lo = __tlbip_range_lo(asid, domain, \ + scale, num, tlb_level); \ + __tlbip(rop, __lo, \ + __tlbip_addr_hi(__flush_start)); \ + __tlbip_user(rop, __lo, \ + __tlbip_addr_hi(__flush_start)); \ + __flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ + __flush_pages -= __TLBI_RANGE_PAGES(num, scale);\ + } \ + scale--; \ + } \ +} while (0) + static inline bool __flush_tlb_range_limit_excess(unsigned long start, unsigned long end, unsigned long pages, unsigned long stride) { @@ -462,12 +606,22 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long start, return false; } +/* + * Issue the TLBI operations for the range [start, end) without a trailing + * DSB, which the caller is responsible for issuing. + * + * When the CPUs the mm has run on fit into a single TLBI Domain, the + * invalidation is multicast to that domain with the TLBIP *E1IS + * instructions, sparing the CPUs outside of it from having to process the + * broadcast. + */ static inline void __flush_tlb_range_nosync(struct mm_struct *mm, - unsigned long start, unsigned long end, - unsigned long stride, bool last_level, - int tlb_level) + unsigned long start, unsigned long end, + unsigned long stride, bool last_level, + int tlb_level) { unsigned long asid, pages; + int domain, scope; start = round_down(start, stride); end = round_up(end, stride); @@ -481,10 +635,33 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm, dsb(ishst); asid = ASID(mm); - if (last_level) - __flush_tlb_range_op(vale1is, start, pages, stride, asid, tlb_level, true); - else - __flush_tlb_range_op(vae1is, start, pages, stride, asid, tlb_level, true); + scope = flush_tlb_user_scope(mm); + if (scope == TLB_FLUSH_SCOPE_DOMAIN) { + domain = pick_best_domain(mm_cpumask(mm)); + + if (last_level) + __flush_tlb_range_domain_op(vale1is, rvale1is, + start, pages, stride, asid, domain, + tlb_level); + else + __flush_tlb_range_domain_op(vae1is, rvae1is, + start, pages, stride, asid, domain, + tlb_level); + } else if (scope == TLB_FLUSH_SCOPE_BROADCAST) { + if (last_level) + __flush_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level, true); + else + __flush_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level, true); + } else { + if (last_level) + __flush_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level, true); + else + __flush_tlb_range_op(vae1, start, pages, stride, + asid, tlb_level, true); + } mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } -- 2.33.0