From: Linu Cherian <linu.cherian@arm.com> mainline inclusion from mainline-v7.3-rc1 commit 94104e3cfa8036ac2457aa0c2ca1351e16d77159 category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9850 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ------------------------------------------- - As bbml2_noabort is functionally equivalent to bbml3, rename cpu/system_supports_bbml2_noabort to cpu/system_supports_bbml3. The ARM64 capability name is also renamed accordingly. - As BBML2_NOABORT or the equivalent BBML3 is the kernel requirement for setting up linear map with block/contpte mappings and not BBML2, replace all bbml2 references with bbml3. FEAT_BBML3, is introduced as part of 2025 Architecture Extensions. https://developer.arm.com/documentation/109697/2026_03/2025-Architecture-Ext... No functional changes are introduced with this patch. Reviewed-by: Gavin Shan <gshan@redhat.com> Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com> Signed-off-by: Linu Cherian <linu.cherian@arm.com> Signed-off-by: Will Deacon <will@kernel.org> Conflicts: arch/arm64/include/asm/cpufeature.h arch/arm64/kernel/cpufeature.c arch/arm64/mm/contpte.c arch/arm64/mm/mmu.c arch/arm64/tools/cpucaps [Context conflicts.] Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- arch/arm64/include/asm/cpufeature.h | 6 ++-- arch/arm64/include/asm/mmu.h | 2 +- arch/arm64/kernel/cpufeature.c | 32 ++++++------------- arch/arm64/mm/contpte.c | 21 +++++-------- arch/arm64/mm/mmu.c | 48 ++++++++++++++--------------- arch/arm64/mm/proc.S | 4 +-- arch/arm64/tools/cpucaps | 2 +- 7 files changed, 49 insertions(+), 66 deletions(-) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index e8f3c6a2eb63..e0137a7c2cfb 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -932,11 +932,11 @@ static inline bool system_supports_haft(void) cpus_have_final_cap(ARM64_HAFT); } -bool cpu_supports_bbml2_noabort(void); +bool cpu_supports_bbml3(void); -static inline bool system_supports_bbml2_noabort(void) +static inline bool system_supports_bbml3(void) { - return alternative_has_cap_unlikely(ARM64_HAS_BBML2_NOABORT); + return alternative_has_cap_unlikely(ARM64_HAS_BBML3); } int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 72a6782e13df..6af4930f5c2d 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -72,7 +72,7 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, extern void *fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot); extern void mark_linear_text_alias_ro(void); extern int split_kernel_leaf_mapping(unsigned long start, unsigned long end); -extern void init_idmap_kpti_bbml2_flag(void); +extern void init_idmap_kpti_bbml3_flag(void); extern void linear_map_maybe_split_to_ptes(void); extern bool kaslr_requires_kpti(void); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 4005901fee37..a76e8b6a31ce 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1927,7 +1927,7 @@ static void __init kpti_install_ng_mappings(void) if (arm64_use_ng_mappings) return; - init_idmap_kpti_bbml2_flag(); + init_idmap_kpti_bbml3_flag(); stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask); } @@ -2137,28 +2137,16 @@ static bool hvhe_possible(const struct arm64_cpu_capabilities *entry, return cpuid_feature_extract_unsigned_field(val, ARM64_SW_FEATURE_OVERRIDE_HVHE); } -bool cpu_supports_bbml2_noabort(void) +bool cpu_supports_bbml3(void) { - /* - * We want to allow usage of BBML2 in as wide a range of kernel contexts - * as possible. This list is therefore an allow-list of known-good - * implementations that both support BBML2 and additionally, fulfill the - * extra constraint of never generating TLB conflict aborts when using - * the relaxed BBML2 semantics (such aborts make use of BBML2 in certain - * kernel contexts difficult to prove safe against recursive aborts). - * - * Note that implementations can only be considered "known-good" if their - * implementors attest to the fact that the implementation never raises - * TLB conflict aborts for BBML2 mapping granularity changes. - */ - static const struct midr_range supports_bbml2_noabort_list[] = { + /* CPUs that support BBML3 but dont advertise through ID_AA64MMFR2_EL1 */ + static const struct midr_range supports_bbml3_list[] = { MIDR_REV_RANGE(MIDR_CORTEX_X4, 0, 3, 0xf), MIDR_REV_RANGE(MIDR_NEOVERSE_V3, 0, 2, 0xf), {} }; - /* Does our cpu guarantee to never raise TLB conflict aborts? */ - if (!is_midr_in_range_list(supports_bbml2_noabort_list)) + if (!is_midr_in_range_list(supports_bbml3_list)) return false; /* @@ -2169,9 +2157,9 @@ bool cpu_supports_bbml2_noabort(void) return true; } -static bool has_bbml2_noabort(const struct arm64_cpu_capabilities *caps, int scope) +static bool has_bbml3(const struct arm64_cpu_capabilities *caps, int scope) { - return cpu_supports_bbml2_noabort(); + return cpu_supports_bbml3(); } #ifdef CONFIG_ARM64_PAN @@ -3213,10 +3201,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP) }, { - .desc = "BBM Level 2 without TLB conflict abort", - .capability = ARM64_HAS_BBML2_NOABORT, + .desc = "BBM Level 3", + .capability = ARM64_HAS_BBML3, .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, - .matches = has_bbml2_noabort, + .matches = has_bbml3, }, { .desc = "Non-maskable Interrupts present", diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 03a9f09db7e1..bf6de7de1e25 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -69,7 +69,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, } /* - * On eliding the __tlb_flush_range() under BBML2+noabort: + * On eliding the __tlb_flush_range() under BBML3: * * NOTE: Instead of using N=16 as the contiguous block length, we use * N=4 for clarity. @@ -115,7 +115,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * contiguous TLB entry, which is a micro-optimisation opportunity, * but does not affect correctness. * - * In the BBML2 case, the change is avoiding the intermediate tlbi+dsb. + * In the BBML3 case, the change is avoiding the intermediate tlbi+dsb. * This means a few things, but notably other PEs will still "see" any * stale cached TLB entries. This could lead to a "contiguous bit * misprogramming" issue until the final tlbi+dsb of the changed page, @@ -138,21 +138,16 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * are present, and a write is made to this address, do we fault or * is the write permitted (via amalgamation)? * - * The relevant Arm ARM DDI 0487L.a requirements are RNGLXZ and RJQQTC, - * and together state that when BBML1 or BBML2 are implemented, either - * a TLB conflict abort is raised (which we expressly forbid), or will - * "produce an OA, access permissions, and memory attributes that are - * consistent with any of the programmed translation table values". - * - * That is to say, will either raise a TLB conflict, or produce one of - * the cached TLB entries, but never amalgamate. + * With BBML3 implemented, no TLB conflict abort is raised and the OA, + * access permissions and memory attributes produced is one of the cached + * TLB entries, but never amalgamate. * * Thus, as the page tables are only considered "consistent" after * the final tlbi+dsb (which evicts both the single stale (RW,n) TLB * entry as well as the new contiguous (RO,c) TLB entry), omitting the * initial tlbi+dsb is correct. * - * It is also important to note that at the end of the BBML2 folding + * It is also important to note that at the end of the BBML3 folding * case, we are still left with potentially all N TLB entries still * cached (the N-1 non-contiguous ptes, and the single contiguous * block). However, over time, natural TLB pressure will cause the @@ -194,7 +189,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * * |____| <--- tlbi + dsb * - * For BBML2, we again remove the intermediate tlbi+dsb. Here, there + * For BBML3, we again remove the intermediate tlbi+dsb. Here, there * are no issues, as the final tlbi+dsb covering the changed page is * guaranteed to remove the original large contiguous (RW,c) TLB entry, * as well as the intermediate (RW,n) TLB entry; the next access will @@ -204,7 +199,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * regardless. */ - if (!system_supports_bbml2_noabort()) + if (!system_supports_bbml3()) __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3); __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 8b0bcecee433..0c330b5aa8ea 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -657,18 +657,18 @@ static int split_kernel_leaf_mapping_locked(unsigned long addr) static inline bool force_pte_mapping(void) { - const bool bbml2 = system_capabilities_finalized() ? - system_supports_bbml2_noabort() : cpu_supports_bbml2_noabort(); + const bool bbml3 = system_capabilities_finalized() ? + system_supports_bbml3() : cpu_supports_bbml3(); if (debug_pagealloc_enabled()) return true; - if (bbml2) + if (bbml3) return false; return rodata_full || arm64_kfence_can_set_direct_map() || is_realm_world(); } static DEFINE_MUTEX(pgtable_split_lock); -static bool linear_map_requires_bbml2; +static bool linear_map_requires_bbml3; int split_kernel_leaf_mapping(unsigned long start, unsigned long end) { @@ -681,15 +681,15 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end) * always pte-mapped), we must not go any further because taking the * mutex below may sleep. Do not call force_pte_mapping() here because * it could return a confusing result if called from a secondary cpu - * prior to finalizing caps. Instead, linear_map_requires_bbml2 gives us + * prior to finalizing caps. Instead, linear_map_requires_bbml3 gives us * what we need. */ - if (!linear_map_requires_bbml2 || is_kfence_address((void *)start)) + if (!linear_map_requires_bbml3 || is_kfence_address((void *)start)) return 0; - if (!system_supports_bbml2_noabort()) { + if (!system_supports_bbml3()) { /* - * !BBML2_NOABORT systems should not be trying to change + * BBML3 systems should not be trying to change * permissions on anything that is not pte-mapped in the first * place. Just return early and let the permission change code * raise a warning if not already pte-mapped. @@ -706,8 +706,8 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end) /* * Boot-time: Started secondary cpus but don't know if they - * support BBML2_NOABORT yet. Can't allow splitting in this - * window in case they don't. + * support BBML3 yet. Can't allow splitting in this window + * in case they don't. */ if (WARN_ON(num_online_cpus() > 1)) return -EBUSY; @@ -812,11 +812,11 @@ static int range_split_to_ptes(unsigned long start, unsigned long end, gfp_t gfp return ret; } -u32 idmap_kpti_bbml2_flag; +u32 idmap_kpti_bbml3_flag; -void __init init_idmap_kpti_bbml2_flag(void) +void __init init_idmap_kpti_bbml3_flag(void) { - WRITE_ONCE(idmap_kpti_bbml2_flag, 1); + WRITE_ONCE(idmap_kpti_bbml3_flag, 1); /* Must be visible to other CPUs before stop_machine() is called. */ smp_mb(); } @@ -825,7 +825,7 @@ static int __init linear_map_split_to_ptes(void *__unused) { /* * Repainting the linear map must be done by CPU0 (the boot CPU) because - * that's the only CPU that we know supports BBML2. The other CPUs will + * that's the only CPU that we know supports BBML3. The other CPUs will * be held in a waiting area with the idmap active. */ if (!smp_processor_id()) { @@ -838,7 +838,7 @@ static int __init linear_map_split_to_ptes(void *__unused) /* * Wait for all secondary CPUs to be put into the waiting area. */ - smp_cond_load_acquire(&idmap_kpti_bbml2_flag, VAL == num_online_cpus()); + smp_cond_load_acquire(&idmap_kpti_bbml3_flag, VAL == num_online_cpus()); /* * Walk all of the linear map [lstart, lend), except the kernel @@ -857,7 +857,7 @@ static int __init linear_map_split_to_ptes(void *__unused) * Relies on dsb in flush_tlb_kernel_range() to avoid reordering * before any page table split operations. */ - WRITE_ONCE(idmap_kpti_bbml2_flag, 0); + WRITE_ONCE(idmap_kpti_bbml3_flag, 0); } else { typedef void (wait_split_fn)(void); extern wait_split_fn wait_linear_map_split_to_ptes; @@ -866,7 +866,7 @@ static int __init linear_map_split_to_ptes(void *__unused) wait_fn = (void *)__pa_symbol(wait_linear_map_split_to_ptes); /* - * At least one secondary CPU doesn't support BBML2 so cannot + * At least one secondary CPU doesn't support BBML3 so cannot * tolerate the size of the live mappings changing. So have the * secondary CPUs wait for the boot CPU to make the changes * with the idmap active and init_mm inactive. @@ -881,8 +881,8 @@ static int __init linear_map_split_to_ptes(void *__unused) void __init linear_map_maybe_split_to_ptes(void) { - if (linear_map_requires_bbml2 && !cpus_have_cap(ARM64_HAS_BBML2_NOABORT)) { - init_idmap_kpti_bbml2_flag(); + if (linear_map_requires_bbml3 && !cpus_have_cap(ARM64_HAS_BBML3)) { + init_idmap_kpti_bbml3_flag(); stop_machine(linear_map_split_to_ptes, NULL, cpu_online_mask); } } @@ -1024,7 +1024,7 @@ bool arch_kfence_init_pool(void) mutex_unlock(&pgtable_split_lock); /* - * Since the system supports bbml2_noabort, tlb invalidation is not + * Since the system supports bbml3, tlb invalidation is not * required here; the pgtable mappings have been split to pte but larger * entries may safely linger in the TLB. */ @@ -1059,7 +1059,7 @@ static void __init map_mem(pgd_t *pgdp) early_kfence_pool = arm64_kfence_alloc_pool(); - linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map(); + linear_map_requires_bbml3 = !force_pte_mapping() && can_set_direct_map(); if (force_pte_mapping() || is_virtcca_cvm_world()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; @@ -1253,14 +1253,14 @@ static void __init create_idmap(void) __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX, early_pgtable_alloc, 0); - if (linear_map_requires_bbml2 || IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) { - u64 pa = __pa_symbol(&idmap_kpti_bbml2_flag); + if (linear_map_requires_bbml3 || IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) { + u64 pa = __pa_symbol(&idmap_kpti_bbml3_flag); /* * The KPTI G-to-nG conversion code needs a read-write mapping * of its synchronization flag in the ID map. This is also used * when splitting the linear map to ptes if a secondary CPU - * doesn't support bbml2. + * doesn't support bbml3. */ __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL, early_pgtable_alloc, 0); diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index bc8022c84b14..8582a53f8419 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -268,7 +268,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) mov x5, x3 // preserve temp_pte arg mrs swapper_ttb, ttbr1_el1 - adr_l flag_ptr, idmap_kpti_bbml2_flag + adr_l flag_ptr, idmap_kpti_bbml3_flag cbnz cpu, __idmap_kpti_secondary @@ -377,7 +377,7 @@ SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes) flag_ptr .req x4 mrs swapper_ttb, ttbr1_el1 - adr_l flag_ptr, idmap_kpti_bbml2_flag + adr_l flag_ptr, idmap_kpti_bbml3_flag __idmap_cpu_set_reserved_ttbr1 x16, x17 scondary_cpu_wait: diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 7956030820b6..3c63a5080b83 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -116,7 +116,7 @@ HAS_HW_XCALL_XINT WORKAROUND_PHYTIUM_FT3386 HAS_COPY_OPT HAS_LSUI -HAS_BBML2_NOABORT +HAS_BBML3 KABI_RESERVE_10 KABI_RESERVE_11 KABI_RESERVE_12 -- 2.43.0