From: Mark Brown broonie@kernel.org
mainline inclusion from mainline-v5.5-rc3 commit 09e3c22a86f6889db0e93fb29d9255081a126f64 category:feature bugzilla:NA CVE:NA
-------------------
Refactor the code which checks to see if we need to use non-global mappings to use a variable instead of checking with the CPU capabilities each time, doing the initial check for KPTI early in boot before we start allocating memory so we still avoid transitioning to non-global mappings in common cases.
Since this variable always matches our decision about non-global mappings this means we can also combine arm64_kernel_use_ng_mappings() and arm64_unmap_kernel_at_el0() into a single function, the variable simply stores the result and the decision code is elsewhere. We could just have the users check the variable directly but having a function makes it clear that these uses are read-only.
The result is that we simplify the code a bit and reduces the amount of code executed at runtime.
Signed-off-by: Mark Brown broonie@kernel.org Reviewed-by: Suzuki K Poulose suzuki.poulose@arm.com Signed-off-by: Will Deacon will@kernel.org
Conflicts: arch/arm64/kernel/cpufeature.c arch/arm64/include/asm/mmu.h
Signed-off-by: Chen Jun chenjun102@huawei.com Reviewed-by: Hanjun Guo guohanjun@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- arch/arm64/include/asm/mmu.h | 67 ++-------------------------------- arch/arm64/kernel/cpufeature.c | 37 +++++++++++++++++-- arch/arm64/kernel/setup.c | 7 ++++ 3 files changed, 45 insertions(+), 66 deletions(-)
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index dc520b4e6b88..069ced02c8fa 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -50,71 +50,11 @@ extern int res_mem_count; */ #define ASID(mm) ((mm)->context.id.counter & 0xffff)
-static inline bool arm64_kernel_unmapped_at_el0(void) -{ - return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) && - cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0); -} +extern bool arm64_use_ng_mappings;
-/* - * This check is triggered during the early boot before the cpufeature - * is initialised. Checking the status on the local CPU allows the boot - * CPU to detect the need for non-global mappings and thus avoiding a - * pagetable re-write after all the CPUs are booted. This check will be - * anyway run on individual CPUs, allowing us to get the consistent - * state once the SMP CPUs are up and thus make the switch to non-global - * mappings if required. - */ -static inline bool kaslr_requires_kpti(void) -{ - u64 ftr; - - if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE)) - return false; - - /* - * E0PD does a similar job to KPTI so can be used instead - * where available. - */ - if (IS_ENABLED(CONFIG_ARM64_E0PD)) { - ftr = read_sysreg_s(SYS_ID_AA64MMFR2_EL1); - if ((ftr >> ID_AA64MMFR2_E0PD_SHIFT) & 0xf) - return false; - } - - return kaslr_offset() > 0; -} - -static inline bool arm64_kernel_use_ng_mappings(void) +static inline bool arm64_kernel_unmapped_at_el0(void) { - /* What's a kpti? Use global mappings if we don't know. */ - if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) - return false; - - /* - * Note: this function is called before the CPU capabilities have - * been configured, so our early mappings will be global. If we - * later determine that kpti is required, then - * kpti_install_ng_mappings() will make them non-global. - */ - if (arm64_kernel_unmapped_at_el0()) - return true; - - /* - * Once we are far enough into boot for capabilities to be - * ready we will have confirmed if we are using non-global - * mappings so don't need to consider anything else here. - */ - if (static_branch_likely(&arm64_const_caps_ready)) - return false; - - /* - * KASLR is enabled so we're going to be enabling kpti on non-broken - * CPUs regardless of their susceptibility to Meltdown. Rather - * than force everybody to go through the G -> nG dance later on, - * just put down non-global mappings from the beginning - */ - return kaslr_requires_kpti(); + return arm64_use_ng_mappings; }
typedef void (*bp_hardening_cb_t)(void); @@ -167,6 +107,7 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, pgprot_t prot, bool page_mappings_only); extern void *fixmap_remap_fdt(phys_addr_t dt_phys); extern void mark_linear_text_alias_ro(void); +extern bool kaslr_requires_kpti(void);
#endif /* !__ASSEMBLY__ */ #endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 7bbd8809e15a..191cb8beddda 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -56,6 +56,9 @@ static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM6 /* Need also bit for ARM64_CB_PATCH */ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
+bool arm64_use_ng_mappings = false; +EXPORT_SYMBOL(arm64_use_ng_mappings); + /* * Flag to indicate if we have computed the system wide * capabilities based on the boot time active CPUs. This @@ -1020,6 +1023,35 @@ static bool has_cache_dic(const struct arm64_cpu_capabilities *entry, return ctr & BIT(CTR_DIC_SHIFT); }
+/* + * This check is triggered during the early boot before the cpufeature + * is initialised. Checking the status on the local CPU allows the boot + * CPU to detect the need for non-global mappings and thus avoiding a + * pagetable re-write after all the CPUs are booted. This check will be + * anyway run on individual CPUs, allowing us to get the consistent + * state once the SMP CPUs are up and thus make the switch to non-global + * mappings if required. + */ +bool kaslr_requires_kpti(void) +{ + u64 ftr; + + if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE)) + return false; + + /* + * E0PD does a similar job to KPTI so can be used instead + * where available. + */ + if (IS_ENABLED(CONFIG_ARM64_E0PD)) { + ftr = read_sysreg_s(SYS_ID_AA64MMFR2_EL1); + if ((ftr >> ID_AA64MMFR2_E0PD_SHIFT) & 0xf) + return false; + } + + return kaslr_offset() > 0; +} + static bool __meltdown_safe = true; static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
@@ -1120,10 +1152,9 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) extern kpti_remap_fn idmap_kpti_install_ng_mappings; kpti_remap_fn *remap_fn;
- static bool kpti_applied = false; int cpu = smp_processor_id();
- if (kpti_applied) + if (arm64_use_ng_mappings) return;
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); @@ -1133,7 +1164,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) cpu_uninstall_idmap();
if (!cpu) - kpti_applied = true; + arm64_use_ng_mappings = true;
return; } diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index b3569d16af19..aa499eb2a923 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -308,6 +308,13 @@ void __init setup_arch(char **cmdline_p)
*cmdline_p = boot_command_line;
+ /* + * If know now we are going to need KPTI then use non-global + * mappings from the start, avoiding the cost of rewriting + * everything later. + */ + arm64_use_ng_mappings = kaslr_requires_kpti(); + early_fixmap_init(); early_ioremap_init();