hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/release-management/issues/IBV2E4 -------------------------------- Support hardware Xcall and Xint. Support hardware xcall. Hardware xcall provides a separate entry for el0 syscall handling, so we can use it to customize and respond to system call relatively quickly. Hardware xint provides a separate entry for interrupt handling, so we can use it to customize and respond to interrupts relatively quickly. When enter idle state, the actlr register will reset to an architecturally UNKNOWN value on warm reset. So save and restore actlr_el1 and actlr_el2 register value when enter and exit idle state. Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/arm64/include/asm/actlr.h | 15 ++++++ arch/arm64/include/asm/cpufeature.h | 8 +++ arch/arm64/include/asm/vectors.h | 1 + arch/arm64/kernel/cpufeature.c | 81 +++++++++++++++++++++++++++++ arch/arm64/kernel/entry-common.c | 15 +++++- arch/arm64/kernel/entry.S | 52 +++++++++++++++++- arch/arm64/kernel/idle.c | 40 ++++++++++++++ arch/arm64/tools/cpucaps | 2 +- 8 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 arch/arm64/include/asm/actlr.h diff --git a/arch/arm64/include/asm/actlr.h b/arch/arm64/include/asm/actlr.h new file mode 100644 index 000000000000..3a44b6876e81 --- /dev/null +++ b/arch/arm64/include/asm/actlr.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025 - Huawei Ltd. + */ + +#ifndef __ASM_ACTLR_H +#define __ASM_ACTLR_H + +#define ACTLR_ELx_XCALL_SHIFT 20 +#define ACTLR_ELx_XCALL (UL(1) << ACTLR_ELx_XCALL_SHIFT) + +#define ACTLR_ELx_XINT_SHIFT (21) +#define ACTLR_ELx_XINT (UL(1) << ACTLR_ELx_XINT_SHIFT) + +#endif /* __ASM_ACTLR_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index dd758b04fd54..88cdf27fe701 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -831,6 +831,14 @@ static inline bool system_has_full_ptr_auth(void) return system_supports_address_auth() && system_supports_generic_auth(); } +#ifdef CONFIG_ACTLR_XCALL_XINT +static __always_inline bool system_uses_xcall_xint(void) +{ + return IS_ENABLED(CONFIG_ACTLR_XCALL_XINT) && + cpus_have_const_cap(ARM64_HAS_HW_XCALL_XINT); +} +#endif + static __always_inline bool system_uses_irq_prio_masking(void) { return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h index bc9a2145f419..0addb4bd2b3d 100644 --- a/arch/arm64/include/asm/vectors.h +++ b/arch/arm64/include/asm/vectors.h @@ -11,6 +11,7 @@ #include <asm/fixmap.h> extern char vectors[]; +extern char vectors_xcall_xint[]; extern char tramp_vectors[]; extern char __bp_harden_el1_vectors[]; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 5c3d206785bd..b13858668877 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -76,6 +76,7 @@ #include <linux/kasan.h> #include <linux/percpu.h> +#include <asm/actlr.h> #include <asm/cpu.h> #include <asm/cpufeature.h> #include <asm/cpu_ops.h> @@ -2466,6 +2467,77 @@ static bool has_xint_support(const struct arm64_cpu_capabilities *entry, int __u } #endif +#ifdef CONFIG_ACTLR_XCALL_XINT +static bool has_arch_xcall_xint_support(const struct arm64_cpu_capabilities *entry, int scope) +{ + /* List of CPUs that support Xcall/Xint */ + static const struct midr_range xcall_xint_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_HISI_HIP12), + { /* sentinel */ } + }; + + return is_midr_in_range_list(read_cpuid_id(), xcall_xint_cpus); +} + +static void enable_xcall_xint_vectors(void) +{ + /* + * Upon CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY is enabled, + * the vbar_el1 is set to the vectors starts from __bp_harden_el1_vectors. + * Kernel will jump to the xcall/xint vectors from the trampoline vector + * defined in the macro tramp_ventry. + */ + if (__this_cpu_read(this_cpu_vector) != vectors) + return; + + /* + * Upon KAISER is enabled, the vbar_el1 is set to the vectors starts + * from tramp_vectors. Kernel will jump to the vectors_xcall_xint from + * the trampoline vector defined in the macro tramp_ventry. + */ + if (arm64_kernel_unmapped_at_el0()) + return; + + /* + * If neither KAISER or BHB_MITIGATION is enabled, then we switch + * the vbar_el1 from the default vectors to the xcall/xint vectors + * at once. + */ + write_sysreg(vectors_xcall_xint, vbar_el1); + isb(); +} + +static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__unused) +{ + int cpu = smp_processor_id(); + u64 actlr_el1, actlr_el2; + u64 el; + + el = read_sysreg(CurrentEL); + if (el == CurrentEL_EL2) { + /* + * Enable EL2 trap when access ACTLR_EL1 in guest kernel. + */ + write_sysreg_s(read_sysreg_s(SYS_HCR_EL2) | HCR_TACR, SYS_HCR_EL2); + actlr_el2 = read_sysreg(actlr_el2); + actlr_el2 |= (ACTLR_ELx_XINT | ACTLR_ELx_XCALL); + write_sysreg(actlr_el2, actlr_el2); + isb(); + actlr_el2 = read_sysreg(actlr_el2); + pr_info("actlr_el2: %llx, cpu:%d\n", actlr_el2, cpu); + } else { + actlr_el1 = read_sysreg(actlr_el1); + actlr_el1 |= (ACTLR_ELx_XINT | ACTLR_ELx_XCALL); + write_sysreg(actlr_el1, actlr_el1); + isb(); + actlr_el1 = read_sysreg(actlr_el1); + pr_info("actlr_el1: %llx, cpu:%d\n", actlr_el1, cpu); + } + + enable_xcall_xint_vectors(); +} +#endif + static const struct arm64_cpu_capabilities arm64_features[] = { { .capability = ARM64_ALWAYS_BOOT, @@ -3031,6 +3103,15 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_ls64_v, ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LS64, LS64_V) }, +#endif +#ifdef CONFIG_ACTLR_XCALL_XINT + { + .desc = "Hardware Xcall and Xint Support", + .capability = ARM64_HAS_HW_XCALL_XINT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_arch_xcall_xint_support, + .cpu_enable = cpu_enable_arch_xcall_xint, + }, #endif {}, }; diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index b7cc01130ff5..a90231346751 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -609,7 +609,7 @@ static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, } -asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) +asmlinkage void noinstr el0t_64_fast_irq_handler(struct pt_regs *regs) { el0_xint(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq); } @@ -831,7 +831,7 @@ static void noinstr el0_xcall(struct pt_regs *regs) fast_exit_to_user_mode(regs); } -asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs) +asmlinkage void noinstr el0t_64_fast_syscall_handler(struct pt_regs *regs) { el0_xcall(regs); } @@ -1051,6 +1051,17 @@ UNHANDLED(el0t, 32, fiq) UNHANDLED(el0t, 32, error) #endif /* CONFIG_AARCH32_EL0 */ +#ifdef CONFIG_ACTLR_XCALL_XINT +asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs) +{ + el0_svc(regs); +} +asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) +{ + el0_interrupt(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq); +} +#endif + #ifdef CONFIG_VMAP_STACK asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) { diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index da3809632f0f..801a521613ad 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -48,7 +48,13 @@ mrs x30, tpidrro_el0 msr tpidrro_el0, xzr .else + alternative_if ARM64_HAS_HW_XCALL_XINT + mrs x30, tpidrro_el0 + msr tpidrro_el0, xzr + alternative_else mov x30, xzr + nop + alternative_endif .endif .Lskip_tramp_vectors_cleanup\@: .endif @@ -570,6 +576,29 @@ SYM_CODE_START(vectors) kernel_ventry 0, t, 32, error // Error 32-bit EL0 SYM_CODE_END(vectors) + .align 11 +SYM_CODE_START(vectors_xcall_xint) + kernel_ventry 1, t, 64, sync // Synchronous EL1t + kernel_ventry 1, t, 64, irq // IRQ EL1t + kernel_ventry 1, t, 64, fiq // FIQ EL1t + kernel_ventry 1, t, 64, error // Error EL1t + + kernel_ventry 1, h, 64, sync // Synchronous EL1h + kernel_ventry 1, h, 64, irq // IRQ EL1h + kernel_ventry 1, h, 64, fiq // FIQ EL1h + kernel_ventry 1, h, 64, error // Error EL1h + + kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0 + kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0 + kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 + kernel_ventry 0, t, 64, error // Error 64-bit EL0 + + kernel_ventry 0, t, 64, xcall // XCALL synchronous 64-bit EL0 + kernel_ventry 0, t, 64, xint // XINT 64-bit EL0 + kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 + kernel_ventry 0, t, 32, error // Error 32-bit EL0 +SYM_CODE_END(vectors_xcall_xint) + #ifdef CONFIG_VMAP_STACK SYM_CODE_START_LOCAL(__bad_stack) /* @@ -648,7 +677,7 @@ SYM_CODE_END(__bad_stack) kernel_entry 0, 64 #endif mov x0, sp - bl el0t_64_xcall_handler + bl el0t_64_fast_syscall_handler #ifdef CONFIG_SECURITY_FEATURE_BYPASS kernel_exit 0, xcall #else @@ -696,7 +725,7 @@ SYM_CODE_END(__bad_stack) kernel_entry 0, 64 #endif mov x0, sp - bl el0t_64_xint_handler + bl el0t_64_fast_irq_handler #ifdef CONFIG_SECURITY_FEATURE_BYPASS kernel_exit 0, xint #else @@ -759,6 +788,10 @@ SYM_CODE_END(el\el\ht\()_\regsize\()_\label) entry_handler 0, t, 64, fiq entry_handler 0, t, 64, error +#ifdef CONFIG_ACTLR_XCALL_XINT + entry_handler 0, t, 64, xcall + entry_handler 0, t, 64, xint +#endif entry_handler 0, t, 32, sync entry_handler 0, t, 32, irq entry_handler 0, t, 32, fiq @@ -849,6 +882,12 @@ alternative_else_nop_endif msr tpidrro_el0, x30 // Restored in kernel_ventry .endif + .if \regsize == 32 + alternative_if ARM64_HAS_HW_XCALL_XINT + msr tpidrro_el0, x30 // Restored in kernel_ventry + alternative_else_nop_endif + .endif + .if \bhb == BHB_MITIGATION_LOOP /* * This sequence must appear before the first indirect branch. i.e. the @@ -873,7 +912,12 @@ alternative_else_nop_endif 2: tramp_map_kernel x30 alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 +alternative_if_not ARM64_HAS_HW_XCALL_XINT tramp_data_read_var x30, vectors + b 3f +alternative_else_nop_endif + tramp_data_read_var x30, vectors_xcall_xint +3: alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM prfm plil1strm, [x30, #(1b - \vector_start)] alternative_else_nop_endif @@ -881,7 +925,11 @@ alternative_else_nop_endif msr vbar_el1, x30 isb .else +alternative_if_not ARM64_HAS_HW_XCALL_XINT adr_l x30, vectors +alternative_else + adr_l x30, vectors_xcall_xint +alternative_endif .endif // \kpti == 1 .if \bhb == BHB_MITIGATION_FW diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 3a0b59aa12e2..46a40b693da8 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -44,3 +44,43 @@ void noinstr arch_cpu_idle(void) cpu_do_idle(); } EXPORT_SYMBOL_GPL(arch_cpu_idle); + +#ifdef CONFIG_ACTLR_XCALL_XINT +struct arm_cpuidle_xcall_xint_context { + unsigned long actlr_el1; + unsigned long actlr_el2; +}; + +DEFINE_PER_CPU_ALIGNED(struct arm_cpuidle_xcall_xint_context, contexts); + +void arch_cpu_idle_enter(void) +{ + struct arm_cpuidle_xcall_xint_context *context; + + if (!system_uses_xcall_xint()) + return; + + context = &get_cpu_var(contexts); + context->actlr_el1 = read_sysreg(actlr_el1); + if (read_sysreg(CurrentEL) == CurrentEL_EL2) + context->actlr_el2 = read_sysreg(actlr_el2); + put_cpu_var(contexts); +} + +void arch_cpu_idle_exit(void) +{ + struct arm_cpuidle_xcall_xint_context *context; + + if (!system_uses_xcall_xint()) + return; + + context = &get_cpu_var(contexts); + write_sysreg(context->actlr_el1, actlr_el1); + if (read_sysreg(CurrentEL) == CurrentEL_EL2) + write_sysreg(context->actlr_el2, actlr_el2); + put_cpu_var(contexts); +} +#else +void arch_cpu_idle_enter(void) {} +void arch_cpu_idle_exit(void) {} +#endif diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 27d93050e5da..ae0268822d61 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -112,7 +112,7 @@ HAS_XCALL HAS_XINT HAS_LS64 HAS_LS64_V -KABI_RESERVE_5 +HAS_HW_XCALL_XINT KABI_RESERVE_6 KABI_RESERVE_7 KABI_RESERVE_8 -- 2.34.1