From: Liao Chang <liaochang1@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/release-management/issues/ID5CMS -------------------------------- Use the symbol el0t_64_svc_entry as the dispatcher of svc exception handler: - el0_slow_syscall: use kernel_entry and ret_to_user to do exception context switch, additionally use el0_svc to invoke syscall functions. - el0_fast_syscall: use hw_xcall_save_base_regs and hw_xcall_restore_base_regs to do low-overhead context switch, additionally use el0_svc to invoke syscall functions. - el0_xcall_syscall: use hw_xcall_save_base_regs and hw_xcall_restore_base_regs to do low-overhead context switch, additionally use el0_xcall to invoke dynamically load syscall functions. Signed-off-by: Liao Chang <liaochang1@huawei.com> --- arch/arm64/include/asm/exception.h | 1 - arch/arm64/kernel/entry-common.c | 26 ++++++++------ arch/arm64/kernel/entry.S | 3 +- arch/arm64/kernel/xcall/entry.S | 55 +++++++++++++++++++++++++++--- arch/arm64/kernel/xcall/xcall.c | 6 ++++ 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index 94338104a18c..1d87f724719d 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -83,6 +83,5 @@ void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigne #ifdef CONFIG_ACTLR_XCALL_XINT asmlinkage void el0t_64_xint_handler(struct pt_regs *regs); -asmlinkage void el0t_64_xcall_handler(struct pt_regs *regs); #endif #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index f4a21c66856a..a405ccc185f1 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -207,7 +207,7 @@ static __always_inline void fast_enter_from_user_mode(struct pt_regs *regs) mte_disable_tco_entry(current); #endif } -#endif +#endif /* CONFIG_FAST_SYSCALL || CONFIG_FAST_IRQ */ /* * Handle IRQ/context state management when entering an NMI from user/kernel @@ -818,8 +818,8 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) } #ifdef CONFIG_FAST_SYSCALL -/* Copy from el0_sync */ -static void noinstr el0_xcall(struct pt_regs *regs) +/* dynamically load syscall handler */ +asmlinkage void noinstr el0_xcall_syscall(struct pt_regs *regs) { fast_enter_from_user_mode(regs); #ifndef CONFIG_SECURITY_FEATURE_BYPASS @@ -831,11 +831,21 @@ static void noinstr el0_xcall(struct pt_regs *regs) fast_exit_to_user_mode(regs); } -asmlinkage void noinstr el0t_64_fast_syscall_handler(struct pt_regs *regs) +/* low-overhead syscall handler */ +asmlinkage void noinstr el0_fast_syscall(struct pt_regs *regs) { - el0_xcall(regs); -} + fast_enter_from_user_mode(regs); +#ifndef CONFIG_SECURITY_FEATURE_BYPASS + cortex_a76_erratum_1463225_svc_handler(); #endif + fp_user_discard(); + local_daif_restore(DAIF_PROCCTX); + do_el0_svc(regs); + fast_exit_to_user_mode(regs); +} + +asmlinkage void noinstr el0_slow_syscall(struct pt_regs *regs) __alias(el0_svc); +#endif /* CONFIG_FAST_SYSCALL */ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) { @@ -1052,10 +1062,6 @@ UNHANDLED(el0t, 32, error) #endif /* CONFIG_AARCH32_EL0 */ #ifdef CONFIG_ACTLR_XCALL_XINT -asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs) -{ - el0_xcall(regs); -} asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) { el0_interrupt(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 5648a3119f90..cd39e17af8b0 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -685,7 +685,7 @@ SYM_CODE_END(__bad_stack) kernel_entry 0, 64 #endif mov x0, sp - bl el0t_64_fast_syscall_handler + bl el0_fast_syscall #ifdef CONFIG_SECURITY_FEATURE_BYPASS kernel_exit 0, xcall #else @@ -797,7 +797,6 @@ SYM_CODE_END(el\el\ht\()_\regsize\()_\label) entry_handler 0, t, 64, error #ifdef CONFIG_ACTLR_XCALL_XINT - entry_handler 0, t, 64, xcall entry_handler 0, t, 64, xint #endif entry_handler 0, t, 32, sync diff --git a/arch/arm64/kernel/xcall/entry.S b/arch/arm64/kernel/xcall/entry.S index 7b75e8651a2a..abd5f97d5525 100644 --- a/arch/arm64/kernel/xcall/entry.S +++ b/arch/arm64/kernel/xcall/entry.S @@ -151,12 +151,55 @@ alternative_else_nop_endif sb .endm /* .macro hw_xcal_restore_base_regs */ -SYM_CODE_START(xcall_entry) +#define __NR_fast_syscalls 512 + +SYM_CODE_START_LOCAL(el0t_64_svc_entry) + /* Hijack SVC to dynamically load syscalls via '/proc/xcall/comm' */ + ldr x20, [sp, #S_SYSCALLNO] // ESR.bits[25,18] + cmp x20, 0 + bne el0t_64_xcall_entry + + /* Hijack SVC to low overhead syscalls via '/prox/[pid]/xcall' */ + ldr_this_cpu x21, __cpu_fast_syscall, x20 // per_cpu table + and x20, x8, #__NR_fast_syscalls - 1 // trunk syscno less than 512 + ldrb w20, [x21, x20] // memory overhead is 512(B) + cmp x20, 0 + bne el0t_fast_syscall + + ldp x20, x21, [sp, #16 * 10] + kernel_entry 0, 64 + mov x0, sp + bl el0_slow_syscall + b ret_to_user +SYM_INNER_LABEL(el0t_64_xcall_entry, SYM_L_GLOBAL) + lsr x20, x20, #4 + adr x21, .xcall_func_table + ldr w20, [x21, x20, lsl #2] + add x20, x20, x21 + br x20 + /* ISS==0F~FF: Entry to optimized and customized syscalls + */ +.xcall_func_table: + .rept 15 + .word el0t_xcall_syscall - .xcall_func_table + .endr +SYM_CODE_END(el0t_64_svc_entry) + +SYM_CODE_START_LOCAL(el0t_xcall_syscall) + ldp x20, x21, [sp, #16 * 10] hw_xcall_save_base_regs mov x0, sp - bl el0t_64_xcall_handler + bl el0_xcall_syscall hw_xcal_restore_base_regs -SYM_CODE_END(xcall_entry) +SYM_CODE_END(el0t_xcall_syscall) + +SYM_CODE_START_LOCAL(el0t_fast_syscall) + ldp x20, x21, [sp, #16 * 10] + hw_xcall_save_base_regs + mov x0, sp + bl el0_fast_syscall + hw_xcal_restore_base_regs +SYM_CODE_END(el0t_fast_syscall) .macro xcall_ventry .align 7 @@ -170,6 +213,10 @@ SYM_CODE_END(xcall_entry) msr tpidrro_el0, xzr .Lskip_tramp_vectors_cleanup\@: sub sp, sp, #PT_REGS_SIZE - b xcall_entry + stp x20, x21, [sp, #16 * 10] + /* Decode ESR.ICC bits[25,18] for use later */ + mrs x21, esr_el1 + ubfx w20, w21, #16, #8 + b el0t_64_xcall_entry .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm diff --git a/arch/arm64/kernel/xcall/xcall.c b/arch/arm64/kernel/xcall/xcall.c index 31072c0402f4..c0907c01c9c7 100644 --- a/arch/arm64/kernel/xcall/xcall.c +++ b/arch/arm64/kernel/xcall/xcall.c @@ -37,3 +37,9 @@ void xcall_task_free(struct task_struct *p) if (static_branch_unlikely(&xcall_enable)) kfree(p->xinfo); } + +#define __NR_fast_syscalls 512 +static u8 fast_syscall_enabled[__NR_fast_syscalls + 1] = { + [0 ... __NR_fast_syscalls] = 0, +}; +asmlinkage DEFINE_PER_CPU(u8*, __cpu_fast_syscall) = fast_syscall_enabled; -- 2.34.1