From: Liao Chang <liaochang1@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/release-management/issues/ID5CMS -------------------------------- Use the symbol el0t_64_svc_entry as the dispatcher of svc exception handler: - el0_slow_syscall: use kernel_entry and ret_to_user to do exception context switch, additionally use el0_svc to invoke syscall functions. - el0_fast_syscall: use hw_xcall_save_base_regs and hw_xcall_restore_base_regs to do low-overhead context switch, additionally use el0_svc to invoke syscall functions. - el0_xcall_syscall: use hw_xcall_save_base_regs and hw_xcall_restore_base_regs to do low-overhead context switch, additionally use el0_xcall to invoke dynamically load syscall functions. Signed-off-by: Liao Chang <liaochang1@huawei.com> Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/arm64/include/asm/exception.h | 1 - arch/arm64/include/asm/xcall.h | 3 +- arch/arm64/kernel/entry-common.c | 26 +++--- arch/arm64/kernel/entry.S | 86 +++---------------- arch/arm64/kernel/process.c | 5 ++ arch/arm64/kernel/xcall/entry.S | 127 +++++++++++++++++++++++++++-- arch/arm64/kernel/xcall/xcall.c | 16 +++- fs/proc/proc_xcall.c | 80 ++++++++---------- 8 files changed, 206 insertions(+), 138 deletions(-) diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index 94338104a18c..1d87f724719d 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -83,6 +83,5 @@ void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigne #ifdef CONFIG_ACTLR_XCALL_XINT asmlinkage void el0t_64_xint_handler(struct pt_regs *regs); -asmlinkage void el0t_64_xcall_handler(struct pt_regs *regs); #endif #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/xcall.h b/arch/arm64/include/asm/xcall.h index baf766e8c987..735a14870e4e 100644 --- a/arch/arm64/include/asm/xcall.h +++ b/arch/arm64/include/asm/xcall.h @@ -89,11 +89,12 @@ DECLARE_STATIC_KEY_FALSE(xcall_enable); struct xcall_info { /* Must be first! */ - DECLARE_BITMAP(xcall_enable, __NR_syscalls); + u8 xcall_enable[__NR_syscalls + 1]; }; #define TASK_XINFO(p) ((struct xcall_info *)p->xinfo) int xcall_init_task(struct task_struct *p, struct task_struct *orig); void xcall_task_free(struct task_struct *p); +void xcall_info_switch(struct task_struct *p); #endif /* __ASM_XCALL_H */ diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index f4a21c66856a..c72993bb4563 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -207,7 +207,7 @@ static __always_inline void fast_enter_from_user_mode(struct pt_regs *regs) mte_disable_tco_entry(current); #endif } -#endif +#endif /* CONFIG_FAST_SYSCALL || CONFIG_FAST_IRQ */ /* * Handle IRQ/context state management when entering an NMI from user/kernel @@ -818,8 +818,8 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) } #ifdef CONFIG_FAST_SYSCALL -/* Copy from el0_sync */ -static void noinstr el0_xcall(struct pt_regs *regs) +/* dynamically load syscall handler */ +asmlinkage void noinstr el0_xcall_syscall(struct pt_regs *regs) { fast_enter_from_user_mode(regs); #ifndef CONFIG_SECURITY_FEATURE_BYPASS @@ -831,11 +831,21 @@ static void noinstr el0_xcall(struct pt_regs *regs) fast_exit_to_user_mode(regs); } -asmlinkage void noinstr el0t_64_fast_syscall_handler(struct pt_regs *regs) +/* low-overhead syscall handler */ +asmlinkage void noinstr el0_fast_syscall(struct pt_regs *regs) { - el0_xcall(regs); -} + fast_enter_from_user_mode(regs); +#ifndef CONFIG_SECURITY_FEATURE_BYPASS + cortex_a76_erratum_1463225_svc_handler(); #endif + fp_user_discard(); + local_daif_restore(DAIF_PROCCTX); + do_el0_svc(regs); + fast_exit_to_user_mode(regs); +} + +asmlinkage void el0_slow_syscall(struct pt_regs *regs) __alias(el0_svc); +#endif /* CONFIG_FAST_SYSCALL */ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) { @@ -1052,10 +1062,6 @@ UNHANDLED(el0t, 32, error) #endif /* CONFIG_AARCH32_EL0 */ #ifdef CONFIG_ACTLR_XCALL_XINT -asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs) -{ - el0_xcall(regs); -} asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) { el0_interrupt(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 5648a3119f90..cceb4526745f 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -552,6 +552,10 @@ tsk .req x28 // current thread_info .text +#ifdef CONFIG_FAST_SYSCALL +#include "xcall/entry.S" +#endif + /* * Exception vectors. */ @@ -569,7 +573,11 @@ SYM_CODE_START(vectors) kernel_ventry 1, h, 64, fiq // FIQ EL1h kernel_ventry 1, h, 64, error // Error EL1h +#ifdef CONFIG_FAST_SYSCALL + sync_ventry // Synchronous 64-bit EL0 +#else kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0 +#endif kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0 kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 kernel_ventry 0, t, 64, error // Error 64-bit EL0 @@ -581,8 +589,6 @@ SYM_CODE_START(vectors) SYM_CODE_END(vectors) #ifdef CONFIG_ACTLR_XCALL_XINT -#include "xcall/entry.S" - .align 11 SYM_CODE_START(vectors_xcall_xint) kernel_ventry 1, t, 64, sync // Synchronous EL1t @@ -595,7 +601,11 @@ SYM_CODE_START(vectors_xcall_xint) kernel_ventry 1, h, 64, fiq // FIQ EL1h kernel_ventry 1, h, 64, error // Error EL1h +#ifdef CONFIG_FAST_SYSCALL + sync_ventry // Synchronous 64-bit EL0 +#else kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0 +#endif kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0 kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 kernel_ventry 0, t, 64, error // Error 64-bit EL0 @@ -605,7 +615,7 @@ SYM_CODE_START(vectors_xcall_xint) kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 kernel_ventry 0, t, 32, error // Error 32-bit EL0 SYM_CODE_END(vectors_xcall_xint) -#endif +#endif /* CONFIG_ACTLR_XCALL_XINT */ #ifdef CONFIG_VMAP_STACK SYM_CODE_START_LOCAL(__bad_stack) @@ -637,65 +647,6 @@ SYM_CODE_START_LOCAL(__bad_stack) SYM_CODE_END(__bad_stack) #endif /* CONFIG_VMAP_STACK */ -#ifdef CONFIG_FAST_SYSCALL - .macro check_esr_el1_ec_svc64 - /* Only support SVC64 for now */ - mrs x20, esr_el1 - lsr w20, w20, #ESR_ELx_EC_SHIFT - cmp x20, #ESR_ELx_EC_SVC64 - .endm - - .macro check_syscall_nr - cmp x8, __NR_syscalls - .endm - - .macro check_xcall_enable - /* x21 = task_struct->xinfo->xcall_enable */ - ldr_this_cpu x20, __entry_task, x21 - ldr x21, [x20, #TSK_XCALL] - /* x20 = sc_no / 8 */ - lsr x20, x8, 3 - ldr x21, [x21, x20] - /* x8 = sc_no % 8 */ - and x8, x8, 7 - mov x20, 1 - lsl x20, x20, x8 - and x21, x21, x20 - cmp x21, 0 - .endm - - .macro check_xcall_pre_kernel_entry - stp x20, x21, [sp, #0] - /* is ESR_ELx_EC_SVC64 */ - check_esr_el1_ec_svc64 - bne .Lskip_xcall\@ - /* x8 >= __NR_syscalls */ - check_syscall_nr - bhs .Lskip_xcall\@ - str x8, [sp, #16] - /* is xcall enabled */ - check_xcall_enable - ldr x8, [sp, #16] - beq .Lskip_xcall\@ - ldp x20, x21, [sp, #0] - /* do xcall */ -#ifdef CONFIG_SECURITY_FEATURE_BYPASS - kernel_entry 0, 64, xcall -#else - kernel_entry 0, 64 -#endif - mov x0, sp - bl el0t_64_fast_syscall_handler -#ifdef CONFIG_SECURITY_FEATURE_BYPASS - kernel_exit 0, xcall -#else - b ret_to_user -#endif -.Lskip_xcall\@: - ldp x20, x21, [sp, #0] - .endm -#endif - #ifdef CONFIG_FAST_IRQ .macro check_xint_pre_kernel_entry stp x0, x1, [sp, #0] @@ -748,16 +699,6 @@ SYM_CODE_END(__bad_stack) .macro entry_handler el:req, ht:req, regsize:req, label:req SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) -#ifdef CONFIG_FAST_SYSCALL - .if \el == 0 && \regsize == 64 && \label == sync - /* Only support el0 aarch64 sync exception */ - alternative_if_not ARM64_HAS_XCALL - b .Lret_to_kernel_entry\@ - alternative_else_nop_endif - check_xcall_pre_kernel_entry - .Lret_to_kernel_entry\@: - .endif -#endif #ifdef CONFIG_FAST_IRQ .if \regsize == 64 && \label == irq && \el == 0 && \ht == t alternative_if_not ARM64_HAS_XINT @@ -797,7 +738,6 @@ SYM_CODE_END(el\el\ht\()_\regsize\()_\label) entry_handler 0, t, 64, error #ifdef CONFIG_ACTLR_XCALL_XINT - entry_handler 0, t, 64, xcall entry_handler 0, t, 64, xint #endif entry_handler 0, t, 32, sync diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index fe3f89445fcb..e9e5ce956f15 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -55,6 +55,7 @@ #include <asm/stacktrace.h> #include <asm/switch_to.h> #include <asm/system_misc.h> +#include <asm/xcall.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> @@ -472,6 +473,10 @@ DEFINE_PER_CPU(struct task_struct *, __entry_task); static void entry_task_switch(struct task_struct *next) { __this_cpu_write(__entry_task, next); +#ifdef CONFIG_FAST_SYSCALL + if (static_branch_unlikely(&xcall_enable)) + xcall_info_switch(next); +#endif } /* diff --git a/arch/arm64/kernel/xcall/entry.S b/arch/arm64/kernel/xcall/entry.S index 7b75e8651a2a..f531f1e17f48 100644 --- a/arch/arm64/kernel/xcall/entry.S +++ b/arch/arm64/kernel/xcall/entry.S @@ -84,7 +84,7 @@ alternative_else_nop_endif #endif .endm /* .macro hw_xcall_save_base_regs */ - .macro hw_xcal_restore_base_regs + .macro hw_xcall_restore_base_regs #ifdef CONFIG_ARM64_PSEUDO_NMI alternative_if_not ARM64_HAS_GIC_PRIO_MASKING b .Lskip_pmr_restore\@ @@ -149,15 +149,89 @@ alternative_else_nop_endif add sp, sp, #PT_REGS_SIZE // restore sp eret sb - .endm /* .macro hw_xcal_restore_base_regs */ + .endm /* .macro hw_xcall_restore_base_regs */ -SYM_CODE_START(xcall_entry) +SYM_CODE_START_LOCAL(el0t_64_svc_entry) +alternative_if_not ARM64_HAS_HW_XCALL_XINT + /* Hijack SVC to dynamically load syscalls via '/proc/xcall/comm' */ + ldr x20, [sp, #S_SYSCALLNO] // ESR.bits[15,0] + cmp x20, 0xfff + b.ge el0t_64_xcall_entry +alternative_else_nop_endif + + /* Hijack SVC to low overhead syscalls via '/prox/[pid]/xcall' */ + cmp x8, __NR_syscalls + b.ge .slow_syscall + ldr_this_cpu x21, __xcall_info, x20 + ldrb w20, [x21, x8] + cmp x20, 0 + bne el0t_fast_syscall + +.slow_syscall: + ldp x20, x21, [sp, #16 * 10] + kernel_entry 0, 64 + mov x0, sp + bl el0_slow_syscall + b ret_to_user +SYM_INNER_LABEL(el0t_64_xcall_entry, SYM_L_GLOBAL) + lsr x20, x20, #12 + adr x21, .xcall_func_table + ldr w20, [x21, x20, lsl #2] + add x20, x20, x21 + br x20 + /* ISS==0F~FF: Entry to optimized and customized syscalls + */ +.xcall_func_table: + .rept 16 + .word el0t_xcall_syscall - .xcall_func_table + .endr +SYM_CODE_END(el0t_64_svc_entry) + +SYM_CODE_START_LOCAL(el0t_xcall_syscall) + ldp x20, x21, [sp, #16 * 10] hw_xcall_save_base_regs mov x0, sp - bl el0t_64_xcall_handler - hw_xcal_restore_base_regs -SYM_CODE_END(xcall_entry) + bl el0_xcall_syscall + hw_xcall_restore_base_regs +SYM_CODE_END(el0t_xcall_syscall) +SYM_CODE_START_LOCAL(el0t_fast_syscall) + ldp x20, x21, [sp, #16 * 10] + hw_xcall_save_base_regs + mov x0, sp + bl el0_fast_syscall + hw_xcall_restore_base_regs +SYM_CODE_END(el0t_fast_syscall) + +SYM_CODE_START_LOCAL(el0t_64_sync_ventry) + kernel_ventry 0, t, 64, sync +SYM_CODE_END(el0t_64_sync_ventry) + +SYM_CODE_START_LOCAL(el0t_64_sync_ventry_vector) + ldp x20, x21, [sp, #16 * 10] + add sp, sp, #PT_REGS_SIZE + kernel_ventry 0, t, 64, sync +SYM_CODE_END(el0t_64_sync_ventry_vector) + +SYM_CODE_START_LOCAL(el0t_64_sync_table) + // 0 - (ESR_ELx_EC_SVC64 - 1) + .rept ESR_ELx_EC_SVC64 + .word el0t_64_sync_table - el0t_64_sync_ventry_vector + .endr + // ESR_ELx_EC_SVC64 + .word el0t_64_sync_table - el0t_64_svc_entry + // (ESR_ELx_EC_SVC64 + 1) - ESR_ELx_EC_MAX + .rept ESR_ELx_EC_MAX - ESR_ELx_EC_SVC64 + .word el0t_64_sync_table - el0t_64_sync_ventry_vector + .endr +SYM_CODE_END(el0t_64_sync_table) + +/*********************************************** + * * + * Xcall exception entry code for 920G CPU * + * * + ***********************************************/ +#ifdef CONFIG_ACTLR_XCALL_XINT .macro xcall_ventry .align 7 .Lventry_start\@: @@ -170,6 +244,45 @@ SYM_CODE_END(xcall_entry) msr tpidrro_el0, xzr .Lskip_tramp_vectors_cleanup\@: sub sp, sp, #PT_REGS_SIZE - b xcall_entry + stp x20, x21, [sp, #16 * 10] + /* Decode ESR.ICC bits[15, 0] for use later */ + mrs x21, esr_el1 + uxth w20, w21 + b el0t_64_xcall_entry +.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? + .endm +#endif /* CONFIG_ACTLR_XCALL_XINT */ + +/**************************************************************** + * * + * Sync exception entry code for early CPUs before 920G * + * * + ****************************************************************/ + .macro sync_ventry + .align 7 +.Lventry_start\@: + /* + * This must be the first instruction of the EL0 vector entries. It is + * skipped by the trampoline vectors, to trigger the cleanup. + */ + b .Lskip_tramp_vectors_cleanup\@ + mrs x30, tpidrro_el0 + msr tpidrro_el0, xzr +.Lskip_tramp_vectors_cleanup\@: +alternative_if_not ARM64_HAS_XCALL + b el0t_64_sync_ventry +alternative_else_nop_endif + sub sp, sp, #PT_REGS_SIZE + /* Save ESR and ICC.bits[15,0] for use later */ + stp x20, x21, [sp, #16 * 10] + mrs x20, esr_el1 + uxth w21, w20 + stp x20, x21, [sp, #(S_SYSCALLNO - 8)] + /* Using jump table for different exception causes */ + lsr w21, w20, #ESR_ELx_EC_SHIFT + adr x20, el0t_64_sync_table + ldr w21, [x20, x21, lsl #2] + sub x20, x20, x21 + br x20 .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm diff --git a/arch/arm64/kernel/xcall/xcall.c b/arch/arm64/kernel/xcall/xcall.c index 31072c0402f4..96e6274571d3 100644 --- a/arch/arm64/kernel/xcall/xcall.c +++ b/arch/arm64/kernel/xcall/xcall.c @@ -17,8 +17,9 @@ static inline int sw_xcall_init_task(struct task_struct *p, struct task_struct * return -ENOMEM; if (orig->xinfo) { - bitmap_copy(TASK_XINFO(p)->xcall_enable, TASK_XINFO(orig)->xcall_enable, - __NR_syscalls); + memcpy(TASK_XINFO(p)->xcall_enable, + TASK_XINFO(orig)->xcall_enable, + (__NR_syscalls + 1) * sizeof(u8)); } return 0; @@ -37,3 +38,14 @@ void xcall_task_free(struct task_struct *p) if (static_branch_unlikely(&xcall_enable)) kfree(p->xinfo); } + +static u8 default_xcall_info[__NR_syscalls + 1] = { + [0 ... __NR_syscalls] = 0, +}; +DEFINE_PER_CPU(u8*, __xcall_info) = default_xcall_info; + +void xcall_info_switch(struct task_struct *task) +{ + if (TASK_XINFO(task)->xcall_enable) + __this_cpu_write(__xcall_info, TASK_XINFO(task)->xcall_enable); +} diff --git a/fs/proc/proc_xcall.c b/fs/proc/proc_xcall.c index 5f45d0799b33..8f7375235803 100644 --- a/fs/proc/proc_xcall.c +++ b/fs/proc/proc_xcall.c @@ -12,9 +12,10 @@ static int xcall_show(struct seq_file *m, void *v) { struct inode *inode = m->private; - struct task_struct *p; - unsigned int rs, re; + int start = -1, first = 1; struct xcall_info *xinfo; + struct task_struct *p; + int scno = 0; if (!static_key_enabled(&xcall_enable)) return -EACCES; @@ -27,14 +28,27 @@ static int xcall_show(struct seq_file *m, void *v) if (!xinfo) goto out; - for (rs = 0, bitmap_next_set_region(xinfo->xcall_enable, &rs, &re, __NR_syscalls); - rs < re; rs = re + 1, - bitmap_next_set_region(xinfo->xcall_enable, &rs, &re, __NR_syscalls)) { - if (rs == (re - 1)) - seq_printf(m, "%d,", rs); - else - seq_printf(m, "%d-%d,", rs, re - 1); + for (scno = 0; scno <= __NR_syscalls; scno++) { + if (scno == __NR_syscalls || !xinfo->xcall_enable[scno]) { + if (start == -1) + continue; + + if (!first) + seq_puts(m, ","); + + if (start == scno - 1) + seq_printf(m, "%d", start); + else + seq_printf(m, "%d-%d", start, scno - 1); + + first = 0; + start = -1; + } else { + if (start == -1) + start = scno; + } } + seq_puts(m, "\n"); out: put_task_struct(p); @@ -47,45 +61,28 @@ static int xcall_open(struct inode *inode, struct file *filp) return single_open(filp, xcall_show, inode); } -static int xcall_enable_one(struct xcall_info *xinfo, unsigned int sc_no) -{ - test_and_set_bit(sc_no, xinfo->xcall_enable); - return 0; -} - -static int xcall_disable_one(struct xcall_info *xinfo, unsigned int sc_no) -{ - test_and_clear_bit(sc_no, xinfo->xcall_enable); - return 0; -} - -static ssize_t xcall_write(struct file *file, const char __user *buf, +static ssize_t xcall_write(struct file *file, const char __user *ubuf, size_t count, loff_t *offset) { - struct inode *inode = file_inode(file); - struct task_struct *p; - char buffer[5]; - const size_t maxlen = sizeof(buffer) - 1; unsigned int sc_no = __NR_syscalls; + struct task_struct *p; + char buf[5]; int ret = 0; - int is_clear = 0; - struct xcall_info *xinfo; if (!static_key_enabled(&xcall_enable)) return -EACCES; - memset(buffer, 0, sizeof(buffer)); - if (!count || copy_from_user(buffer, buf, count > maxlen ? maxlen : count)) - return -EFAULT; - - p = get_proc_task(inode); - if (!p || !p->xinfo) + p = get_proc_task(file_inode(file)); + if (!p || !TASK_XINFO(p)) return -ESRCH; - if (buffer[0] == '!') - is_clear = 1; + memset(buf, '\0', 5); + if (!count || (count > 5) || copy_from_user(buf, ubuf, count)) { + ret = -EFAULT; + goto out; + } - if (kstrtouint(buffer + is_clear, 10, &sc_no)) { + if (kstrtouint((buf + (int)(buf[0] == '!')), 10, &sc_no)) { ret = -EINVAL; goto out; } @@ -95,13 +92,8 @@ static ssize_t xcall_write(struct file *file, const char __user *buf, goto out; } - xinfo = TASK_XINFO(p); - if (!is_clear && !test_bit(sc_no, xinfo->xcall_enable)) - ret = xcall_enable_one(xinfo, sc_no); - else if (is_clear && test_bit(sc_no, xinfo->xcall_enable)) - ret = xcall_disable_one(xinfo, sc_no); - else - ret = -EINVAL; + (TASK_XINFO(p))->xcall_enable[sc_no] = (int)(buf[0] != '!'); + ret = 0; out: put_task_struct(p); -- 2.34.1