hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/release-management/issues/IB6JLE -------------------------------- Refactor the xcall proc code by move to proc_xcall.c and replace fast_syscall_enabled() with system_supports_xcall(). Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/arm64/include/asm/cpufeature.h | 6 ++ arch/arm64/kernel/cpufeature.c | 5 -- fs/proc/Makefile | 1 + fs/proc/base.c | 109 -------------------------- fs/proc/internal.h | 4 + fs/proc/proc_xcall.c | 114 ++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 114 deletions(-) create mode 100644 fs/proc/proc_xcall.c diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 88cdf27fe701..00c3caddac98 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -889,6 +889,12 @@ static inline bool system_supports_haft(void) cpus_have_final_cap(ARM64_HAFT); } +static __always_inline bool system_supports_xcall(void) +{ + return IS_ENABLED(CONFIG_FAST_SYSCALL) && + cpus_have_const_cap(ARM64_HAS_XCALL); +} + int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); bool try_emulate_mrs(struct pt_regs *regs, u32 isn); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index b13858668877..2a5001455ad6 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2438,11 +2438,6 @@ static int __init xcall_setup(char *str) } __setup("xcall", xcall_setup); -bool fast_syscall_enabled(void) -{ - return is_xcall_support; -} - static bool has_xcall_support(const struct arm64_cpu_capabilities *entry, int __unused) { return is_xcall_support; diff --git a/fs/proc/Makefile b/fs/proc/Makefile index daa43e10b40b..b17f9b9eb652 100644 --- a/fs/proc/Makefile +++ b/fs/proc/Makefile @@ -38,3 +38,4 @@ proc-$(CONFIG_MEMORY_RELIABLE) += mem_reliable.o obj-$(CONFIG_ETMEM_SCAN) += etmem_scan.o obj-$(CONFIG_ETMEM_SWAP) += etmem_swap.o proc-${CONFIG_ETMEM} += etmem_proc.o +proc-$(CONFIG_FAST_SYSCALL) += proc_xcall.o diff --git a/fs/proc/base.c b/fs/proc/base.c index e62bb31bcbb1..62ec367d7f94 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3454,115 +3454,6 @@ static const struct file_operations proc_pid_sg_level_operations = { }; #endif -#ifdef CONFIG_FAST_SYSCALL -bool fast_syscall_enabled(void); - -static int xcall_show(struct seq_file *m, void *v) -{ - struct inode *inode = m->private; - struct task_struct *p; - unsigned int rs, re; - - if (!fast_syscall_enabled()) - return -EACCES; - - p = get_proc_task(inode); - if (!p) - return -ESRCH; - - if (!p->xcall_enable) - goto out; - - seq_printf(m, "Enabled Total[%d/%d]:", bitmap_weight(p->xcall_enable, __NR_syscalls), - __NR_syscalls); - - for (rs = 0, bitmap_next_set_region(p->xcall_enable, &rs, &re, __NR_syscalls); - rs < re; rs = re + 1, - bitmap_next_set_region(p->xcall_enable, &rs, &re, __NR_syscalls)) { - rs == (re - 1) ? seq_printf(m, "%d,", rs) : - seq_printf(m, "%d-%d,", rs, re - 1); - } - seq_puts(m, "\n"); -out: - put_task_struct(p); - - return 0; -} - -static int xcall_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, xcall_show, inode); -} - -static int xcall_enable_one(struct task_struct *p, unsigned int sc_no) -{ - bitmap_set(p->xcall_enable, sc_no, 1); - return 0; -} - -static int xcall_disable_one(struct task_struct *p, unsigned int sc_no) -{ - bitmap_clear(p->xcall_enable, sc_no, 1); - return 0; -} - -static ssize_t xcall_write(struct file *file, const char __user *buf, - size_t count, loff_t *offset) -{ - struct inode *inode = file_inode(file); - struct task_struct *p; - char buffer[TASK_COMM_LEN]; - const size_t maxlen = sizeof(buffer) - 1; - unsigned int sc_no = __NR_syscalls; - int ret = 0; - int is_clear = 0; - - if (!fast_syscall_enabled()) - return -EACCES; - - memset(buffer, 0, sizeof(buffer)); - if (!count || copy_from_user(buffer, buf, count > maxlen ? maxlen : count)) - return -EFAULT; - - p = get_proc_task(inode); - if (!p || !p->xcall_enable) - return -ESRCH; - - if (buffer[0] == '!') - is_clear = 1; - - if (kstrtouint(buffer + is_clear, 10, &sc_no)) { - ret = -EINVAL; - goto out; - } - - if (sc_no >= __NR_syscalls) { - ret = -EINVAL; - goto out; - } - - if (!is_clear && !test_bit(sc_no, p->xcall_enable)) - ret = xcall_enable_one(p, sc_no); - else if (is_clear && test_bit(sc_no, p->xcall_enable)) - ret = xcall_disable_one(p, sc_no); - else - ret = -EINVAL; - -out: - put_task_struct(p); - - return ret ? ret : count; -} - -static const struct file_operations proc_pid_xcall_operations = { - .open = xcall_open, - .read = seq_read, - .write = xcall_write, - .llseek = seq_lseek, - .release = single_release, -}; -#endif - /* * Thread groups */ diff --git a/fs/proc/internal.h b/fs/proc/internal.h index a86dba479fff..d5509cb5019c 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -334,3 +334,7 @@ static inline void pde_force_lookup(struct proc_dir_entry *pde) /* /proc/net/ entries can be changed under us by setns(CLONE_NEWNET) */ pde->proc_dops = &proc_net_dentry_ops; } + +#ifdef CONFIG_FAST_SYSCALL +extern const struct file_operations proc_pid_xcall_operations; +#endif diff --git a/fs/proc/proc_xcall.c b/fs/proc/proc_xcall.c new file mode 100644 index 000000000000..7d0cdd87a3c6 --- /dev/null +++ b/fs/proc/proc_xcall.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * xcall related proc code + * + * Copyright (C) 2025 Huawei Ltd. + */ +#include <linux/cpufeature.h> +#include <linux/sched.h> +#include <linux/seq_file.h> +#include "internal.h" + +static int xcall_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + unsigned int rs, re; + + if (!system_supports_xcall()) + return -EACCES; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + if (!p->xcall_enable) + goto out; + + for (rs = 0, bitmap_next_set_region(p->xcall_enable, &rs, &re, __NR_syscalls); + rs < re; rs = re + 1, + bitmap_next_set_region(p->xcall_enable, &rs, &re, __NR_syscalls)) { + if (rs == (re - 1)) + seq_printf(m, "%d,", rs); + else + seq_printf(m, "%d-%d,", rs, re - 1); + } + seq_puts(m, "\n"); +out: + put_task_struct(p); + + return 0; +} + +static int xcall_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, xcall_show, inode); +} + +static int xcall_enable_one(struct task_struct *p, unsigned int sc_no) +{ + test_and_set_bit(sc_no, p->xcall_enable); + return 0; +} + +static int xcall_disable_one(struct task_struct *p, unsigned int sc_no) +{ + test_and_clear_bit(sc_no, p->xcall_enable); + return 0; +} + +static ssize_t xcall_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + char buffer[5]; + const size_t maxlen = sizeof(buffer) - 1; + unsigned int sc_no = __NR_syscalls; + int ret = 0; + int is_clear = 0; + + if (!system_supports_xcall()) + return -EACCES; + + memset(buffer, 0, sizeof(buffer)); + if (!count || copy_from_user(buffer, buf, count > maxlen ? maxlen : count)) + return -EFAULT; + + p = get_proc_task(inode); + if (!p || !p->xcall_enable) + return -ESRCH; + + if (buffer[0] == '!') + is_clear = 1; + + if (kstrtouint(buffer + is_clear, 10, &sc_no)) { + ret = -EINVAL; + goto out; + } + + if (sc_no >= __NR_syscalls) { + ret = -EINVAL; + goto out; + } + + if (!is_clear && !test_bit(sc_no, p->xcall_enable)) + ret = xcall_enable_one(p, sc_no); + else if (is_clear && test_bit(sc_no, p->xcall_enable)) + ret = xcall_disable_one(p, sc_no); + else + ret = -EINVAL; + +out: + put_task_struct(p); + + return ret ? ret : count; +} + +const struct file_operations proc_pid_xcall_operations = { + .open = xcall_open, + .read = seq_read, + .write = xcall_write, + .llseek = seq_lseek, + .release = single_release, +}; -- 2.34.1

In the svc exception handling process of the ARM64, some auxiliary functions such as debug/trace and core functions such as sched/signal need to be considered. As a result, the svc exception handling process is very "lengthy". In some short syscalls (such as lseek(), getpid(), etc.) handles the syscall function itself with a small percentage of instructions, which also leads to inefficient execution of such syscalls. The idea of xcall can be regarded as a fast svc exception handling path. In the processing process, only the context save and restore and syscall functions are considered. It can be understood that xcall is a fast syscall processing mechanism between vdso and normal syscalls. The svc instruction with the immediate 0xffff is used to transfer xcall information. At the synchronization exception handling entry of the arm64 vectors, the ESR_EL1 register is used to distinguish xcall from normal syscalls. Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/arm64/Kconfig.turbo | 5 +- arch/arm64/include/asm/cpufeature.h | 6 - arch/arm64/include/asm/exception.h | 5 + arch/arm64/include/asm/mmu_context.h | 7 + arch/arm64/include/asm/xcall.h | 74 +++++++++- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/cpufeature.c | 31 +++- arch/arm64/kernel/entry-common.c | 2 +- arch/arm64/kernel/entry.S | 30 ++-- arch/arm64/kernel/xcall.c | 35 ----- arch/arm64/kernel/xcall/Makefile | 2 + arch/arm64/kernel/xcall/entry.S | 208 +++++++++++++++++++++++++++ arch/arm64/kernel/xcall/xcall.c | 67 +++++++++ arch/arm64/tools/cpucaps | 2 +- fs/proc/proc_xcall.c | 62 +++++++- 15 files changed, 468 insertions(+), 70 deletions(-) delete mode 100644 arch/arm64/kernel/xcall.c create mode 100644 arch/arm64/kernel/xcall/Makefile create mode 100644 arch/arm64/kernel/xcall/entry.S create mode 100644 arch/arm64/kernel/xcall/xcall.c diff --git a/arch/arm64/Kconfig.turbo b/arch/arm64/Kconfig.turbo index 76ad4ea567c3..a1148df4b37c 100644 --- a/arch/arm64/Kconfig.turbo +++ b/arch/arm64/Kconfig.turbo @@ -36,7 +36,7 @@ config FAST_IRQ config DEBUG_FEATURE_BYPASS bool "Bypass debug feature in fast syscall" - depends on FAST_SYSCALL || FAST_IRQ + depends on FAST_SYSCALL || FAST_IRQ || ACTLR_XCALL_XINT depends on !LOCKDEP default n help @@ -51,7 +51,7 @@ config SECURITY_FEATURE_BYPASS bool "Bypass security feature in fast syscall" depends on !ARM64_MTE depends on !KASAN_HW_TAGS - depends on FAST_SYSCALL || FAST_IRQ + depends on FAST_SYSCALL || FAST_IRQ || ACTLR_XCALL_XINT default n help This to bypass security feature in fast syscall. @@ -63,6 +63,7 @@ config SECURITY_FEATURE_BYPASS config ACTLR_XCALL_XINT bool "Hardware XCALL and Xint support" + depends on FAST_SYSCALL default n help Use the 0x600 as the offset to the exception vector base address for diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 00c3caddac98..88cdf27fe701 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -889,12 +889,6 @@ static inline bool system_supports_haft(void) cpus_have_final_cap(ARM64_HAFT); } -static __always_inline bool system_supports_xcall(void) -{ - return IS_ENABLED(CONFIG_FAST_SYSCALL) && - cpus_have_const_cap(ARM64_HAS_XCALL); -} - int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); bool try_emulate_mrs(struct pt_regs *regs, u32 isn); diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index ad688e157c9b..d69f0e6d53f8 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -77,4 +77,9 @@ void do_serror(struct pt_regs *regs, unsigned long esr); void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far); + +#ifdef CONFIG_ACTLR_XCALL_XINT +asmlinkage void el0t_64_xint_handler(struct pt_regs *regs); +asmlinkage void el0t_64_xcall_handler(struct pt_regs *regs); +#endif #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index a6fb325424e7..39595fa03491 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -24,6 +24,9 @@ #include <asm/cputype.h> #include <asm/sysreg.h> #include <asm/tlbflush.h> +#ifdef CONFIG_ACTLR_XCALL_XINT +#include <asm/xcall.h> +#endif extern bool rodata_full; @@ -264,6 +267,10 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, if (prev != next) __switch_mm(next); +#ifdef CONFIG_ACTLR_XCALL_XINT + cpu_switch_xcall_entry(tsk); +#endif + /* * Update the saved TTBR0_EL1 of the scheduled-in task as the previous * value may have not been initialised yet (activate_mm caller) or the diff --git a/arch/arm64/include/asm/xcall.h b/arch/arm64/include/asm/xcall.h index d45c32eef7f8..71f27bad8185 100644 --- a/arch/arm64/include/asm/xcall.h +++ b/arch/arm64/include/asm/xcall.h @@ -3,10 +3,14 @@ #define __ASM_XCALL_H #include <linux/jump_label.h> +#include <linux/percpu.h> #include <linux/sched.h> #include <linux/types.h> -DECLARE_STATIC_KEY_FALSE(xcall_enable); +#include <asm/actlr.h> + +extern bool is_xcall_support; +extern bool is_hw_xcall_support; struct xcall_info { /* Must be first! */ @@ -17,4 +21,72 @@ struct xcall_info { int xcall_init_task(struct task_struct *p, struct task_struct *orig); void xcall_task_free(struct task_struct *p); + +#ifdef CONFIG_ACTLR_XCALL_XINT +struct hw_xcall_info { + /* Must be first! */ + void *xcall_entry[__NR_syscalls]; + bool xcall_scno_enabled; +}; + +#define TASK_HW_XINFO(p) ((struct hw_xcall_info *)p->xinfo) +#define XCALL_ENTRY_SIZE (sizeof(unsigned long) * __NR_syscalls) + +extern void xcall_entry(void); +extern void no_xcall_entry(void); + +static inline bool is_xcall_entry(struct hw_xcall_info *xinfo, unsigned int sc_no) +{ + return xinfo->xcall_entry[sc_no] == xcall_entry; +} + +static inline int has_xcall_scno_enabled(struct hw_xcall_info *xinfo) +{ + unsigned int i; + + for (i = 0; i < __NR_syscalls; i++) { + if (is_xcall_entry(xinfo, i)) + return true; + } + + return false; +} + +static inline int set_xcall_entry(struct hw_xcall_info *xinfo, unsigned int sc_no) +{ + xinfo->xcall_entry[sc_no] = xcall_entry; + xinfo->xcall_scno_enabled = true; + + return 0; +} + +static inline int set_no_xcall_entry(struct hw_xcall_info *xinfo, unsigned int sc_no) +{ + xinfo->xcall_entry[sc_no] = no_xcall_entry; + if (!has_xcall_scno_enabled(xinfo)) + xinfo->xcall_scno_enabled = false; + + return 0; +} + +static inline void cpu_enable_arch_xcall(void) +{ + u64 el = read_sysreg(CurrentEL); + + if (el == CurrentEL_EL2) + write_sysreg(read_sysreg(actlr_el2) | ACTLR_ELx_XCALL, actlr_el2); + else + write_sysreg(read_sysreg(actlr_el1) | ACTLR_ELx_XCALL, actlr_el1); +} + +static inline void cpu_switch_xcall_entry(struct task_struct *tsk) +{ + if (!is_hw_xcall_support || !tsk->xinfo) + return; + + if (TASK_HW_XINFO(tsk)->xcall_scno_enabled) + cpu_enable_arch_xcall(); +} +#endif /* CONFIG_ACTLR_XCALL_XINT */ + #endif /*__ASM_XCALL_H*/ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 5fe59ab95979..20b8c6466965 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -79,7 +79,7 @@ obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o obj-$(CONFIG_ARM64_ILP32) += vdso-ilp32/ -obj-$(CONFIG_FAST_SYSCALL) += xcall.o +obj-$(CONFIG_FAST_SYSCALL) += xcall/ obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o virtcca_cvm_tsi.o diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index cc6ba1a7736e..871ff2bd0b57 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2431,7 +2431,9 @@ static void mpam_extra_caps(void) #ifdef CONFIG_FAST_SYSCALL #include <asm/xcall.h> -DEFINE_STATIC_KEY_FALSE(xcall_enable); +static DEFINE_STATIC_KEY_FALSE(xcall_enable); +bool is_xcall_support; +bool is_hw_xcall_support; static int __init xcall_setup(char *str) { @@ -2442,7 +2444,15 @@ __setup("xcall", xcall_setup); static bool has_xcall_support(const struct arm64_cpu_capabilities *entry, int __unused) { - return static_branch_unlikely(&xcall_enable); + if (is_hw_xcall_support) + return false; + + if (static_branch_unlikely(&xcall_enable)) { + is_xcall_support = true; + return true; + } + + return false; } #endif @@ -2473,7 +2483,12 @@ static bool has_arch_xcall_xint_support(const struct arm64_cpu_capabilities *ent { /* sentinel */ } }; - return is_midr_in_range_list(read_cpuid_id(), xcall_xint_cpus); + if (is_midr_in_range_list(read_cpuid_id(), xcall_xint_cpus)) { + is_hw_xcall_support = true; + return true; + } + + return false; } static void enable_xcall_xint_vectors(void) @@ -2504,7 +2519,7 @@ static void enable_xcall_xint_vectors(void) isb(); } -static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__unused) +static void cpu_enable_arch_xint(void) { int cpu = smp_processor_id(); u64 actlr_el1, actlr_el2; @@ -2517,20 +2532,24 @@ static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__un */ write_sysreg_s(read_sysreg_s(SYS_HCR_EL2) | HCR_TACR, SYS_HCR_EL2); actlr_el2 = read_sysreg(actlr_el2); - actlr_el2 |= (ACTLR_ELx_XINT | ACTLR_ELx_XCALL); + actlr_el2 |= ACTLR_ELx_XINT; write_sysreg(actlr_el2, actlr_el2); isb(); actlr_el2 = read_sysreg(actlr_el2); pr_info("actlr_el2: %llx, cpu:%d\n", actlr_el2, cpu); } else { actlr_el1 = read_sysreg(actlr_el1); - actlr_el1 |= (ACTLR_ELx_XINT | ACTLR_ELx_XCALL); + actlr_el1 |= ACTLR_ELx_XINT; write_sysreg(actlr_el1, actlr_el1); isb(); actlr_el1 = read_sysreg(actlr_el1); pr_info("actlr_el1: %llx, cpu:%d\n", actlr_el1, cpu); } +} +static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__unused) +{ + cpu_enable_arch_xint(); enable_xcall_xint_vectors(); } #endif diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index a90231346751..1e8171c1efe7 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -1054,7 +1054,7 @@ UNHANDLED(el0t, 32, error) #ifdef CONFIG_ACTLR_XCALL_XINT asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs) { - el0_svc(regs); + el0_xcall(regs); } asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) { diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 3d0e5cc1d904..338d596c46ca 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -581,6 +581,8 @@ SYM_CODE_START(vectors) SYM_CODE_END(vectors) #ifdef CONFIG_ACTLR_XCALL_XINT +#include "xcall/entry.S" + .align 11 SYM_CODE_START(vectors_xcall_xint) kernel_ventry 1, t, 64, sync // Synchronous EL1t @@ -598,7 +600,7 @@ SYM_CODE_START(vectors_xcall_xint) kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 kernel_ventry 0, t, 64, error // Error 64-bit EL0 - kernel_ventry 0, t, 64, xcall // XCALL synchronous 64-bit EL0 + xcall_ventry 0, t, 64, xcall // XCALL synchronous 64-bit EL0 kernel_ventry 0, t, 64, xint // XINT 64-bit EL0 kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 kernel_ventry 0, t, 32, error // Error 32-bit EL0 @@ -647,10 +649,22 @@ SYM_CODE_END(__bad_stack) cmp x8, __NR_syscalls .endm - .macro check_xcall_enable + .macro check_xcall_pre_kernel_entry + stp x20, x21, [sp, #0] + /* is ESR_ELx_EC_SVC64 */ + check_esr_el1_ec_svc64 + bne .Lskip_xcall\@ + /* x8 >= __NR_syscalls */ + check_syscall_nr + bhs .Lskip_xcall\@ + + /* is xcall enabled */ /* x21 = task_struct->xinfo->xcall_enable */ ldr_this_cpu x20, __entry_task, x21 ldr x21, [x20, #TSK_XCALL] + /* is task_struct->xinfo == NULL */ + cbz x21, .Lskip_xcall\@ + str x8, [sp, #16] /* x20 = sc_no / 8 */ lsr x20, x8, 3 ldr x21, [x21, x20] @@ -660,19 +674,7 @@ SYM_CODE_END(__bad_stack) lsl x20, x20, x8 and x21, x21, x20 cmp x21, 0 - .endm - .macro check_xcall_pre_kernel_entry - stp x20, x21, [sp, #0] - /* is ESR_ELx_EC_SVC64 */ - check_esr_el1_ec_svc64 - bne .Lskip_xcall\@ - /* x8 >= __NR_syscalls */ - check_syscall_nr - bhs .Lskip_xcall\@ - str x8, [sp, #16] - /* is xcall enabled */ - check_xcall_enable ldr x8, [sp, #16] beq .Lskip_xcall\@ ldp x20, x21, [sp, #0] diff --git a/arch/arm64/kernel/xcall.c b/arch/arm64/kernel/xcall.c deleted file mode 100644 index f6a507a3aed0..000000000000 --- a/arch/arm64/kernel/xcall.c +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * xcall related code - * - * Copyright (C) 2025 Huawei Ltd. - */ - -#include <linux/bitmap.h> -#include <linux/slab.h> -#include <asm/xcall.h> - -int xcall_init_task(struct task_struct *p, struct task_struct *orig) -{ - if (!static_branch_unlikely(&xcall_enable)) - return 0; - - p->xinfo = kzalloc(sizeof(struct xcall_info), GFP_KERNEL); - if (!p->xinfo) - return -ENOMEM; - - if (orig->xinfo) { - bitmap_copy(TASK_XINFO(p)->xcall_enable, TASK_XINFO(orig)->xcall_enable, - __NR_syscalls); - } - - return 0; -} - -void xcall_task_free(struct task_struct *p) -{ - if (!static_branch_unlikely(&xcall_enable)) - return; - - kfree(p->xinfo); -} diff --git a/arch/arm64/kernel/xcall/Makefile b/arch/arm64/kernel/xcall/Makefile new file mode 100644 index 000000000000..0168bd190793 --- /dev/null +++ b/arch/arm64/kernel/xcall/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += xcall.o diff --git a/arch/arm64/kernel/xcall/entry.S b/arch/arm64/kernel/xcall/entry.S new file mode 100644 index 000000000000..4a3948aef8b2 --- /dev/null +++ b/arch/arm64/kernel/xcall/entry.S @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Low-level exception handling code + * + * Copyright (C) 2012 ARM Ltd. + * Authors: Catalin Marinas <catalin.marinas@arm.com> + * Will Deacon <will.deacon@arm.com> + */ + + .macro hw_xcall_entry + stp x0, x1, [sp, #16 * 0] // save x0~x29 + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + clear_gp_regs // clear x0~x29 + mrs x21, sp_el0 + ldr_this_cpu tsk, __entry_task, x20 + msr sp_el0, tsk + +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * Enable IA for in-kernel PAC if the task had it disabled. Although + * this could be implemented with an unconditional MRS which would avoid + * a load, this was measured to be slower on Cortex-A75 and Cortex-A76. + * + * Install the kernel IA key only if IA was enabled in the task. If IA + * was disabled on kernel exit then we would have left the kernel IA + * installed so there is no need to install it again. + */ + ldr x0, [tsk, THREAD_SCTLR_USER] + + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23 + b 2f +1: + mrs x0, sctlr_el1 + orr x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: +alternative_else_nop_endif +#endif + +alternative_if ARM64_HAS_ADDRESS_AUTH + isb +alternative_else_nop_endif + + mrs x22, elr_el1 + mrs x23, spsr_el1 + stp lr, x21, [sp, #S_LR] // save LR,USER SP + + stp xzr, xzr, [sp, #S_STACKFRAME] + add x29, sp, #S_STACKFRAME // calc FP + + stp x22, x23, [sp, #S_PC] // save ELR,SPSR + + mov w21, #NO_SYSCALL + str w21, [sp, #S_SYSCALLNO] + + +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if_not ARM64_HAS_GIC_PRIO_MASKING + b .Lskip_pmr_save\@ +alternative_else_nop_endif + + mrs_s x20, SYS_ICC_PMR_EL1 + str x20, [sp, #S_PMR_SAVE] + mov x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET + msr_s SYS_ICC_PMR_EL1, x20 + +.Lskip_pmr_save\@: +#endif + .endm + + .macro hw_xcall_exit +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if_not ARM64_HAS_GIC_PRIO_MASKING + b .Lskip_pmr_restore\@ +alternative_else_nop_endif + + ldr x20, [sp, #S_PMR_SAVE] + msr_s SYS_ICC_PMR_EL1, x20 + + /* Ensure priority change is seen by redistributor */ +alternative_if_not ARM64_HAS_GIC_PRIO_RELAXED_SYNC + dsb sy +alternative_else_nop_endif + +.Lskip_pmr_restore\@: +#endif + + ldp x21, x22, [sp, #S_PC] + ldr x23, [sp, #S_SP] + msr sp_el0, x23 // restore USER SP + +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * IA was enabled for in-kernel PAC. Disable it now if needed, or + * alternatively install the user's IA. All other per-task keys and + * SCTLR bits were updated on task switch. + * + * No kernel C function calls after this. + */ + ldr x0, [tsk, THREAD_SCTLR_USER] + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_user tsk, x0, x1, x2 + b 2f +1: + mrs x0, sctlr_el1 + bic x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: +alternative_else_nop_endif +#endif + + msr elr_el1, x21 // restore ELR + msr spsr_el1, x22 // restore SPSR + + ldp x0, x1, [sp, #16 * 0] // restore x0~x29 + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + ldr lr, [sp, #S_LR] + add sp, sp, #PT_REGS_SIZE // restore sp + eret + sb + .endm + +SYM_CODE_START(no_xcall_entry) + ldp x20, x21, [sp, #0] + kernel_entry 0, 64 + mov x0, sp + bl el0t_64_sync_handler + b ret_to_user +SYM_CODE_END(no_xcall_entry) + +SYM_CODE_START(xcall_entry) + ldp x20, x21, [sp, #0] + hw_xcall_entry + mov x0, sp + bl el0t_64_xcall_handler + hw_xcall_exit +SYM_CODE_END(xcall_entry) + + .macro check_hw_xcall_pre_kernel_entry + /* x8 >= __NR_syscalls */ + cmp x8, __NR_syscalls + bhs .Lskip_xcall\@ + + stp x20, x21, [sp, #0] + /* x21 = task_struct->xinfo->xcall_entry */ + ldr_this_cpu x20, __entry_task, x21 + ldr x21, [x20, #TSK_XCALL] + /* x21 = task_struct->xinfo->xcall_entry[x8] */ + ldr x21, [x21, x8, lsl #3] + br x21 +.Lskip_xcall\@: + .endm + +SYM_CODE_START_LOCAL(el0t_64_hw_xcall) + check_hw_xcall_pre_kernel_entry + kernel_entry 0, 64 + mov x0, sp + bl el0t_64_sync_handler + b ret_to_user +SYM_CODE_END(el0t_64_hw_xcall) + + .macro xcall_ventry, el:req, ht:req, regsize:req, label:req + .align 7 +.Lventry_start\@: + /* + * This must be the first instruction of the EL0 vector entries. It is + * skipped by the trampoline vectors, to trigger the cleanup. + */ + b .Lskip_tramp_vectors_cleanup\@ + mrs x30, tpidrro_el0 + msr tpidrro_el0, xzr +.Lskip_tramp_vectors_cleanup\@: + + sub sp, sp, #PT_REGS_SIZE + b el0t_64_hw_xcall +.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? + .endm diff --git a/arch/arm64/kernel/xcall/xcall.c b/arch/arm64/kernel/xcall/xcall.c new file mode 100644 index 000000000000..0651f37c297a --- /dev/null +++ b/arch/arm64/kernel/xcall/xcall.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * xcall related code + * + * Copyright (C) 2025 Huawei Ltd. + */ + +#include <linux/bitmap.h> +#include <linux/percpu.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <asm/xcall.h> + +static inline int sw_xcall_init_task(struct task_struct *p, struct task_struct *orig) +{ + p->xinfo = kzalloc(sizeof(struct xcall_info), GFP_KERNEL); + if (!p->xinfo) + return -ENOMEM; + + if (orig->xinfo) { + bitmap_copy(TASK_XINFO(p)->xcall_enable, TASK_XINFO(orig)->xcall_enable, + __NR_syscalls); + } + + return 0; +} + +#ifdef CONFIG_ACTLR_XCALL_XINT +static inline int hw_xcall_init_task(struct task_struct *p, struct task_struct *orig) +{ + int i; + + p->xinfo = kzalloc(sizeof(struct hw_xcall_info), GFP_KERNEL); + if (!p->xinfo) + return -ENOMEM; + + for (i = 0; i < __NR_syscalls; i++) + TASK_HW_XINFO(p)->xcall_entry[i] = no_xcall_entry; + + if (orig->xinfo) { + memcpy(p->xinfo, orig->xinfo, XCALL_ENTRY_SIZE); + TASK_HW_XINFO(p)->xcall_scno_enabled = TASK_HW_XINFO(orig)->xcall_scno_enabled; + } + + return 0; +} +#endif + +int xcall_init_task(struct task_struct *p, struct task_struct *orig) +{ + if (!is_hw_xcall_support && !is_xcall_support) + return 0; + +#ifdef CONFIG_ACTLR_XCALL_XINT + if (is_hw_xcall_support) + return hw_xcall_init_task(p, orig); +#endif + return sw_xcall_init_task(p, orig); +} + +void xcall_task_free(struct task_struct *p) +{ + if (!is_hw_xcall_support && !is_xcall_support) + return; + + kfree(p->xinfo); +} diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index ae0268822d61..4fc5ce93f3ea 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -108,11 +108,11 @@ WORKAROUND_SPECULATIVE_UNPRIV_LOAD WORKAROUND_HISILICON_ERRATUM_162100125 WORKAROUND_HISI_HIP08_RU_PREFETCH WORKAROUND_HISILICON_1980005 +HAS_HW_XCALL_XINT HAS_XCALL HAS_XINT HAS_LS64 HAS_LS64_V -HAS_HW_XCALL_XINT KABI_RESERVE_6 KABI_RESERVE_7 KABI_RESERVE_8 diff --git a/fs/proc/proc_xcall.c b/fs/proc/proc_xcall.c index 97504bc60823..a720b457905b 100644 --- a/fs/proc/proc_xcall.c +++ b/fs/proc/proc_xcall.c @@ -10,6 +10,35 @@ #include <asm/xcall.h> #include "internal.h" +static void hw_xcall_show(struct task_struct *p, struct seq_file *m) +{ + struct hw_xcall_info *hw_xinfo = TASK_HW_XINFO(p); + unsigned int i, start = 0, end = 0; + bool in_range = false; + + if (!hw_xinfo) + return; + + for (i = 0; i < __NR_syscalls; i++) { + bool scno_xcall_enable = is_xcall_entry(hw_xinfo, i); + + if (scno_xcall_enable && !in_range) { + in_range = true; + start = i; + } + + if ((!scno_xcall_enable || i == __NR_syscalls - 1) && in_range) { + in_range = false; + end = scno_xcall_enable ? i : i - 1; + if (i == start + 1) + seq_printf(m, "%u,", start); + else + seq_printf(m, "%u-%u,", start, end); + } + } + seq_puts(m, "\n"); +} + static int xcall_show(struct seq_file *m, void *v) { struct inode *inode = m->private; @@ -17,13 +46,20 @@ static int xcall_show(struct seq_file *m, void *v) unsigned int rs, re; struct xcall_info *xinfo; - if (!system_supports_xcall()) + if (!is_hw_xcall_support && !is_xcall_support) return -EACCES; p = get_proc_task(inode); if (!p) return -ESRCH; +#ifdef CONFIG_ACTLR_XCALL_XINT + if (is_hw_xcall_support) { + hw_xcall_show(p, m); + goto out; + } +#endif + xinfo = TASK_XINFO(p); if (!xinfo) goto out; @@ -60,6 +96,19 @@ static int xcall_disable_one(struct xcall_info *xinfo, unsigned int sc_no) return 0; } +static int set_hw_xcall(struct task_struct *p, unsigned int sc_no, bool is_clear) +{ + struct hw_xcall_info *hw_xinfo = TASK_HW_XINFO(p); + + if (!is_clear && !is_xcall_entry(hw_xinfo, sc_no)) + return set_xcall_entry(hw_xinfo, sc_no); + + if (is_clear && is_xcall_entry(hw_xinfo, sc_no)) + return set_no_xcall_entry(hw_xinfo, sc_no); + + return -EINVAL; +} + static ssize_t xcall_write(struct file *file, const char __user *buf, size_t count, loff_t *offset) { @@ -72,7 +121,7 @@ static ssize_t xcall_write(struct file *file, const char __user *buf, int is_clear = 0; struct xcall_info *xinfo; - if (!system_supports_xcall()) + if (!is_hw_xcall_support && !is_xcall_support) return -EACCES; memset(buffer, 0, sizeof(buffer)); @@ -83,7 +132,6 @@ static ssize_t xcall_write(struct file *file, const char __user *buf, if (!p || !p->xinfo) return -ESRCH; - xinfo = TASK_XINFO(p); if (buffer[0] == '!') is_clear = 1; @@ -97,6 +145,14 @@ static ssize_t xcall_write(struct file *file, const char __user *buf, goto out; } +#ifdef CONFIG_ACTLR_XCALL_XINT + if (is_hw_xcall_support) { + ret = set_hw_xcall(p, sc_no, is_clear); + goto out; + } +#endif + + xinfo = TASK_XINFO(p); if (!is_clear && !test_bit(sc_no, xinfo->xcall_enable)) ret = xcall_enable_one(xinfo, sc_no); else if (is_clear && test_bit(sc_no, xinfo->xcall_enable)) -- 2.34.1

Use __NR_syscalls as a special syscall number to handle illegal syscall by csel. Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/arm64/include/asm/xcall.h | 4 ++-- arch/arm64/kernel/xcall/entry.S | 11 +++++++---- arch/arm64/kernel/xcall/xcall.c | 6 +++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/xcall.h b/arch/arm64/include/asm/xcall.h index 28c2fda9aaa0..dcbc35c6e3c5 100644 --- a/arch/arm64/include/asm/xcall.h +++ b/arch/arm64/include/asm/xcall.h @@ -25,12 +25,12 @@ void xcall_task_free(struct task_struct *p); #ifdef CONFIG_ACTLR_XCALL_XINT struct hw_xcall_info { /* Must be first! */ - void *xcall_entry[__NR_syscalls]; + void *xcall_entry[__NR_syscalls + 1]; bool xcall_scno_enabled; }; #define TASK_HW_XINFO(p) ((struct hw_xcall_info *)p->xinfo) -#define XCALL_ENTRY_SIZE (sizeof(unsigned long) * __NR_syscalls) +#define XCALL_ENTRY_SIZE (sizeof(unsigned long) * (__NR_syscalls + 1)) DECLARE_PER_CPU(void **, __cpu_xcall_entry); extern void xcall_entry(void); diff --git a/arch/arm64/kernel/xcall/entry.S b/arch/arm64/kernel/xcall/entry.S index 4b7dc4d2ff9d..382aa3d0ca07 100644 --- a/arch/arm64/kernel/xcall/entry.S +++ b/arch/arm64/kernel/xcall/entry.S @@ -168,13 +168,16 @@ SYM_CODE_START(xcall_entry) SYM_CODE_END(xcall_entry) .macro check_hw_xcall_pre_kernel_entry + stp x20, x21, [sp, #0] + str x8, [sp, #16] + mov x21, __NR_syscalls /* x8 >= __NR_syscalls */ cmp x8, __NR_syscalls - bhs .Lskip_xcall\@ + csel x20, x8, x21, lt - stp x20, x21, [sp, #0] - ldr_this_cpu x21, __cpu_xcall_entry, x20 - ldr x21, [x21, x8, lsl #3] + ldr_this_cpu x21, __cpu_xcall_entry, x8 + ldr x21, [x21, x20, lsl #3] + ldr x8, [sp, #16] br x21 .Lskip_xcall\@: .endm diff --git a/arch/arm64/kernel/xcall/xcall.c b/arch/arm64/kernel/xcall/xcall.c index 580eb29b9cc0..8339035d5e53 100644 --- a/arch/arm64/kernel/xcall/xcall.c +++ b/arch/arm64/kernel/xcall/xcall.c @@ -26,8 +26,8 @@ static inline int sw_xcall_init_task(struct task_struct *p, struct task_struct * } #ifdef CONFIG_ACTLR_XCALL_XINT -static void *default_syscall_table[__NR_syscalls] = { - [0 ... __NR_syscalls - 1] = no_xcall_entry, +static void *default_syscall_table[__NR_syscalls + 1] = { + [0 ... __NR_syscalls] = no_xcall_entry, }; asmlinkage DEFINE_PER_CPU(void **, __cpu_xcall_entry) = default_syscall_table; @@ -39,7 +39,7 @@ static inline int hw_xcall_init_task(struct task_struct *p, struct task_struct * if (!p->xinfo) return -ENOMEM; - for (i = 0; i < __NR_syscalls; i++) + for (i = 0; i < __NR_syscalls + 1; i++) TASK_HW_XINFO(p)->xcall_entry[i] = no_xcall_entry; if (orig->xinfo) { -- 2.34.1