From: Yipeng Zou <zouyipeng@huawei.com> Since xcall has been supported, we can switch all syscall to xcall by using libxcall or set to it /proc/$PID/xcall. This patch introduces xcall_select to support a special syscall implemented. First you need define one XCALL_DEFINEx function in kernel. Then you can switch one syscall to this function by: echo @$syscall_nr > /proc/$PID/xcall Make sure it has been enabled in xcall before switch to xcall function. Signed-off-by: Yipeng Zou <zouyipeng@huawei.com> Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/arm64/include/asm/exception.h | 3 ++ arch/arm64/include/asm/syscall.h | 14 +++++++ arch/arm64/include/asm/syscall_wrapper.h | 25 ++++++++++++ arch/arm64/kernel/sys.c | 20 +++++++++ arch/arm64/kernel/syscall.c | 14 ++++++- fs/proc/base.c | 52 +++++++++++++++++++++--- include/linux/sched.h | 3 +- include/linux/syscalls.h | 12 ++++++ include/uapi/asm-generic/unistd.h | 10 +++++ kernel/fork.c | 12 ++++++ 10 files changed, 157 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index 4b7994bd2b94..b427bd61dd31 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -55,4 +55,7 @@ void do_el0_svc(struct pt_regs *regs); void do_el0_svc_compat(struct pt_regs *regs); void do_el0_fpac(struct pt_regs *regs, unsigned long esr); void do_el1_fpac(struct pt_regs *regs, unsigned long esr); +#ifdef CONFIG_FAST_SYSCALL +void do_el0_xcall(struct pt_regs *regs); +#endif #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 45ac51ba05fc..9b108442c162 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -12,6 +12,9 @@ typedef long (*syscall_fn_t)(const struct pt_regs *regs); extern const syscall_fn_t sys_call_table[]; +#ifdef CONFIG_FAST_SYSCALL +extern const syscall_fn_t x_call_table[]; +#endif #ifdef CONFIG_AARCH32_EL0 extern const syscall_fn_t a32_sys_call_table[]; @@ -99,4 +102,15 @@ static inline int syscall_get_arch(struct task_struct *task) return AUDIT_ARCH_AARCH64; } +#ifdef CONFIG_FAST_SYSCALL +asmlinkage long __arm64_sys_ni_syscall(const struct pt_regs *__unused); + +static inline int syscall_is_xcall_register(unsigned int sc_no) +{ + if (x_call_table[sc_no] == __arm64_sys_ni_syscall) + return 0; + return 1; +} +#endif + #endif /* __ASM_SYSCALL_H */ diff --git a/arch/arm64/include/asm/syscall_wrapper.h b/arch/arm64/include/asm/syscall_wrapper.h index d30217c21eff..70150380a01f 100644 --- a/arch/arm64/include/asm/syscall_wrapper.h +++ b/arch/arm64/include/asm/syscall_wrapper.h @@ -48,6 +48,31 @@ #endif /* CONFIG_COMPAT */ +#ifdef CONFIG_FAST_SYSCALL +#define __XCALL_DEFINEx(x, name, ...) \ + asmlinkage long __arm64_xcall_sys##name(const struct pt_regs *regs); \ + ALLOW_ERROR_INJECTION(__arm64_xcall_sys##name, ERRNO); \ + static long __se_xcall_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ + static inline long __do_xcall_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ + asmlinkage long __arm64_xcall_sys##name(const struct pt_regs *regs) \ + { \ + return __se_xcall_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \ + } \ + static long __se_xcall_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ + { \ + long ret = __do_xcall_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__)); \ + __MAP(x,__SC_TEST,__VA_ARGS__); \ + __PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__)); \ + return ret; \ + } \ + static inline long __do_xcall_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) + +#define XCALL_DEFINE0(sname) \ + asmlinkage long __arm64_xcall_sys_##sname(const struct pt_regs *__unused); \ + ALLOW_ERROR_INJECTION(__arm64_xcall_sys_##sname, ERRNO); \ + asmlinkage long __arm64_xcall_sys_##sname(const struct pt_regs *__unused) +#endif + #define __SYSCALL_DEFINEx(x, name, ...) \ asmlinkage long __arm64_sys##name(const struct pt_regs *regs); \ ALLOW_ERROR_INJECTION(__arm64_sys##name, ERRNO); \ diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index d5ffaaab31a7..6508f6130bbf 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -48,6 +48,10 @@ asmlinkage long __arm64_sys_ni_syscall(const struct pt_regs *__unused) */ #define __arm64_sys_personality __arm64_sys_arm64_personality +#ifdef CONFIG_FAST_SYSCALL +#undef __XCALL +#endif + #undef __SYSCALL #define __SYSCALL(nr, sym) asmlinkage long __arm64_##sym(const struct pt_regs *); #include <asm/unistd.h> @@ -59,3 +63,19 @@ const syscall_fn_t sys_call_table[__NR_syscalls] = { [0 ... __NR_syscalls - 1] = __arm64_sys_ni_syscall, #include <asm/unistd.h> }; + +#ifdef CONFIG_FAST_SYSCALL +#undef __SYSCALL + +#undef __XCALL +#define __XCALL(nr, sym) asmlinkage long __arm64_xcall_##sym(const struct pt_regs *); +#include <asm/unistd.h> + +#undef __XCALL +#define __XCALL(nr, sym) [nr] = __arm64_xcall_##sym, + +const syscall_fn_t x_call_table[__NR_syscalls] = { + [0 ... __NR_syscalls - 1] = __arm64_sys_ni_syscall, +#include <asm/unistd.h> +}; +#endif diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 72a84fe4a2a7..b8ed5d8c24d6 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -280,7 +280,9 @@ static inline void delouse_pt_regs(struct pt_regs *regs) #ifdef CONFIG_FAST_SYSCALL void do_el0_xcall(struct pt_regs *regs) { - const syscall_fn_t *t = sys_call_table; + unsigned int scno, scno_nr; + const syscall_fn_t *t; + int xcall_nr; #ifdef CONFIG_ARM64_ILP32 if (is_ilp32_compat_task()) { @@ -289,6 +291,16 @@ void do_el0_xcall(struct pt_regs *regs) } #endif + scno = regs->regs[8]; + scno_nr = __NR_syscalls; + + xcall_nr = array_index_nospec(scno, scno_nr); + if (scno < scno_nr && current->xcall_select && + test_bit(xcall_nr, current->xcall_select)) + t = x_call_table; + else + t = sys_call_table; + fp_user_discard(); el0_xcall_common(regs, regs->regs[8], __NR_syscalls, t); } diff --git a/fs/proc/base.c b/fs/proc/base.c index 3206960c4bd7..4c6fdda92fa4 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3596,7 +3596,7 @@ static int xcall_show(struct seq_file *m, void *v) { struct inode *inode = m->private; struct task_struct *p; - unsigned int rs, re; + unsigned int rs, re, sc_no; if (!fast_syscall_enabled()) return -EACCES; @@ -3617,7 +3617,19 @@ static int xcall_show(struct seq_file *m, void *v) rs == (re - 1) ? seq_printf(m, "%d,", rs) : seq_printf(m, "%d-%d,", rs, re - 1); } - seq_puts(m, "\n"); + seq_printf(m, "\nAvailable:\n"); + + for (sc_no = 0; sc_no < __NR_syscalls; sc_no++) { + if (!syscall_is_xcall_register(sc_no)) + continue; + + seq_printf(m, "NR_syscall: %3d: enabled: %d ", sc_no, test_bit(sc_no, p->xcall_enable)); + + if (p->xcall_select) + seq_printf(m, "xcall_select: %d\n", test_bit(sc_no, p->xcall_select)); + else + seq_printf(m, "xcall_select: NULL\n"); + } out: put_task_struct(p); @@ -3631,6 +3643,15 @@ static int xcall_open(struct inode *inode, struct file *filp) static int xcall_enable_one(struct task_struct *p, unsigned int sc_no) { + /* Alloc in First */ + if (!bitmap_weight(p->xcall_enable, __NR_syscalls)) { + BUG_ON(p->xcall_select); + p->xcall_select = bitmap_zalloc(__NR_syscalls, GFP_KERNEL); + if (!p->xcall_select) + return -EINVAL; + } + + bitmap_clear(p->xcall_select, sc_no, 1); bitmap_set(p->xcall_enable, sc_no, 1); return 0; } @@ -3638,6 +3659,21 @@ static int xcall_enable_one(struct task_struct *p, unsigned int sc_no) static int xcall_disable_one(struct task_struct *p, unsigned int sc_no) { bitmap_clear(p->xcall_enable, sc_no, 1); + bitmap_clear(p->xcall_select, sc_no, 1); + + /* Free in Last */ + if (!bitmap_weight(p->xcall_enable, __NR_syscalls)) { + BUG_ON(!p->xcall_select); + bitmap_free(p->xcall_select); + p->xcall_select = NULL; + } + return 0; +} + +static int xcall_select_table(struct task_struct *p, unsigned int sc_no) +{ + BUG_ON(!p->xcall_select); + test_and_change_bit(sc_no, p->xcall_select); return 0; } @@ -3650,7 +3686,7 @@ static ssize_t xcall_write(struct file *file, const char __user *buf, const size_t maxlen = sizeof(buffer) - 1; unsigned int sc_no = __NR_syscalls; int ret = 0; - int is_clear = 0; + int is_clear = 0, is_switch = 0; if (!fast_syscall_enabled()) return -EACCES; @@ -3665,8 +3701,10 @@ static ssize_t xcall_write(struct file *file, const char __user *buf, if (buffer[0] == '!') is_clear = 1; + else if ((buffer[0] == '@')) + is_switch = 1; - if (kstrtouint(buffer + is_clear, 10, &sc_no)) { + if (kstrtouint(buffer + is_clear + is_switch, 10, &sc_no)) { ret = -EINVAL; goto out; } @@ -3676,9 +3714,11 @@ static ssize_t xcall_write(struct file *file, const char __user *buf, goto out; } - if (!is_clear && !test_bit(sc_no, p->xcall_enable)) + if (is_switch && syscall_is_xcall_register(sc_no) && test_bit(sc_no, p->xcall_enable)) + ret = xcall_select_table(p, sc_no); + else if (!is_switch && !is_clear && !test_bit(sc_no, p->xcall_enable)) ret = xcall_enable_one(p, sc_no); - else if (is_clear && test_bit(sc_no, p->xcall_enable)) + else if (!is_switch && is_clear && test_bit(sc_no, p->xcall_enable)) ret = xcall_disable_one(p, sc_no); else ret = -EINVAL; diff --git a/include/linux/sched.h b/include/linux/sched.h index 18361e35a377..a377bae2064e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1479,10 +1479,11 @@ struct task_struct { #endif #if defined(CONFIG_FAST_SYSCALL) KABI_USE(15, unsigned long *xcall_enable) + KABI_USE(16, unsigned long *xcall_select) #else KABI_RESERVE(15) -#endif KABI_RESERVE(16) +#endif KABI_AUX_PTR(task_struct) /* CPU-specific state of this task: */ diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 509733f9214f..0e379bcd8194 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -226,6 +226,18 @@ static inline int is_syscall_trace_event(struct trace_event_call *tp_event) SYSCALL_METADATA(sname, x, __VA_ARGS__) \ __SYSCALL_DEFINEx(x, sname, __VA_ARGS__) +#ifdef CONFIG_FAST_SYSCALL +#define XCALL_DEFINE1(name, ...) XCALL_DEFINEx(1, _##name, __VA_ARGS__) +#define XCALL_DEFINE2(name, ...) XCALL_DEFINEx(2, _##name, __VA_ARGS__) +#define XCALL_DEFINE3(name, ...) XCALL_DEFINEx(3, _##name, __VA_ARGS__) +#define XCALL_DEFINE4(name, ...) XCALL_DEFINEx(4, _##name, __VA_ARGS__) +#define XCALL_DEFINE5(name, ...) XCALL_DEFINEx(5, _##name, __VA_ARGS__) +#define XCALL_DEFINE6(name, ...) XCALL_DEFINEx(6, _##name, __VA_ARGS__) + +#define XCALL_DEFINEx(x, sname, ...) \ + __XCALL_DEFINEx(x, sname, __VA_ARGS__) +#endif + #define __PROTECT(...) asmlinkage_protect(__VA_ARGS__) /* diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 064f512fefd9..9b38861d9ea8 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -16,18 +16,28 @@ #define __SYSCALL(x, y) #endif +#ifndef __XCALL +#define __XCALL(x, y) +#endif + #if __BITS_PER_LONG == 32 || defined(__SYSCALL_COMPAT) #define __SC_3264(_nr, _32, _64) __SYSCALL(_nr, _32) +#define __XCALL_SC_3264(_nr, _32, _64) __XCALL(_nr, _32) #else #define __SC_3264(_nr, _32, _64) __SYSCALL(_nr, _64) +#define __XCALL_SC_3264(_nr, _32, _64) __XCALL(_nr, _64) #endif #ifdef __SYSCALL_COMPAT #define __SC_COMP(_nr, _sys, _comp) __SYSCALL(_nr, _comp) #define __SC_COMP_3264(_nr, _32, _64, _comp) __SYSCALL(_nr, _comp) +#define __XCALL_SC_COMP(_nr, _sys, _comp) __XCALL(_nr, _comp) +#define __XCALL_SC_COMP_3264(_nr, _32, _64, _comp) __XCALL(_nr, _comp) #else #define __SC_COMP(_nr, _sys, _comp) __SYSCALL(_nr, _sys) #define __SC_COMP_3264(_nr, _32, _64, _comp) __SC_3264(_nr, _32, _64) +#define __XCALL_SC_COMP(_nr, _sys, _comp) __XCALL(_nr, _sys) +#define __XCALL_SC_COMP_3264(_nr, _32, _64, _comp) __XCALL_SC_3264(_nr, _32, _64) #endif #define __NR_io_setup 0 diff --git a/kernel/fork.c b/kernel/fork.c index bd7afeb364ab..b884ac9cdece 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -483,6 +483,9 @@ void free_task(struct task_struct *tsk) #ifdef CONFIG_FAST_SYSCALL if (tsk->xcall_enable) bitmap_free(tsk->xcall_enable); + + if (tsk->xcall_select) + bitmap_free(tsk->xcall_select); #endif free_task_struct(tsk); @@ -1016,6 +1019,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #ifdef CONFIG_FAST_SYSCALL tsk->xcall_enable = NULL; + tsk->xcall_select = NULL; #endif return tsk; @@ -2103,6 +2107,14 @@ static __latent_entropy struct task_struct *copy_process( if (current->xcall_enable) bitmap_copy(p->xcall_enable, current->xcall_enable, __NR_syscalls); + + if (current->xcall_select) { + p->xcall_select = bitmap_zalloc(__NR_syscalls, GFP_KERNEL); + if (!p->xcall_select) + goto bad_fork_free; + + bitmap_copy(p->xcall_select, current->xcall_select, __NR_syscalls); + } #endif #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY -- 2.34.1