After switch arm64 to Generic Entry, a new hotspot syscall_exit_work() appeared because syscall_exit_work() is no longer inlined. so inline syscall_exit_work(), and it has 2.6% performance uplift on perf bench basic syscall on kunpeng920 as below. | Metric | W/O this patch | With this patch | Change | | ---------- | -------------- | --------------- | --------- | | Total time | 2.171 [sec] | 2.114 [sec] | ↓2.6% | | usecs/op | 0.217192 | 0.211453 | ↓2.6% | | ops/sec | 4,604,225 | 4,729,178 | ↑2.7% | Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- include/linux/entry-common.h | 63 ++++++++++++++++++++++++++++++++++- kernel/entry/syscall-common.c | 59 ++------------------------------ 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index cd6dacb2d8bf..2f84377fb016 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -2,6 +2,7 @@ #ifndef __LINUX_ENTRYCOMMON_H #define __LINUX_ENTRYCOMMON_H +#include <linux/audit.h> #include <linux/irq-entry-common.h> #include <linux/ptrace.h> #include <linux/seccomp.h> @@ -128,6 +129,41 @@ static __always_inline long syscall_enter_from_user_mode(struct pt_regs *regs, l return ret; } +/* + * If SYSCALL_EMU is set, then the only reason to report is when + * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall + * instruction has been already reported in syscall_enter_from_user_mode(). + */ +static __always_inline bool report_single_step(unsigned long work) +{ + if (work & SYSCALL_WORK_SYSCALL_EMU) + return false; + + return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP; +} + +/** + * arch_ptrace_report_syscall_exit - Architecture specific + * ptrace_report_syscall_exit. + * + * Invoked from syscall_exit_work() to wrap ptrace_report_syscall_exit(). + * + * The main purpose is to support arch-specific ptrace_report_syscall_exit + * implementation. + */ +static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs, + int step); + +#ifndef arch_ptrace_report_syscall_exit +static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs, + int step) +{ + ptrace_report_syscall_exit(regs, step); +} +#endif + +void __trace_sys_exit(struct pt_regs *regs, long ret); + /** * syscall_exit_work - Handle work before returning to user mode * @regs: Pointer to current pt_regs @@ -135,7 +171,32 @@ static __always_inline long syscall_enter_from_user_mode(struct pt_regs *regs, l * * Do one-time syscall specific work. */ -void syscall_exit_work(struct pt_regs *regs, unsigned long work); +static __always_inline void syscall_exit_work(struct pt_regs *regs, unsigned long work) +{ + bool step; + + /* + * If the syscall was rolled back due to syscall user dispatching, + * then the tracers below are not invoked for the same reason as + * the entry side was not invoked in syscall_trace_enter(): The ABI + * of these syscalls is unknown. + */ + if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { + if (unlikely(current->syscall_dispatch.on_dispatch)) { + current->syscall_dispatch.on_dispatch = false; + return; + } + } + + audit_syscall_exit(regs); + + if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT) + __trace_sys_exit(regs, syscall_get_return_value(current, regs)); + + step = report_single_step(work); + if (step || work & SYSCALL_WORK_SYSCALL_TRACE) + arch_ptrace_report_syscall_exit(regs, step); +} /* * Syscall specific exit to user mode preparation. Runs with interrupts diff --git a/kernel/entry/syscall-common.c b/kernel/entry/syscall-common.c index 27310e611567..1636f49c58d2 100644 --- a/kernel/entry/syscall-common.c +++ b/kernel/entry/syscall-common.c @@ -90,62 +90,7 @@ noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs) instrumentation_end(); } -/* - * If SYSCALL_EMU is set, then the only reason to report is when - * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall - * instruction has been already reported in syscall_enter_from_user_mode(). - */ -static inline bool report_single_step(unsigned long work) -{ - if (work & SYSCALL_WORK_SYSCALL_EMU) - return false; - - return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP; -} - -/** - * arch_ptrace_report_syscall_exit - Architecture specific - * ptrace_report_syscall_exit. - * - * Invoked from syscall_exit_work() to wrap ptrace_report_syscall_exit(). - * - * The main purpose is to support arch-specific ptrace_report_syscall_exit - * implementation. - */ -static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs, - int step); - -#ifndef arch_ptrace_report_syscall_exit -static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs, - int step) +void __trace_sys_exit(struct pt_regs *regs, long ret) { - ptrace_report_syscall_exit(regs, step); -} -#endif - -void syscall_exit_work(struct pt_regs *regs, unsigned long work) -{ - bool step; - - /* - * If the syscall was rolled back due to syscall user dispatching, - * then the tracers below are not invoked for the same reason as - * the entry side was not invoked in syscall_trace_enter(): The ABI - * of these syscalls is unknown. - */ - if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { - if (unlikely(current->syscall_dispatch.on_dispatch)) { - current->syscall_dispatch.on_dispatch = false; - return; - } - } - - audit_syscall_exit(regs); - - if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT) - trace_sys_exit(regs, syscall_get_return_value(current, regs)); - - step = report_single_step(work); - if (step || work & SYSCALL_WORK_SYSCALL_TRACE) - arch_ptrace_report_syscall_exit(regs, step); + trace_sys_exit(regs, ret); } -- 2.34.1