From: Sven Schnelle svens@linux.ibm.com
mainline inclusion from mainline-v6.8-rc1 commit caf4062e35b21cd7d3d35ac2f58f9765d02d32a0 category: performance bugzilla: https://gitee.com/openeuler/kernel/issues/I9OJ0P CVE: NA
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...
------------------------------------------------------
To allow inlining of enter_from_user_mode(), move it to entry-common.h.
Signed-off-by: Sven Schnelle svens@linux.ibm.com Signed-off-by: Thomas Gleixner tglx@linutronix.de Link: https://lore.kernel.org/r/20231218074520.1998026-3-svens@linux.ibm.com Signed-off-by: liwei liwei728@huawei.com --- include/linux/entry-common.h | 15 ++++++++++++++- kernel/entry/common.c | 26 +++----------------------- 2 files changed, 17 insertions(+), 24 deletions(-)
diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index 6a6e98f3805f..c4205390448e 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -11,6 +11,7 @@ #include <linux/livepatch.h> #include <linux/resume_user_mode.h> #include <linux/tick.h> +#include <linux/kmsan.h>
#include <asm/entry-common.h>
@@ -102,7 +103,19 @@ static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs) {} * done between establishing state and enabling interrupts. The caller must * enable interrupts before invoking syscall_enter_from_user_mode_work(). */ -void enter_from_user_mode(struct pt_regs *regs); +static __always_inline void enter_from_user_mode(struct pt_regs *regs) +{ + arch_enter_from_user_mode(regs); + lockdep_hardirqs_off(CALLER_ADDR0); + + CT_WARN_ON(__ct_state() != CONTEXT_USER); + user_exit_irqoff(); + + instrumentation_begin(); + kmsan_unpoison_entry_regs(regs); + trace_hardirqs_off_finish(); + instrumentation_end(); +}
/** * syscall_enter_from_user_mode_prepare - Establish state and enable interrupts diff --git a/kernel/entry/common.c b/kernel/entry/common.c index bf545a7a5388..3c41bc7f5d68 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -15,26 +15,6 @@ #define CREATE_TRACE_POINTS #include <trace/events/syscalls.h>
-/* See comment for enter_from_user_mode() in entry-common.h */ -static __always_inline void __enter_from_user_mode(struct pt_regs *regs) -{ - arch_enter_from_user_mode(regs); - lockdep_hardirqs_off(CALLER_ADDR0); - - CT_WARN_ON(__ct_state() != CONTEXT_USER); - user_exit_irqoff(); - - instrumentation_begin(); - kmsan_unpoison_entry_regs(regs); - trace_hardirqs_off_finish(); - instrumentation_end(); -} - -void noinstr enter_from_user_mode(struct pt_regs *regs) -{ - __enter_from_user_mode(regs); -} - static inline void syscall_enter_audit(struct pt_regs *regs, long syscall) { if (unlikely(audit_context())) { @@ -111,7 +91,7 @@ noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall) { long ret;
- __enter_from_user_mode(regs); + enter_from_user_mode(regs);
instrumentation_begin(); local_irq_enable(); @@ -123,7 +103,7 @@ noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs) { - __enter_from_user_mode(regs); + enter_from_user_mode(regs); instrumentation_begin(); local_irq_enable(); instrumentation_end(); @@ -272,7 +252,7 @@ __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs) { - __enter_from_user_mode(regs); + enter_from_user_mode(regs); }
noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)