From: Yipeng Zou <zouyipeng@huawei.com> hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/release-management/issues/IB6JLE -------------------------------- Fix following warnings when enable nohz full CPU, by not skipping context tracking and irq tick idle inform. Also remove el1 xint, as less savings for both assembly and C. ct_state() == CONTEXT_USER WARNING: CPU: 66 PID: 10214 at kernel/sched/core.c:6012 __schedule+0x738/0xcf8 CPU: 66 PID: 10214 Comm: sh Kdump: loaded Not tainted 6.6.0xcall_xint_lcxint6.6_test+ #26 Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022 pstate: 60400009 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : __schedule+0x738/0xcf8 lr : __schedule+0x738/0xcf8 sp : ffff8000d193bd90 pmr_save: 00000060 x29: ffff8000d193bd90 x28: ffff20203e95d100 x27: ffff20203e95d100 x26: 0000000000000000 x25: ffff800080e3fbb8 x24: ffff800081a2a008 x23: ffff800081e15370 x22: 0000000000000000 x21: ffff800081a40d80 x20: ffff20203e95d100 x19: ffff2027dfa1ad80 x18: 0000000000000000 x17: ffffa0275dfda000 x16: ffff800083008000 x15: 0000000000000000 x14: 0000000000000000 x13: 524553555f545845 x12: 544e4f43203d3d20 x11: 00000000ffff7fff x10: ffff80008210b368 x9 : ffff800080157fa8 x8 : 00000000000bffe8 x7 : c0000000ffff7fff x6 : 00000000002bffa8 x5 : ffff2027dfa07ac8 x4 : ffff8000d193bbf0 x3 : ffffa0275dfda000 x2 : 0000000000000000 x1 : 0000000000000000 x0 : ffff20203e95d100 Call trace: __schedule+0x738/0xcf8 schedule+0x58/0xf0 do_notify_resume+0x50/0x350 el0_xint.constprop.0+0xa0/0xe0 el0t_64_xint_handler+0x18/0x28 el0t_64_irq+0x170/0x3a0 ---[ end trace 0000000000000000 ]--- ------------[ cut here ]------------ WARNING: CPU: 66 PID: 0 at kernel/context_tracking.c:128 ct_kernel_exit.constprop.0+0xa0/0xa8 CPU: 66 PID: 0 Comm: swapper/66 Kdump: loaded Tainted: G W 6.6.0xcall_xint_lcxint6.6_test+ #26 Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022 pstate: 80400009 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : ct_kernel_exit.constprop.0+0xa0/0xa8 lr : ct_idle_enter+0x14/0x20 sp : ffff8000839abdb0 pmr_save: 00000060 x29: ffff8000839abdb0 x28: 0000000000000000 x27: 0000000000000000 x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 x23: 0000000000000000 x22: ffff2020001f6540 x21: ffff2020001f6540 x20: 0000000000000005 x19: ffff2027dfa16760 x18: 0000000000000000 x17: ffffa0275dfda000 x16: ffff800083008000 x15: 0000000000000000 x14: 0000000000000000 x13: 524553555f545845 x12: 0000000000000000 x11: 0000000000000000 x10: 0000000000000f90 x9 : ffff8000801bb9b8 x8 : ffff2020001f7530 x7 : 00000000ffff9d08 x6 : 0000000748890425 x5 : 0000000000000000 x4 : ffffa0275dfda000 x3 : ffff8000839abdb0 x2 : ffff800081a3c760 x1 : ffff800081a3c760 x0 : 4000000000000000 Call trace: ct_kernel_exit.constprop.0+0xa0/0xa8 ct_idle_enter+0x14/0x20 default_idle_call+0x38/0x120 cpuidle_idle_call+0x164/0x1a8 do_idle+0xac/0x118 cpu_startup_entry+0x3c/0x50 secondary_start_kernel+0x14c/0x1e8 __secondary_switched+0xb8/0xc0 ---[ end trace 0000000000000000 ]--- ------------[ cut here ]------------ WARNING: CPU: 0 PID: 10214 at arch/arm64/kernel/entry-common.c:105 el0_da+0x98/0x1a8 CPU: 0 PID: 10214 Comm: sh Kdump: loaded Tainted: G W 6.6.0xcall_xint_lcxint6.6_test+ #26 Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022 pstate: 804003c9 (Nzcv DAIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : el0_da+0x98/0x1a8 lr : el0t_64_sync_handler+0xb8/0x130 sp : ffff8000d193be70 pmr_save: 000000f0 x29: ffff8000d193be70 x28: ffff20203e95d100 x27: 0000000000000000 x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 x23: 0000000060000000 x22: 0000ffffbb7a9028 x21: 0000aaaafa8a1c18 x20: 000000009200004f x19: ffff8000d193beb0 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000000 x10: 0000000000000000 x9 : 0000000000000000 x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000000 x5 : 0000000000000000 x4 : ffff8000d193beb0 x3 : 0000000000000000 x2 : ffff800080e36fac x1 : ffff80275df96000 x0 : 0000000000000000 Call trace: el0_da+0x98/0x1a8 el0t_64_sync_handler+0xb8/0x130 el0t_64_sync+0x3b8/0x3c0 ---[ end trace 0000000000000000 ]--- ------------[ cut here ]------------ WARNING: CPU: 0 PID: 15840 at arch/arm64/kernel/entry-common.c:105 el0_ia+0xc0/0x200 CPU: 0 PID: 15840 Comm: sh Kdump: loaded Tainted: G W 6.6.0xcall_xint_lcxint6.6_test+ #26 Hardware name: Huawei TaiShan 2280 V2/BC82AMDC, BIOS 2280-V2 CS V5.B221.01 04/29/2022 pstate: 804003c9 (Nzcv DAIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : el0_ia+0xc0/0x200 lr : el0t_64_sync_handler+0xd0/0x130 sp : ffff8000dc90be70 pmr_save: 000000f0 x29: ffff8000dc90be70 x28: ffff20204f112880 x27: 0000000000000000 x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 x23: 0000000020000000 x22: 0000ffffbb801080 x21: 0000ffffbb801080 x20: 0000000082000007 x19: ffff8000dc90beb0 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000000 x10: 0000000000000000 x9 : 0000000000000000 x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000000 x5 : 0000000000000000 x4 : ffff8000dc90beb0 x3 : 0000000000000000 x2 : 0000000000000000 x1 : ffff80275df96000 x0 : 0000000000000000 Call trace: el0_ia+0xc0/0x200 el0t_64_sync_handler+0xd0/0x130 el0t_64_sync+0x3b8/0x3c0 ---[ end trace 0000000000000000 ]--- Fixes: cf5f2d9ff017 ("arm64: Faster SVC exception handler with xcall") Fixes: 3053668e6b21 ("arm64: Introduce Xint software solution") Signed-off-by: Yipeng Zou <zouyipeng@huawei.com> Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/Kconfig | 1 + arch/arm64/kernel/entry-common.c | 161 ++++++++++++------------------- arch/arm64/kernel/entry.S | 18 ++-- kernel/softirq.c | 4 +- 4 files changed, 70 insertions(+), 114 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 98116fbfcff6..a9d7c5099b96 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1573,6 +1573,7 @@ config FAST_IRQ config DEBUG_FEATURE_BYPASS bool "Bypass debug feature in fast syscall" depends on FAST_SYSCALL || FAST_IRQ + depends on !LOCKDEP default y help This to bypass debug feature in fast syscall. diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 60300e133d5f..4602c107c40a 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -151,6 +151,62 @@ asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) exit_to_user_mode(regs); } +#if defined(CONFIG_FAST_SYSCALL) || defined(CONFIG_FAST_IRQ) +/* + * Copy from exit_to_user_mode_prepare + */ +static __always_inline void fast_exit_to_user_mode_prepare(struct pt_regs *regs) +{ + unsigned long flags; + + local_daif_mask(); + + flags = read_thread_flags(); + if (unlikely(flags & _TIF_WORK_MASK)) + do_notify_resume(regs, flags); + +#ifndef CONFIG_DEBUG_FEATURE_BYPASS + lockdep_sys_exit(); +#endif +} + +/* Copy from __exit_to_user_mode */ +static __always_inline void __fast_exit_to_user_mode(void) +{ +#ifndef CONFIG_DEBUG_FEATURE_BYPASS + trace_hardirqs_on_prepare(); + lockdep_hardirqs_on_prepare(); +#endif + user_enter_irqoff(); +#ifndef CONFIG_DEBUG_FEATURE_BYPASS + lockdep_hardirqs_on(CALLER_ADDR0); +#endif +} + +static __always_inline void fast_exit_to_user_mode(struct pt_regs *regs) +{ + fast_exit_to_user_mode_prepare(regs); +#ifndef CONFIG_DEBUG_FEATURE_BYPASS + mte_check_tfsr_exit(); +#endif + __fast_exit_to_user_mode(); +} + +/* Copy from __enter_from_user_mode */ +static __always_inline void fast_enter_from_user_mode(struct pt_regs *regs) +{ +#ifndef CONFIG_DEBUG_FEATURE_BYPASS + lockdep_hardirqs_off(CALLER_ADDR0); +#endif + CT_WARN_ON(ct_state() != CONTEXT_USER); + user_exit_irqoff(); +#ifndef CONFIG_DEBUG_FEATURE_BYPASS + trace_hardirqs_off_finish(); + mte_disable_tco_entry(current); +#endif +} +#endif + /* * Handle IRQ/context state management when entering an NMI from user/kernel * mode. Before this function is called it is not safe to call regular kernel @@ -513,80 +569,12 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, } #ifdef CONFIG_FAST_IRQ -static __always_inline void __el1_xint(struct pt_regs *regs, - void (*handler)(struct pt_regs *)) -{ -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - enter_from_kernel_mode(regs); -#endif - - xint_enter_rcu(); - do_interrupt_handler(regs, handler); - xint_exit_rcu(); - - arm64_preempt_schedule_irq(); - -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - exit_to_kernel_mode(regs); -#endif -} - -static void noinstr el1_xint(struct pt_regs *regs, u64 nmi_flag, - void (*handler)(struct pt_regs *), - void (*nmi_handler)(struct pt_regs *)) -{ - /* Is there a NMI to handle? */ -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - if (system_uses_nmi() && (read_sysreg(isr_el1) & nmi_flag)) { - __el1_nmi(regs, nmi_handler); - return; - } -#endif - - write_sysreg(DAIF_PROCCTX_NOIRQ, daif); - - if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) - __el1_pnmi(regs, handler); - else - __el1_xint(regs, handler); -} - -asmlinkage void noinstr el1h_64_xint_handler(struct pt_regs *regs) -{ - el1_xint(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq); -} - -static __always_inline void xint_exit_to_user_mode_prepare(struct pt_regs *regs) -{ - unsigned long flags; - - local_daif_mask(); - - flags = read_thread_flags(); - if (unlikely(flags & _TIF_WORK_MASK)) - do_notify_resume(regs, flags); - -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - lockdep_sys_exit(); -#endif -} - -static __always_inline void xint_exit_to_user_mode(struct pt_regs *regs) -{ - xint_exit_to_user_mode_prepare(regs); -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - mte_check_tfsr_exit(); - __exit_to_user_mode(); -#endif -} - static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, void (*handler)(struct pt_regs *), void (*nmi_handler)(struct pt_regs *)) { + fast_enter_from_user_mode(regs); #ifndef CONFIG_DEBUG_FEATURE_BYPASS - enter_from_user_mode(regs); - /* Is there a NMI to handle? */ if (system_uses_nmi() && (read_sysreg(isr_el1) & nmi_flag)) { /* @@ -615,7 +603,7 @@ static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, do_interrupt_handler(regs, handler); xint_exit_rcu(); - xint_exit_to_user_mode(regs); + fast_exit_to_user_mode(regs); } @@ -828,46 +816,17 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) } #ifdef CONFIG_FAST_SYSCALL -/* - * Copy from exit_to_user_mode_prepare - */ -static __always_inline void xcall_exit_to_user_mode_prepare(struct pt_regs *regs) -{ - unsigned long flags; - - local_daif_mask(); - - flags = read_thread_flags(); - if (unlikely(flags & _TIF_WORK_MASK)) - do_notify_resume(regs, flags); - -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - lockdep_sys_exit(); -#endif -} - -static __always_inline void xcall_exit_to_user_mode(struct pt_regs *regs) -{ - xcall_exit_to_user_mode_prepare(regs); -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - mte_check_tfsr_exit(); - __exit_to_user_mode(); -#endif -} - /* Copy from el0_sync */ static void noinstr el0_xcall(struct pt_regs *regs) { -#ifndef CONFIG_DEBUG_FEATURE_BYPASS - enter_from_user_mode(regs); -#endif + fast_enter_from_user_mode(regs); #ifndef CONFIG_SECURITY_FEATURE_BYPASS cortex_a76_erratum_1463225_svc_handler(); #endif fp_user_discard(); local_daif_restore(DAIF_PROCCTX); do_el0_svc(regs); - xcall_exit_to_user_mode(regs); + fast_exit_to_user_mode(regs); } asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e9fd3094623f..da3809632f0f 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -660,7 +660,7 @@ SYM_CODE_END(__bad_stack) #endif #ifdef CONFIG_FAST_IRQ -.macro check_xint_pre_kernel_entry el:req, ht:req +.macro check_xint_pre_kernel_entry stp x0, x1, [sp, #0] stp x2, x3, [sp, #16] @@ -691,20 +691,16 @@ SYM_CODE_END(__bad_stack) ldp x0, x1, [sp, #0] ldp x2, x3, [sp, #16] #ifdef CONFIG_SECURITY_FEATURE_BYPASS - kernel_entry \el, 64, xint + kernel_entry 0, 64, xint #else - kernel_entry \el, 64 + kernel_entry 0, 64 #endif mov x0, sp - bl el\el\ht\()_64_xint_handler + bl el0t_64_xint_handler #ifdef CONFIG_SECURITY_FEATURE_BYPASS - kernel_exit \el, xint + kernel_exit 0, xint #else - .if \el == 0 b ret_to_user - .else - b ret_to_kernel - .endif #endif .Lskip_xint\@: @@ -726,11 +722,11 @@ SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) .endif #endif #ifdef CONFIG_FAST_IRQ - .if \regsize == 64 && \label == irq && (( \el == 0 && \ht == t) || (\el == 1 && \ht == h)) + .if \regsize == 64 && \label == irq && \el == 0 && \ht == t alternative_if_not ARM64_HAS_XINT b .Lskip_check_xint\@ alternative_else_nop_endif - check_xint_pre_kernel_entry \el, \ht + check_xint_pre_kernel_entry .Lskip_check_xint\@: .endif #endif diff --git a/kernel/softirq.c b/kernel/softirq.c index 6dc0ea5baf06..cd8770b2f76c 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -669,11 +669,13 @@ void xint_enter_rcu(void) preempt_count_add(HARDIRQ_OFFSET); #ifndef CONFIG_DEBUG_FEATURE_BYPASS lockdep_hardirq_enter(); +#endif if (tick_nohz_full_cpu(smp_processor_id()) || (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET))) tick_irq_enter(); +#ifndef CONFIG_DEBUG_FEATURE_BYPASS account_hardirq_enter(current); #endif } @@ -695,9 +697,7 @@ static inline void __xint_exit_rcu(void) if (!in_interrupt() && local_softirq_pending()) invoke_softirq(); -#ifndef CONFIG_DEBUG_FEATURE_BYPASS tick_irq_exit(); -#endif } void xint_exit_rcu(void) -- 2.34.1