hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/release-management/issues/ID1OOE -------------------------------- When SMT (Simultaneous Multithreading) is used for mixed deployment of offline and online tasks, offline tasks cause a significant degradation in the QoS of online tasks in SMT. WFxT instructions do not affect SMT performance. So inject interrupts on the CPU cores running the offline tasks to do WFxT. Add VDSO support to do WFxt in userspace when el0 irq return. Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- arch/arm64/Kconfig.turbo | 12 +++++ arch/arm64/include/asm/smt_qos.h | 7 +++ arch/arm64/include/asm/thread_info.h | 5 ++ arch/arm64/include/asm/vdso.h | 6 ++- arch/arm64/include/uapi/asm/unistd.h | 5 ++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/entry-common.c | 38 ++++++++++++++- arch/arm64/kernel/smt_qos.c | 23 +++++++++ arch/arm64/kernel/vdso.c | 25 ++++++++++ arch/arm64/kernel/vdso/Makefile | 10 ++++ arch/arm64/kernel/vdso/smt_qos.c | 44 +++++++++++++++++ arch/arm64/kernel/vdso/smt_qos_trampoline.S | 52 +++++++++++++++++++++ arch/arm64/kernel/vdso/vdso.lds.S | 7 +++ include/linux/mm_types.h | 4 ++ include/linux/syscalls.h | 4 ++ include/vdso/datapage.h | 5 ++ kernel/fork.c | 14 ++++++ kernel/sysctl.c | 15 ++++++ 18 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/smt_qos.h create mode 100644 arch/arm64/kernel/smt_qos.c create mode 100644 arch/arm64/kernel/vdso/smt_qos.c create mode 100644 arch/arm64/kernel/vdso/smt_qos_trampoline.S diff --git a/arch/arm64/Kconfig.turbo b/arch/arm64/Kconfig.turbo index c4a8e4e889aa..99bac9292a48 100644 --- a/arch/arm64/Kconfig.turbo +++ b/arch/arm64/Kconfig.turbo @@ -71,4 +71,16 @@ config ACTLR_XCALL_XINT Use the 0x680 as the offset to the exception vector base address for the Armv8.8 NMI taken from EL0. +config XCALL_SMT_QOS + bool "Xcall SMT QoS Optimization in Offline Mixed Deployment Scenarios" + depends on !QOS_SCHED && !QOS_SCHED_SMART_GRID && SCHED_SMT + depends on FAST_IRQ || ACTLR_XCALL_XINT + default n + help + To reduce the interference of offline tasks on online tasks + in scenarios where SMT (Simultaneous Multithreading) is used + for mixed deployment of offline and online tasks, WFxT instructions + that do not affect SMT performance are executed by injecting PMU + interrupts on the CPU cores running the offline tasks. + endmenu # "Turbo features selection" diff --git a/arch/arm64/include/asm/smt_qos.h b/arch/arm64/include/asm/smt_qos.h new file mode 100644 index 000000000000..d4558b681958 --- /dev/null +++ b/arch/arm64/include/asm/smt_qos.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SMT_QOS_H +#define __ASM_SMT_QOS_H + +extern unsigned int sysctl_delay_cycles; + +#endif /* __ASM_SMT_QOS_H */ diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 379d24059f5b..ed27011113fb 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -48,8 +48,13 @@ struct thread_info { #endif u32 cpu; +#ifdef CONFIG_XCALL_SMT_QOS + u64 qos_context1; // save x0 + u64 qos_context2; // save x1 +#else KABI_RESERVE(1) KABI_RESERVE(2) +#endif }; #define thread_saved_pc(tsk) \ diff --git a/arch/arm64/include/asm/vdso.h b/arch/arm64/include/asm/vdso.h index 0cedfa1cce8a..574bee360405 100644 --- a/arch/arm64/include/asm/vdso.h +++ b/arch/arm64/include/asm/vdso.h @@ -12,7 +12,7 @@ */ #define VDSO_LBASE 0x0 -#define __VVAR_PAGES 2 +#define __VVAR_PAGES 3 #ifndef __ASSEMBLY__ @@ -38,6 +38,10 @@ extern char vdso32_start[], vdso32_end[]; extern char vdso_ilp32_start[], vdso_ilp32_end[]; #endif +#ifdef CONFIG_XCALL_SMT_QOS +struct qos_data *arch_get_qos_data(void *vvar_page); +#endif + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_H */ diff --git a/arch/arm64/include/uapi/asm/unistd.h b/arch/arm64/include/uapi/asm/unistd.h index 079139c04b14..8254d10a6384 100644 --- a/arch/arm64/include/uapi/asm/unistd.h +++ b/arch/arm64/include/uapi/asm/unistd.h @@ -36,3 +36,8 @@ #define __ARCH_WANT_MEMFD_SECRET #include <asm-generic/unistd.h> + +#ifndef __NR_vdso_wfxt_return +#define __NR_vdso_wfxt_return (__NR_arch_specific_syscall + 11) +__SYSCALL(__NR_vdso_wfxt_return, sys_vdso_wfxt_return) +#endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index d67126570cb2..9784594112ab 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -80,6 +80,7 @@ obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o obj-$(CONFIG_ARM64_ILP32) += vdso-ilp32/ obj-$(CONFIG_FAST_SYSCALL) += xcall/ +obj-$(CONFIG_XCALL_SMT_QOS) += smt_qos.o obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o virtcca_cvm_tsi.o diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 0fc502cf7276..55e6acaf4777 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -13,6 +13,9 @@ #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/thread_info.h> +#ifdef CONFIG_XCALL_SMT_QOS +#include <vdso/vsyscall.h> +#endif #include <asm/cpufeature.h> #include <asm/daifflags.h> @@ -26,6 +29,10 @@ #include <asm/stacktrace.h> #include <asm/sysreg.h> #include <asm/system_misc.h> +#ifdef CONFIG_XCALL_SMT_QOS +#include <asm/vdso.h> +#include <asm/smt_qos.h> +#endif /* * Handle IRQ/context state management when entering from kernel mode. @@ -570,6 +577,29 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, arm64_exit_nmi(regs); } +#ifdef CONFIG_XCALL_SMT_QOS +static void set_return_to_vdso(struct pt_regs *regs) +{ + struct qos_data *qos_data = NULL; + unsigned long trampoline = 0; + + if (current->mm && current->mm->smt_qos_page) { + qos_data = arch_get_qos_data(page_address(current->mm->smt_qos_page)); + trampoline = (unsigned long)VDSO_SYMBOL(current->mm->context.vdso, + smt_qos_trampoline); + } + + if (!cpus_have_const_cap(ARM64_HAS_WFXT) || !qos_data || + !trampoline || !sysctl_delay_cycles) + return; + + qos_data->delay_cycles = sysctl_delay_cycles; + task_thread_info(current)->qos_context1 = regs->pc; + task_thread_info(current)->qos_context2 = regs->regs[8]; + regs->pc = trampoline; +} +#endif + #if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ACTLR_XCALL_XINT) static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, void (*handler)(struct pt_regs *), @@ -588,6 +618,9 @@ static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, do_interrupt_handler(regs, nmi_handler); arm64_exit_nmi(regs); +#ifdef CONFIG_XCALL_SMT_QOS + set_return_to_vdso(regs); +#endif fast_exit_to_user_mode(regs); return; } @@ -603,10 +636,13 @@ static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, do_interrupt_handler(regs, handler); xint_exit_rcu(); +#ifdef CONFIG_XCALL_SMT_QOS + set_return_to_vdso(regs); +#endif + fast_exit_to_user_mode(regs); } - asmlinkage void noinstr el0t_64_fast_irq_handler(struct pt_regs *regs) { el0_xint(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq); diff --git a/arch/arm64/kernel/smt_qos.c b/arch/arm64/kernel/smt_qos.c new file mode 100644 index 000000000000..17ef3c42a8b1 --- /dev/null +++ b/arch/arm64/kernel/smt_qos.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "SMT QoS: " fmt + +#include <linux/sizes.h> +#include <linux/sched.h> +#include <linux/export.h> +#include <linux/module.h> +#include <linux/syscalls.h> +#include <linux/compiler_types.h> + +#include <asm/smt_qos.h> + +unsigned int sysctl_delay_cycles = 10000000; + +SYSCALL_DEFINE0(vdso_wfxt_return) +{ + struct pt_regs *regs = current_pt_regs(); + + regs->pc = task_thread_info(current)->qos_context1; + regs->regs[8] = task_thread_info(current)->qos_context2; + + return regs->regs[0]; +} diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index fff989a09f75..0ff0b3569802 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -40,6 +40,7 @@ enum vdso_abi { enum vvar_pages { VVAR_DATA_PAGE_OFFSET, VVAR_TIMENS_PAGE_OFFSET, + VVAR_SMT_QOS_PAGE_OFFSET, VVAR_NR_PAGES, }; @@ -126,6 +127,13 @@ static int __init __vdso_init(enum vdso_abi abi) return 0; } +#ifdef CONFIG_XCALL_SMT_QOS +struct qos_data *arch_get_qos_data(void *vvar_page) +{ + return (struct qos_data *)(vvar_page); +} +#endif + #ifdef CONFIG_TIME_NS struct vdso_data *arch_get_vdso_data(void *vvar_page) { @@ -161,6 +169,18 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) } #endif +#ifdef CONFIG_XCALL_SMT_QOS +static struct page *find_qos_vvar_page(struct vm_area_struct *vma) +{ + if (likely(vma->vm_mm == current->mm)) + return current->mm->smt_qos_page; + + WARN(1, "smt qos page accessed remotely"); + + return NULL; +} +#endif + static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, struct vm_area_struct *vma, struct vm_fault *vmf) { @@ -188,6 +208,11 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, pfn = sym_to_pfn(vdso_data); break; #endif /* CONFIG_TIME_NS */ +#ifdef CONFIG_XCALL_SMT_QOS + case VVAR_SMT_QOS_PAGE_OFFSET: + pfn = page_to_pfn(find_qos_vvar_page(vma)); + break; +#endif default: return VM_FAULT_SIGBUS; } diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index b46df9f340f9..8a023bb821dc 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -9,7 +9,11 @@ # Include the generic Makefile to check the built vdso. include $(srctree)/lib/vdso/Makefile +ifdef CONFIG_XCALL_SMT_QOS +obj-vdso := vgettimeofday.o note.o sigreturn.o smt_qos.o smt_qos_trampoline.o +else obj-vdso := vgettimeofday.o note.o sigreturn.o +endif # Build rules targets := $(obj-vdso) vdso.so vdso.so.dbg @@ -40,6 +44,12 @@ CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \ $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \ $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \ -Wmissing-prototypes -Wmissing-declarations +ifdef CONFIG_XCALL_SMT_QOS +CFLAGS_REMOVE_smt_qos.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \ + $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \ + $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \ + -Wmissing-prototypes -Wmissing-declarations +endif KASAN_SANITIZE := n KCSAN_SANITIZE := n UBSAN_SANITIZE := n diff --git a/arch/arm64/kernel/vdso/smt_qos.c b/arch/arm64/kernel/vdso/smt_qos.c new file mode 100644 index 000000000000..cdbd6adbe24d --- /dev/null +++ b/arch/arm64/kernel/vdso/smt_qos.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ARM64 userspace implementations of smt qos trampoline + * + */ +#include <vdso/helpers.h> +#include <linux/errno.h> + +#define wfit(val) asm volatile("msr s0_3_c1_c0_1, %0" \ + : : "r" (val) : "memory") +#define wfet(val) asm volatile("msr s0_3_c1_c0_0, %0" \ + : : "r" (val) : "memory") + +static __always_inline struct qos_data *__arch_get_qos_vdso_data(void) +{ + return &_qos_data; +} + +static unsigned long long get_cycles(void) +{ + unsigned long long val; + + asm volatile("mrs %0, cntvct_el0" : "=r" (val)); + + return val; +} + +static void delay(unsigned long cycles) +{ + unsigned long long start = get_cycles(); + unsigned long long end = start + cycles; + + wfit(end); + while ((get_cycles() - start) < cycles) + wfet(end); +} + + +void do_smt_qos_trampoline(void) +{ + struct qos_data *qos_data = __arch_get_qos_vdso_data(); + + delay(qos_data->delay_cycles); +} diff --git a/arch/arm64/kernel/vdso/smt_qos_trampoline.S b/arch/arm64/kernel/vdso/smt_qos_trampoline.S new file mode 100644 index 000000000000..d604762a95ff --- /dev/null +++ b/arch/arm64/kernel/vdso/smt_qos_trampoline.S @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include <linux/linkage.h> +#include <asm/assembler.h> +#include <asm/unistd.h> + + .text +SYM_CODE_START(__kernel_smt_qos_trampoline) + sub sp, sp, #16 * 16 + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + str x30, [sp, #16 * 15] + + bl do_smt_qos_trampoline + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + ldr x30, [sp, #16 * 15] + add sp, sp, #16 * 16 + + /* + * Call vdso_wfxt_return to restore context + */ + mov x8, 255 + svc #0xffff + +SYM_CODE_END(__kernel_smt_qos_trampoline) diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S index 2e126ab79ecd..467ae0a61890 100644 --- a/arch/arm64/kernel/vdso/vdso.lds.S +++ b/arch/arm64/kernel/vdso/vdso.lds.S @@ -21,6 +21,9 @@ SECTIONS PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE); #ifdef CONFIG_TIME_NS PROVIDE(_timens_data = _vdso_data + PAGE_SIZE); +#endif +#ifdef CONFIG_XCALL_SMT_QOS + PROVIDE(_qos_data = _vdso_data + PAGE_SIZE); #endif . = VDSO_LBASE + SIZEOF_HEADERS; @@ -103,6 +106,7 @@ VERSION __kernel_gettimeofday; __kernel_clock_gettime; __kernel_clock_getres; + __kernel_smt_qos_trampoline; local: *; }; } @@ -111,3 +115,6 @@ VERSION * Make the sigreturn code visible to the kernel. */ VDSO_sigtramp = __kernel_rt_sigreturn; +#ifdef CONFIG_XCALL_SMT_QOS +VDSO_smt_qos_trampoline = __kernel_smt_qos_trampoline; +#endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 64c38b09e18d..e9250be17f93 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1016,7 +1016,11 @@ struct mm_struct { #else KABI_RESERVE(1) #endif +#ifdef CONFIG_FAST_SYSCALL + KABI_USE(2, struct page *smt_qos_page) +#else KABI_RESERVE(2) +#endif KABI_RESERVE(3) KABI_RESERVE(4) KABI_RESERVE(5) diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 36c592e43d65..5a91f219c8cb 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -948,6 +948,10 @@ asmlinkage long sys_cachestat(unsigned int fd, struct cachestat __user *cstat, unsigned int flags); asmlinkage long sys_map_shadow_stack(unsigned long addr, unsigned long size, unsigned int flags); +#ifdef CONFIG_XCALL_SMT_QOS +asmlinkage long sys_vdso_wfxt_return(void); +#endif + /* * Architecture-specific system calls */ diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index f28f51c2c8f8..04b098153b4b 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -115,6 +115,10 @@ struct vdso_data { struct arch_vdso_data arch_data; }; +struct qos_data { + unsigned long delay_cycles; +}; + /* * We use the hidden visibility to prevent the compiler from generating a GOT * relocation. Not only is going through a GOT useless (the entry couldn't and @@ -126,6 +130,7 @@ struct vdso_data { */ extern struct vdso_data _vdso_data[CS_BASES] __attribute__((visibility("hidden"))); extern struct vdso_data _timens_data[CS_BASES] __attribute__((visibility("hidden"))); +extern struct qos_data _qos_data __attribute__((visibility("hidden"))); /* * The generic vDSO implementation requires that gettimeofday.h diff --git a/kernel/fork.c b/kernel/fork.c index 78663ca68160..6c3956cb027f 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -974,6 +974,13 @@ void __mmdrop(struct mm_struct *mm) mm_destroy_cid(mm); mm_counter_destroy(mm); +#ifdef CONFIG_FAST_SYSCALL + if (mm->smt_qos_page) { + __free_page(mm->smt_qos_page); + mm->smt_qos_page = NULL; + } +#endif + free_mm(mm); } EXPORT_SYMBOL_GPL(__mmdrop); @@ -1390,6 +1397,13 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, sp_init_mm(mm); mm->user_ns = get_user_ns(user_ns); lru_gen_init_mm(mm); + +#ifdef CONFIG_FAST_SYSCALL + mm->smt_qos_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!mm->smt_qos_page) + goto fail_cid; +#endif + return mm; fail_cid: diff --git a/kernel/sysctl.c b/kernel/sysctl.c index e84df0818517..e012d91b6002 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -81,6 +81,10 @@ #include <linux/rtmutex.h> #endif +#ifdef CONFIG_XCALL_SMT_QOS +#include <asm/smt_qos.h> +#endif + /* shared constants to be used in various sysctls */ const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; EXPORT_SYMBOL(sysctl_vals); @@ -2044,6 +2048,17 @@ static struct ctl_table kern_table[] = { .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_INT_MAX, }, +#endif +#ifdef CONFIG_XCALL_SMT_QOS + { + .procname = "xcall_vdso_delay_cycles", + .data = &sysctl_delay_cycles, + .maxlen = sizeof(sysctl_delay_cycles), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_INT_MAX, + }, #endif { } }; -- 2.34.1