From: Ankur Arora ankur.a.arora@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

Add a timed variant of smp_cond_load_relaxed().

This is useful because arm64 supports polling on a conditional variable by directly waiting on the cacheline instead of spin waiting for the condition to change.

However, an implementation such as this has a problem that it can block forever -- unless there's an explicit timeout or another out-of-band mechanism which allows it to come out of the wait state periodically.

smp_cond_load_relaxed_timeout() supports these semantics by specifying a time-check expression and an associated time-limit.

However, note that for the generic spin-wait implementation we want to minimize the numbers of instructions executed in each iteration. So, limit how often we evaluate the time-check expression by doing it once every smp_cond_time_check_count.

The inner loop in poll_idle() has a substantially similar structure and constraints as smp_cond_load_relaxed_timeout(), so define smp_cond_time_check_count to the same value used in poll_idle().

Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- include/asm-generic/barrier.h | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+)

diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index 1985c22d90ca..f5d6c8444d33 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -275,6 +275,48 @@ do { \ }) #endif

+#ifndef smp_cond_time_check_count +/* + * Limit how often smp_cond_load_relaxed_timeout() evaluates time_expr_ns. + * This helps reduce the number of instructions executed while spin-waiting. + */ +#define smp_cond_time_check_count 200 +#endif + +/** + * smp_cond_load_relaxed_timeout() - (Spin) wait for cond with no ordering + * guarantees until a timeout expires. + * @ptr: pointer to the variable to wait on + * @cond: boolean expression to wait for + * @time_expr_ns: evaluates to the current time + * @time_limit_ns: compared against time_expr_ns + * + * Equivalent to using READ_ONCE() on the condition variable. + * + * Due to C lacking lambda expressions we load the value of *ptr into a + * pre-named variable @VAL to be used in @cond. + */ +#ifndef smp_cond_load_relaxed_timeout +#define smp_cond_load_relaxed_timeout(ptr, cond_expr, time_expr_ns, \ + time_limit_ns) ({ \ + typeof(ptr) __PTR = (ptr); \ + __unqual_scalar_typeof(*ptr) VAL; \ + unsigned int __count = 0; \ + for (;;) { \ + VAL = READ_ONCE(*__PTR); \ + if (cond_expr) \ + break; \ + cpu_relax(); \ + if (__count++ < smp_cond_time_check_count) \ + continue; \ + if ((time_expr_ns) >= time_limit_ns) \ + break; \ + __count = 0; \ + } \ + (typeof(*ptr))VAL; \ +}) +#endif + /* * pmem_wmb() ensures that all stores for which the modification * are written to persistent storage by preceding instructions have

From: Ankur Arora ankur.a.arora@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

ARCH_HAS_CPU_RELAX is defined on architectures that provide an primitive (via cpu_relax()) that can be used as part of a polling mechanism -- one that would be cheaper than spinning in a tight loop.

However, recent changes in poll_idle() mean that a higher level primitive -- smp_cond_load_relaxed() is used for polling. This would in-turn use cpu_relax() or an architecture specific implementation. On ARM64 in particular this turns into a WFE which waits on a store to a cacheline instead of a busy poll.

Accordingly condition the polling drivers on ARCH_HAS_OPTIMIZED_POLL instead of ARCH_HAS_CPU_RELAX. While at it, make both intel-idle and cpuidle-haltpoll explicitly depend on ARCH_HAS_CPU_RELAX.

Suggested-by: Will Deacon will@kernel.org Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- arch/x86/Kconfig | 2 +- drivers/acpi/processor_idle.c | 4 ++-- drivers/cpuidle/Kconfig | 2 +- drivers/cpuidle/Makefile | 2 +- drivers/idle/Kconfig | 1 + include/linux/cpuidle.h | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9be87b701f0f..0ee350a21f3a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -370,7 +370,7 @@ config ARCH_MAY_HAVE_PC_FDC config GENERIC_CALIBRATE_DELAY def_bool y

-config ARCH_HAS_CPU_RELAX +config ARCH_HAS_OPTIMIZED_POLL def_bool y

config ARCH_HIBERNATION_POSSIBLE diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 831fa4a12159..44096406d65d 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -35,7 +35,7 @@ #include <asm/cpu.h> #endif

-#define ACPI_IDLE_STATE_START (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX) ? 1 : 0) +#define ACPI_IDLE_STATE_START (IS_ENABLED(CONFIG_ARCH_HAS_OPTIMIZED_POLL) ? 1 : 0)

static unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER; module_param(max_cstate, uint, 0400); @@ -782,7 +782,7 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr) if (max_cstate == 0) max_cstate = 1;

- if (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX)) { + if (IS_ENABLED(CONFIG_ARCH_HAS_OPTIMIZED_POLL)) { cpuidle_poll_state_init(drv); count = 1; } else { diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index cac5997dca50..75f6e176bbc8 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -73,7 +73,7 @@ endmenu

config HALTPOLL_CPUIDLE tristate "Halt poll cpuidle driver" - depends on X86 && KVM_GUEST + depends on X86 && KVM_GUEST && ARCH_HAS_OPTIMIZED_POLL select CPU_IDLE_GOV_HALTPOLL default y help diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index d103342b7cfc..f29dfd1525b0 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -7,7 +7,7 @@ obj-y += cpuidle.o driver.o governor.o sysfs.o governors/ obj-$(CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED) += coupled.o obj-$(CONFIG_DT_IDLE_STATES) += dt_idle_states.o obj-$(CONFIG_DT_IDLE_GENPD) += dt_idle_genpd.o -obj-$(CONFIG_ARCH_HAS_CPU_RELAX) += poll_state.o +obj-$(CONFIG_ARCH_HAS_OPTIMIZED_POLL) += poll_state.o obj-$(CONFIG_HALTPOLL_CPUIDLE) += cpuidle-haltpoll.o

################################################################################## diff --git a/drivers/idle/Kconfig b/drivers/idle/Kconfig index 6707d2539fc4..6f9b1d48fede 100644 --- a/drivers/idle/Kconfig +++ b/drivers/idle/Kconfig @@ -4,6 +4,7 @@ config INTEL_IDLE depends on CPU_IDLE depends on X86 depends on CPU_SUP_INTEL + depends on ARCH_HAS_OPTIMIZED_POLL help Enable intel_idle, a cpuidle driver that includes knowledge of native Intel hardware idle features. The acpi_idle driver diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h index 3183aeb7f5b4..7e7e58a17b07 100644 --- a/include/linux/cpuidle.h +++ b/include/linux/cpuidle.h @@ -275,7 +275,7 @@ static inline void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, } #endif

-#if defined(CONFIG_CPU_IDLE) && defined(CONFIG_ARCH_HAS_CPU_RELAX) +#if defined(CONFIG_CPU_IDLE) && defined(CONFIG_ARCH_HAS_OPTIMIZED_POLL) void cpuidle_poll_state_init(struct cpuidle_driver *drv); #else static inline void cpuidle_poll_state_init(struct cpuidle_driver *drv) {}

From: Ankur Arora ankur.a.arora@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

Support a waited variant of polling on a conditional variable via smp_cond_relaxed_timeout().

This uses the __cmpwait_relaxed() primitive to do the actual waiting, when the wait can be guaranteed to not block forever (in case there are no stores to the waited for cacheline.) For this we depend on the availability of the event-stream.

For cases when the event-stream is unavailable, we fallback to a spin-waited implementation which is identical to the generic variant.

Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- arch/arm64/include/asm/barrier.h | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+)

diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 1ca947d5c939..ab2515ecd6ca 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -216,6 +216,60 @@ do { \ (typeof(*ptr))VAL; \ })

+#define __smp_cond_load_timeout_spin(ptr, cond_expr, \ + time_expr_ns, time_limit_ns) \ +({ \ + typeof(ptr) __PTR = (ptr); \ + __unqual_scalar_typeof(*ptr) VAL; \ + unsigned int __count = 0; \ + for (;;) { \ + VAL = READ_ONCE(*__PTR); \ + if (cond_expr) \ + break; \ + cpu_relax(); \ + if (__count++ < smp_cond_time_check_count) \ + continue; \ + if ((time_expr_ns) >= time_limit_ns) \ + break; \ + __count = 0; \ + } \ + (typeof(*ptr))VAL; \ +}) + +#define __smp_cond_load_timeout_wait(ptr, cond_expr, \ + time_expr_ns, time_limit_ns) \ +({ \ + typeof(ptr) __PTR = (ptr); \ + __unqual_scalar_typeof(*ptr) VAL; \ + for (;;) { \ + VAL = READ_ONCE(*__PTR); \ + if (cond_expr) \ + break; \ + __cmpwait_relaxed(__PTR, VAL); \ + if ((time_expr_ns) >= time_limit_ns) \ + break; \ + } \ + (typeof(*ptr))VAL; \ +}) + +#define smp_cond_load_relaxed_timeout(ptr, cond_expr, \ + time_expr_ns, time_limit_ns) \ +({ \ + __unqual_scalar_typeof(*ptr) _val; \ + \ + int __wfe = arch_timer_evtstrm_available(); \ + if (likely(__wfe)) \ + _val = __smp_cond_load_timeout_wait(ptr, cond_expr, \ + time_expr_ns, \ + time_limit_ns); \ + else \ + _val = __smp_cond_load_timeout_spin(ptr, cond_expr, \ + time_expr_ns, \ + time_limit_ns); \ + (typeof(*ptr))_val; \ +}) + + #include <asm-generic/barrier.h>

#endif /* __ASSEMBLY__ */

From: Ankur Arora ankur.a.arora@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

Polling in idle with poll_idle() needs TIF_POLLING_NRFLAG support, and a cheap mechanism to do the actual polling via smp_cond_load_relaxed_timeout().

Both of these are present on arm64. So, select ARCH_HAS_OPTIMIZED_POLL to enable it.

Enabling this should help reduce the cost of remote wakeups, since if the target sets TIF_POLLING_NRFLAG (as it does while polling in idle), the scheduler does those just by setting the need-resched bit. This contrasts with sending an IPI, and incurring the cost of handling the interrupt on the receiver.

Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index c11aa218c08e..350c431d0fc1 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -35,6 +35,7 @@ config ARM64 select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_NMI_SAFE_THIS_CPU_OPS select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE + select ARCH_HAS_OPTIMIZED_POLL select ARCH_HAS_NONLEAF_PMD_YOUNG if ARM64_HAFT select ARCH_HAS_PTE_DEVMAP select ARCH_HAS_PTE_SPECIAL

From: Joao Martins joao.m.martins@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

While initializing haltpoll we check if KVM supports the realtime hint and if idle is overridden at boot.

Both of these checks are x86 specific. So, in pursuit of making cpuidle-haltpoll architecture independent, move these checks out of common code.

Signed-off-by: Joao Martins joao.m.martins@oracle.com Signed-off-by: Mihai Carabas mihai.carabas@oracle.com Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- arch/x86/include/asm/cpuidle_haltpoll.h | 1 + arch/x86/kernel/kvm.c | 13 +++++++++++++ drivers/cpuidle/cpuidle-haltpoll.c | 12 +----------- include/linux/cpuidle_haltpoll.h | 5 +++++ 4 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/arch/x86/include/asm/cpuidle_haltpoll.h b/arch/x86/include/asm/cpuidle_haltpoll.h index c8b39c6716ff..8a0a12769c2e 100644 --- a/arch/x86/include/asm/cpuidle_haltpoll.h +++ b/arch/x86/include/asm/cpuidle_haltpoll.h @@ -4,5 +4,6 @@

void arch_haltpoll_enable(unsigned int cpu); void arch_haltpoll_disable(unsigned int cpu); +bool arch_haltpoll_want(bool force);

#endif diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index b8ab9ee5896c..4d45b1db340a 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -1149,4 +1149,17 @@ void arch_haltpoll_disable(unsigned int cpu) smp_call_function_single(cpu, kvm_enable_host_haltpoll, NULL, 1); } EXPORT_SYMBOL_GPL(arch_haltpoll_disable); + +bool arch_haltpoll_want(bool force) +{ + /* Do not load haltpoll if idle= is passed */ + if (boot_option_idle_override != IDLE_NO_OVERRIDE) + return false; + + if (!kvm_para_available()) + return false; + + return kvm_para_has_hint(KVM_HINTS_REALTIME) || force; +} +EXPORT_SYMBOL_GPL(arch_haltpoll_want); #endif diff --git a/drivers/cpuidle/cpuidle-haltpoll.c b/drivers/cpuidle/cpuidle-haltpoll.c index d8515d5c0853..d47906632ce3 100644 --- a/drivers/cpuidle/cpuidle-haltpoll.c +++ b/drivers/cpuidle/cpuidle-haltpoll.c @@ -15,7 +15,6 @@ #include <linux/cpuidle.h> #include <linux/module.h> #include <linux/sched/idle.h> -#include <linux/kvm_para.h> #include <linux/cpuidle_haltpoll.h>

static bool force __read_mostly; @@ -93,21 +92,12 @@ static void haltpoll_uninit(void) haltpoll_cpuidle_devices = NULL; }

-static bool haltpoll_want(void) -{ - return kvm_para_has_hint(KVM_HINTS_REALTIME) || force; -} - static int __init haltpoll_init(void) { int ret; struct cpuidle_driver *drv = &haltpoll_driver;

- /* Do not load haltpoll if idle= is passed */ - if (boot_option_idle_override != IDLE_NO_OVERRIDE) - return -ENODEV; - - if (!kvm_para_available() || !haltpoll_want()) + if (!arch_haltpoll_want(force)) return -ENODEV;

cpuidle_poll_state_init(drv); diff --git a/include/linux/cpuidle_haltpoll.h b/include/linux/cpuidle_haltpoll.h index d50c1e0411a2..68eb7a757120 100644 --- a/include/linux/cpuidle_haltpoll.h +++ b/include/linux/cpuidle_haltpoll.h @@ -12,5 +12,10 @@ static inline void arch_haltpoll_enable(unsigned int cpu) static inline void arch_haltpoll_disable(unsigned int cpu) { } + +static inline bool arch_haltpoll_want(bool force) +{ + return false; +} #endif #endif

From: Ankur Arora ankur.a.arora@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

The cpuidle-haltpoll driver and its namesake governor are selected under KVM_GUEST on X86. KVM_GUEST in-turn selects ARCH_CPUIDLE_HALTPOLL and defines the requisite arch_haltpoll_{enable,disable}() functions.

So remove the explicit dependence of HALTPOLL_CPUIDLE on KVM_GUEST, and instead use ARCH_CPUIDLE_HALTPOLL as proxy for architectural support for haltpoll.

Also change "halt poll" to "haltpoll" in one of the summary clauses, since the second form is used everywhere else.

Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- arch/x86/Kconfig | 1 + drivers/cpuidle/Kconfig | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f3c2d2933240..df023e1cb5dd 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -839,6 +839,7 @@ config KVM_GUEST

config ARCH_CPUIDLE_HALTPOLL def_bool n + depends on KVM_GUEST prompt "Disable host haltpoll when loading haltpoll driver" help If virtualized under KVM, disable host haltpoll. diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index 75f6e176bbc8..c1bebadf22bc 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -35,7 +35,6 @@ config CPU_IDLE_GOV_TEO

config CPU_IDLE_GOV_HALTPOLL bool "Haltpoll governor (for virtualized systems)" - depends on KVM_GUEST help This governor implements haltpoll idle state selection, to be used in conjunction with the haltpoll cpuidle driver, allowing @@ -72,8 +71,8 @@ source "drivers/cpuidle/Kconfig.riscv" endmenu

config HALTPOLL_CPUIDLE - tristate "Halt poll cpuidle driver" - depends on X86 && KVM_GUEST && ARCH_HAS_OPTIMIZED_POLL + tristate "Haltpoll cpuidle driver" + depends on ARCH_CPUIDLE_HALTPOLL && ARCH_HAS_OPTIMIZED_POLL select CPU_IDLE_GOV_HALTPOLL default y help

From: Ankur Arora ankur.a.arora@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

Add architectural support for the cpuidle-haltpoll driver by defining arch_haltpoll_*(). Also define ARCH_CPUIDLE_HALTPOLL to allow cpuidle-haltpoll to be selected.

Tested-by: Haris Okanovic harisokn@amazon.com Tested-by: Misono Tomohiro misono.tomohiro@fujitsu.com Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- arch/arm64/Kconfig | 6 ++++++ arch/arm64/include/asm/cpuidle_haltpoll.h | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 arch/arm64/include/asm/cpuidle_haltpoll.h

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 350c431d0fc1..57b4f89e7d36 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2618,6 +2618,12 @@ config ARCH_HIBERNATION_HEADER config ARCH_SUSPEND_POSSIBLE def_bool y

+config ARCH_CPUIDLE_HALTPOLL + bool "Enable selection of the cpuidle-haltpoll driver" + help + cpuidle-haltpoll allows for adaptive polling based on + current load before entering the idle state. + endmenu # "Power management options"

menu "CPU Power Management" diff --git a/arch/arm64/include/asm/cpuidle_haltpoll.h b/arch/arm64/include/asm/cpuidle_haltpoll.h new file mode 100644 index 000000000000..aa01ae9ad5dd --- /dev/null +++ b/arch/arm64/include/asm/cpuidle_haltpoll.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ARCH_HALTPOLL_H +#define _ARCH_HALTPOLL_H + +static inline void arch_haltpoll_enable(unsigned int cpu) { } +static inline void arch_haltpoll_disable(unsigned int cpu) { } + +static inline bool arch_haltpoll_want(bool force) +{ + /* + * Enabling haltpoll requires KVM support for arch_haltpoll_enable(), + * arch_haltpoll_disable(). + * + * Given that that's missing right now, only allow force loading for + * haltpoll. + */ + return force; +} +#endif

From: Ankur Arora ankur.a.arora@oracle.com

virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IB7PU3 CVE: NA

----------------------------------------------------

Support a WFET based implementation of the waited variant of smp_cond_load_relaxed_timeout().

Signed-off-by: Ankur Arora ankur.a.arora@oracle.com Signed-off-by: lishusen lishusen2@huawei.com --- arch/arm64/include/asm/barrier.h | 12 ++++++++---- arch/arm64/include/asm/cmpxchg.h | 26 +++++++++++++++++--------- 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index ab2515ecd6ca..8f7861ccab7e 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -12,6 +12,7 @@ #include <linux/kasan-checks.h>

#include <asm/alternative-macros.h> +#include <asm/delay-const.h>

#define __nops(n) ".rept " #n "\nnop\n.endr\n" #define nops(n) asm volatile(__nops(n)) @@ -198,7 +199,7 @@ do { \ VAL = READ_ONCE(*__PTR); \ if (cond_expr) \ break; \ - __cmpwait_relaxed(__PTR, VAL); \ + __cmpwait_relaxed(__PTR, VAL, ~0UL); \ } \ (typeof(*ptr))VAL; \ }) @@ -211,7 +212,7 @@ do { \ VAL = smp_load_acquire(__PTR); \ if (cond_expr) \ break; \ - __cmpwait_relaxed(__PTR, VAL); \ + __cmpwait_relaxed(__PTR, VAL, ~0UL); \ } \ (typeof(*ptr))VAL; \ }) @@ -241,11 +242,13 @@ do { \ ({ \ typeof(ptr) __PTR = (ptr); \ __unqual_scalar_typeof(*ptr) VAL; \ + const unsigned long __time_limit_cycles = \ + NSECS_TO_CYCLES(time_limit_ns); \ for (;;) { \ VAL = READ_ONCE(*__PTR); \ if (cond_expr) \ break; \ - __cmpwait_relaxed(__PTR, VAL); \ + __cmpwait_relaxed(__PTR, VAL, __time_limit_cycles); \ if ((time_expr_ns) >= time_limit_ns) \ break; \ } \ @@ -257,7 +260,8 @@ do { \ ({ \ __unqual_scalar_typeof(*ptr) _val; \ \ - int __wfe = arch_timer_evtstrm_available(); \ + int __wfe = arch_timer_evtstrm_available() || \ + alternative_has_cap_unlikely(ARM64_HAS_WFXT); \ if (likely(__wfe)) \ _val = __smp_cond_load_timeout_wait(ptr, cond_expr, \ time_expr_ns, \ diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index d7a540736741..bb842dab5d0e 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -210,7 +210,8 @@ __CMPXCHG_GEN(_mb)

#define __CMPWAIT_CASE(w, sfx, sz) \ static inline void __cmpwait_case_##sz(volatile void *ptr, \ - unsigned long val) \ + unsigned long val, \ + unsigned long time_limit_cycles) \ { \ unsigned long tmp; \ \ @@ -220,10 +221,12 @@ static inline void __cmpwait_case_##sz(volatile void *ptr, \ " ldxr" #sfx "\t%" #w "[tmp], %[v]\n" \ " eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \ " cbnz %" #w "[tmp], 1f\n" \ - " wfe\n" \ + ALTERNATIVE("wfe\n", \ + "msr s0_3_c1_c0_0, %[time_limit_cycles]\n", \ + ARM64_HAS_WFXT) \ "1:" \ : [tmp] "=&r" (tmp), [v] "+Q" (*(u##sz *)ptr) \ - : [val] "r" (val)); \ + : [val] "r" (val), [time_limit_cycles] "r" (time_limit_cycles));\ }

__CMPWAIT_CASE(w, b, 8); @@ -236,17 +239,22 @@ __CMPWAIT_CASE( , , 64); #define __CMPWAIT_GEN(sfx) \ static __always_inline void __cmpwait##sfx(volatile void *ptr, \ unsigned long val, \ + unsigned long time_limit_cycles, \ int size) \ { \ switch (size) { \ case 1: \ - return __cmpwait_case##sfx##_8(ptr, (u8)val); \ + return __cmpwait_case##sfx##_8(ptr, (u8)val, \ + time_limit_cycles); \ case 2: \ - return __cmpwait_case##sfx##_16(ptr, (u16)val); \ + return __cmpwait_case##sfx##_16(ptr, (u16)val, \ + time_limit_cycles); \ case 4: \ - return __cmpwait_case##sfx##_32(ptr, val); \ + return __cmpwait_case##sfx##_32(ptr, val, \ + time_limit_cycles); \ case 8: \ - return __cmpwait_case##sfx##_64(ptr, val); \ + return __cmpwait_case##sfx##_64(ptr, val, \ + time_limit_cycles); \ default: \ BUILD_BUG(); \ } \ @@ -258,7 +266,7 @@ __CMPWAIT_GEN()

#undef __CMPWAIT_GEN

-#define __cmpwait_relaxed(ptr, val) \ - __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) +#define __cmpwait_relaxed(ptr, val, time_limit_cycles) \ + __cmpwait((ptr), (unsigned long)(val), time_limit_cycles, sizeof(*(ptr)))

#endif /* __ASM_CMPXCHG_H */