From: Yeoreum Yun <yeoreum.yun@arm.com> mainline inclusion from mainline-7.1 commit 44adf2bf40efe56c68c1563779ab2047eb0a57ea category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/8904 CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ---------------------------------------------------------------------- Current futex atomic operations are implemented using LL/SC instructions while temporarily clearing PSTATE.PAN and setting PSTATE.TCO (if KASAN_HW_TAGS is enabled). With Armv9.6, FEAT_LSUI provides atomic instructions for user memory access in the kernel without the need for PSTATE bits toggling. Use the FEAT_LSUI instructions to implement the futex atomic operations. Note that some futex operations do not have a matching LSUI instruction, (eor or word-sized cmpxchg). For such cases, use cas{al}t to implement the operation. Signed-off-by: Yeoreum Yun <yeoreum.yun@arm.com> [catalin.marinas@arm.com: add comment on -EAGAIN in __lsui_futex_cmpxchg()] Signed-off-by: Catalin Marinas <catalin.marinas@arm.com> Signed-off-by: Qinxin Xia <xiaqinxin@huawei.com> Signed-off-by: Hongye Lin <linhongye@h-partners.com> --- arch/arm64/include/asm/futex.h | 160 ++++++++++++++++++++++++++++++++- arch/arm64/include/asm/lsui.h | 27 ++++++ 2 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/lsui.h diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 321242e40952..d1d2ff9d323a 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -9,6 +9,7 @@ #include <linux/uaccess.h> #include <asm/errno.h> +#include <asm/lsui.h> #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */ @@ -89,11 +90,166 @@ __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) return ret; } +#ifdef CONFIG_ARM64_LSUI + +/* + * Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as + * PAN toggling is not required. + */ + +#define LSUI_FUTEX_ATOMIC_OP(op, asm_op) \ +static __always_inline int \ +__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ +{ \ + int ret = 0; \ + int oldval; \ + \ + uaccess_ttbr0_enable(); \ + \ + asm volatile("// __lsui_futex_atomic_" #op "\n" \ + __LSUI_PREAMBLE \ +"1: " #asm_op "al %w[oparg], %w[oldval], %[uaddr]\n" \ +"2:\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) \ + : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), \ + [oldval] "=r" (oldval) \ + : [oparg] "r" (oparg) \ + : "memory"); \ + \ + uaccess_ttbr0_disable(); \ + \ + if (!ret) \ + *oval = oldval; \ + return ret; \ +} + +LSUI_FUTEX_ATOMIC_OP(add, ldtadd) +LSUI_FUTEX_ATOMIC_OP(or, ldtset) +LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr) +LSUI_FUTEX_ATOMIC_OP(set, swpt) + +static __always_inline int +__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) +{ + int ret = 0; + + uaccess_ttbr0_enable(); + + asm volatile("// __lsui_cmpxchg64\n" + __LSUI_PREAMBLE +"1: casalt %[oldval], %[newval], %[uaddr]\n" +"2:\n" + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) + : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), + [oldval] "+r" (*oldval) + : [newval] "r" (newval) + : "memory"); + + uaccess_ttbr0_disable(); + + return ret; +} + +static __always_inline int +__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + u64 __user *uaddr64; + bool futex_pos, other_pos; + u32 other, orig_other; + union { + u32 futex[2]; + u64 raw; + } oval64, orig64, nval64; + + uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64)); + futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64)); + other_pos = !futex_pos; + + oval64.futex[futex_pos] = oldval; + if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos)) + return -EFAULT; + + orig64.raw = oval64.raw; + + nval64.futex[futex_pos] = newval; + nval64.futex[other_pos] = oval64.futex[other_pos]; + + if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw)) + return -EFAULT; + + oldval = oval64.futex[futex_pos]; + other = oval64.futex[other_pos]; + orig_other = orig64.futex[other_pos]; + + if (other != orig_other) + return -EAGAIN; + + *oval = oldval; + + return 0; +} + +static __always_inline int +__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) +{ + /* + * Undo the bitwise negation applied to the oparg passed from + * arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN. + */ + return __lsui_futex_atomic_andnot(~oparg, uaddr, oval); +} + +static __always_inline int +__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) +{ + u32 oldval, newval, val; + int ret, i; + + if (get_user(oldval, uaddr)) + return -EFAULT; + + /* + * there are no ldteor/stteor instructions... + */ + for (i = 0; i < FUTEX_MAX_LOOPS; i++) { + newval = oldval ^ oparg; + + ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val); + switch (ret) { + case -EFAULT: + return ret; + case -EAGAIN: + continue; + } + + if (val == oldval) { + *oval = val; + return 0; + } + + oldval = val; + } + + return -EAGAIN; +} + +static __always_inline int +__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + /* + * Callers of futex_atomic_cmpxchg_inatomic() already retry on + * -EAGAIN, no need for another loop of max retries. + */ + return __lsui_cmpxchg32(uaddr, oldval, newval, oval); +} +#endif /* CONFIG_ARM64_LSUI */ + + #define FUTEX_ATOMIC_OP(op) \ static __always_inline int \ __futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ { \ - return __llsc_futex_atomic_##op(oparg, uaddr, oval); \ + return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \ } FUTEX_ATOMIC_OP(add) @@ -105,7 +261,7 @@ FUTEX_ATOMIC_OP(set) static __always_inline int __futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) { - return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval); + return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval); } static inline int diff --git a/arch/arm64/include/asm/lsui.h b/arch/arm64/include/asm/lsui.h new file mode 100644 index 000000000000..8f0d81953eb6 --- /dev/null +++ b/arch/arm64/include/asm/lsui.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_LSUI_H +#define __ASM_LSUI_H + +#include <linux/compiler_types.h> +#include <linux/stringify.h> +#include <asm/alternative.h> +#include <asm/alternative-macros.h> +#include <asm/cpucaps.h> + +#define __LSUI_PREAMBLE ".arch_extension lsui\n" + +#ifdef CONFIG_ARM64_LSUI + +#define __lsui_llsc_body(op, ...) \ +({ \ + alternative_has_cap_unlikely(ARM64_HAS_LSUI) ? \ + __lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \ +}) + +#else /* CONFIG_ARM64_LSUI */ + +#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__) + +#endif /* CONFIG_ARM64_LSUI */ + +#endif /* __ASM_LSUI_H */ -- 2.33.0