From: Marc Zyngier <maz@kernel.org> mainline inclusion from mainline-6.9 commit aeddd5b214c8ce7271d7da7aa64ff20fb9032966 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/8904 CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ---------------------------------------------------------------------- Despite having the control bits for FEAT_SPECRES and FEAT_PACM, the ID registers fields are either incomplete or missing. Fix it. Reviewed-by: Mark Brown <broonie@kernel.org> Signed-off-by: Marc Zyngier <maz@kernel.org> Acked-by: Catalin Marinas <catalin.marinas@arm.com> Link: https://lore.kernel.org/r/20240214131827.2856277-2-maz@kernel.org Signed-off-by: Oliver Upton <oliver.upton@linux.dev> Signed-off-by: Qinxin Xia <xiaqinxin@huawei.com> Signed-off-by: Hongye Lin <linhongye@h-partners.com> --- arch/arm64/tools/sysreg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index c2e7bdedb308..0a7ad72cf9a6 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1489,6 +1489,7 @@ EndEnum UnsignedEnum 43:40 SPECRES 0b0000 NI 0b0001 IMP + 0b0010 COSP_RCTX EndEnum UnsignedEnum 39:36 SB 0b0000 NI @@ -1612,7 +1613,12 @@ EndEnum EndSysreg Sysreg ID_AA64ISAR3_EL1 3 0 0 6 3 -Res0 63:12 +Res0 63:16 +UnsignedEnum 15:12 PACM + 0b0000 NI + 0b0001 TRIVIAL_IMP + 0b0010 FULL_IMP +EndEnum UnsignedEnum 11:8 TLBIW 0b0000 NI 0b0001 IMP -- 2.33.0

From: Yeoreum Yun <yeoreum.yun@arm.com> mainline inclusion from mainline-7.1 commit 7181f718cb0fd47b37d13aad8744cbd6da9f1cbe category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/8904 CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ---------------------------------------------------------------------- Since Armv9.6, FEAT_LSUI introduces atomic instructions that allow privileged code to access user memory without clearing the PSTATE.PAN bit. Add CPU feature detection for FEAT_LSUI. Conflicts: arch/arm64/include/asm/cpucaps.h [OLK-6.6 does not have the cpucap_is_possible() mechanism; the ARM64_HAS_LSUI capability is gated by the CONFIG_ARM64_LSUI #ifdef guard around its arm64_features[] entry instead, consistent with how other config-dependent capabilities are handled in this tree.] Signed-off-by: Yeoreum Yun <yeoreum.yun@arm.com> [catalin.marinas@arm.com: Remove commit log references to SW_PAN] Signed-off-by: Catalin Marinas <catalin.marinas@arm.com> Signed-off-by: Qinxin Xia <xiaqinxin@huawei.com> Signed-off-by: Hongye Lin <linhongye@h-partners.com> --- arch/arm64/kernel/cpufeature.c | 10 ++++++++++ arch/arm64/tools/cpucaps | 1 + 2 files changed, 11 insertions(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index faae2af65de8..86e5404fbfae 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -242,6 +242,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar3[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_LSUI_SHIFT, 4, ID_AA64ISAR3_EL1_LSUI_NI), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FAMINMAX_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -3222,6 +3223,15 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LS64, LS64_V) }, #endif +#ifdef CONFIG_ARM64_LSUI + { + .desc = "Unprivileged Load Store Instructions (LSUI)", + .capability = ARM64_HAS_LSUI, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64ISAR3_EL1, LSUI, IMP) + }, +#endif #ifdef CONFIG_ACTLR_XCALL_XINT { .desc = "Hardware Xcall and Xint Support", diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index ce0c60d48dff..ce3f6beaa47b 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -112,6 +112,7 @@ HAS_XCALL HAS_XINT HAS_LS64 HAS_LS64_V +HAS_LSUI HAS_HW_XCALL_XINT WORKAROUND_PHYTIUM_FT3386 HAS_COPY_OPT -- 2.33.0

From: Yeoreum Yun <yeoreum.yun@arm.com> mainline inclusion from mainline-7.1 commit 44adf2bf40efe56c68c1563779ab2047eb0a57ea category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/8904 CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ---------------------------------------------------------------------- Current futex atomic operations are implemented using LL/SC instructions while temporarily clearing PSTATE.PAN and setting PSTATE.TCO (if KASAN_HW_TAGS is enabled). With Armv9.6, FEAT_LSUI provides atomic instructions for user memory access in the kernel without the need for PSTATE bits toggling. Use the FEAT_LSUI instructions to implement the futex atomic operations. Note that some futex operations do not have a matching LSUI instruction, (eor or word-sized cmpxchg). For such cases, use cas{al}t to implement the operation. Signed-off-by: Yeoreum Yun <yeoreum.yun@arm.com> [catalin.marinas@arm.com: add comment on -EAGAIN in __lsui_futex_cmpxchg()] Signed-off-by: Catalin Marinas <catalin.marinas@arm.com> Signed-off-by: Qinxin Xia <xiaqinxin@huawei.com> Signed-off-by: Hongye Lin <linhongye@h-partners.com> --- arch/arm64/include/asm/futex.h | 160 ++++++++++++++++++++++++++++++++- arch/arm64/include/asm/lsui.h | 27 ++++++ 2 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/lsui.h diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 321242e40952..d1d2ff9d323a 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -9,6 +9,7 @@ #include <linux/uaccess.h> #include <asm/errno.h> +#include <asm/lsui.h> #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */ @@ -89,11 +90,166 @@ __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) return ret; } +#ifdef CONFIG_ARM64_LSUI + +/* + * Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as + * PAN toggling is not required. + */ + +#define LSUI_FUTEX_ATOMIC_OP(op, asm_op) \ +static __always_inline int \ +__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ +{ \ + int ret = 0; \ + int oldval; \ + \ + uaccess_ttbr0_enable(); \ + \ + asm volatile("// __lsui_futex_atomic_" #op "\n" \ + __LSUI_PREAMBLE \ +"1: " #asm_op "al %w[oparg], %w[oldval], %[uaddr]\n" \ +"2:\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) \ + : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), \ + [oldval] "=r" (oldval) \ + : [oparg] "r" (oparg) \ + : "memory"); \ + \ + uaccess_ttbr0_disable(); \ + \ + if (!ret) \ + *oval = oldval; \ + return ret; \ +} + +LSUI_FUTEX_ATOMIC_OP(add, ldtadd) +LSUI_FUTEX_ATOMIC_OP(or, ldtset) +LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr) +LSUI_FUTEX_ATOMIC_OP(set, swpt) + +static __always_inline int +__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) +{ + int ret = 0; + + uaccess_ttbr0_enable(); + + asm volatile("// __lsui_cmpxchg64\n" + __LSUI_PREAMBLE +"1: casalt %[oldval], %[newval], %[uaddr]\n" +"2:\n" + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) + : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), + [oldval] "+r" (*oldval) + : [newval] "r" (newval) + : "memory"); + + uaccess_ttbr0_disable(); + + return ret; +} + +static __always_inline int +__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + u64 __user *uaddr64; + bool futex_pos, other_pos; + u32 other, orig_other; + union { + u32 futex[2]; + u64 raw; + } oval64, orig64, nval64; + + uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64)); + futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64)); + other_pos = !futex_pos; + + oval64.futex[futex_pos] = oldval; + if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos)) + return -EFAULT; + + orig64.raw = oval64.raw; + + nval64.futex[futex_pos] = newval; + nval64.futex[other_pos] = oval64.futex[other_pos]; + + if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw)) + return -EFAULT; + + oldval = oval64.futex[futex_pos]; + other = oval64.futex[other_pos]; + orig_other = orig64.futex[other_pos]; + + if (other != orig_other) + return -EAGAIN; + + *oval = oldval; + + return 0; +} + +static __always_inline int +__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) +{ + /* + * Undo the bitwise negation applied to the oparg passed from + * arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN. + */ + return __lsui_futex_atomic_andnot(~oparg, uaddr, oval); +} + +static __always_inline int +__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) +{ + u32 oldval, newval, val; + int ret, i; + + if (get_user(oldval, uaddr)) + return -EFAULT; + + /* + * there are no ldteor/stteor instructions... + */ + for (i = 0; i < FUTEX_MAX_LOOPS; i++) { + newval = oldval ^ oparg; + + ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val); + switch (ret) { + case -EFAULT: + return ret; + case -EAGAIN: + continue; + } + + if (val == oldval) { + *oval = val; + return 0; + } + + oldval = val; + } + + return -EAGAIN; +} + +static __always_inline int +__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + /* + * Callers of futex_atomic_cmpxchg_inatomic() already retry on + * -EAGAIN, no need for another loop of max retries. + */ + return __lsui_cmpxchg32(uaddr, oldval, newval, oval); +} +#endif /* CONFIG_ARM64_LSUI */ + + #define FUTEX_ATOMIC_OP(op) \ static __always_inline int \ __futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ { \ - return __llsc_futex_atomic_##op(oparg, uaddr, oval); \ + return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \ } FUTEX_ATOMIC_OP(add) @@ -105,7 +261,7 @@ FUTEX_ATOMIC_OP(set) static __always_inline int __futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) { - return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval); + return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval); } static inline int diff --git a/arch/arm64/include/asm/lsui.h b/arch/arm64/include/asm/lsui.h new file mode 100644 index 000000000000..8f0d81953eb6 --- /dev/null +++ b/arch/arm64/include/asm/lsui.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_LSUI_H +#define __ASM_LSUI_H + +#include <linux/compiler_types.h> +#include <linux/stringify.h> +#include <asm/alternative.h> +#include <asm/alternative-macros.h> +#include <asm/cpucaps.h> + +#define __LSUI_PREAMBLE ".arch_extension lsui\n" + +#ifdef CONFIG_ARM64_LSUI + +#define __lsui_llsc_body(op, ...) \ +({ \ + alternative_has_cap_unlikely(ARM64_HAS_LSUI) ? \ + __lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \ +}) + +#else /* CONFIG_ARM64_LSUI */ + +#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__) + +#endif /* CONFIG_ARM64_LSUI */ + +#endif /* __ASM_LSUI_H */ -- 2.33.0

From: Yeoreum Yun <yeoreum.yun@arm.com> mainline inclusion from mainline-7.1 commit 377609ae8b6ac2ef522c9dcecfa20a7123116dbf category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/8904 CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ---------------------------------------------------------------------- Since Armv9.6, FEAT_LSUI supplies the load/store instructions for previleged level to access to access user memory without clearing PSTATE.PAN bit. Add Kconfig option entry for FEAT_LSUI. Signed-off-by: Yeoreum Yun <yeoreum.yun@arm.com> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com> Signed-off-by: Qinxin Xia <xiaqinxin@huawei.com> Signed-off-by: Hongye Lin <linhongye@h-partners.com> --- arch/arm64/Kconfig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 954cc42589f3..ca1041ff7f8c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2486,6 +2486,26 @@ config ARM64_HDBSS endmenu # "ARMv9.5 architectural features" +config AS_HAS_LSUI + def_bool $(as-instr,.arch_extension lsui) + help + Supported by LLVM 20+ and binutils 2.45+. + +menu "ARMv9.6 architectural features" + +config ARM64_LSUI + bool "Support Unprivileged Load Store Instructions (LSUI)" + default y + depends on AS_HAS_LSUI && !CPU_BIG_ENDIAN + help + The Unprivileged Load Store Instructions (LSUI) provides + variants load/store instructions that access user-space memory + from the kernel without clearing PSTATE.PAN bit. + + This feature is supported by LLVM 20+ and binutils 2.45+. + +endmenu # "ARMv9.6 architectural feature" + config ARM64_HISI_IPIV bool "Enable support for IPIV" default y -- 2.33.0