--- .../admin-guide/kernel-parameters.txt | 3 + arch/arm64/Kconfig | 11 ++ arch/arm64/include/asm/asm-uaccess.h | 13 ++ arch/arm64/include/asm/cputype.h | 2 + arch/arm64/include/asm/uaccess.h | 21 ++- arch/arm64/kernel/cpufeature.c | 51 +++++ arch/arm64/lib/copy_from_user.S | 33 ++++ arch/arm64/lib/copy_template_opt.S | 175 ++++++++++++++++++ arch/arm64/tools/cpucaps | 4 +- 9 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 arch/arm64/lib/copy_template_opt.S diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b443a9665e03..e2adafc903c1 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -423,6 +423,9 @@ arm64.nomops [ARM64] Unconditionally disable Memory Copy and Memory Set instructions support + copy_opt_disable [ARM64] Disable optimized copy_from_user + implementation + arm64.nomte [ARM64] Unconditionally disable Memory Tagging Extension support diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 81d2baafdcd6..757d4bbe4251 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1317,6 +1317,14 @@ config HISILICON_ERRATUM_165010801 system watchdog hardlockup detection might be triggered. The arch_timer driver addresses this by proactively increasing affected interrupt priorities. +config ARM64_COPY_FROM_USER_OPT + bool "Optimized copy_from_user for Hisilicon CPUs" + depends on ARCH_HISI + default y + help + Enable an optimized copy_from_user implementation for Hisilicon + CPUs that benefit from LDP instruction based copy routines. + config QCOM_FALKOR_ERRATUM_1003 bool "Falkor E1003: Incorrect translation due to ASID change" default y @@ -2061,6 +2069,9 @@ config ARM64_PAN config AS_HAS_LSE_ATOMICS def_bool $(as-instr,.arch_extension lse) +config AS_HAS_LSUI + def_bool $(as-instr,.arch_extension lsui) + config ARM64_LSE_ATOMICS bool default ARM64_USE_LSE_ATOMICS diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index 7bbebfa5b710..342e77b0d7f2 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -94,4 +94,17 @@ alternative_else_nop_endif _asm_extable_uaccess 8888b, \l; .endm + + .macro user_ldpair l, reg1, reg2, addr, val +8888: ldp \reg1, \reg2, [\addr, \val]; + + _asm_extable_uaccess 8888b, \l; + .endm + + .macro user_ldtpair l, reg1, reg2, addr, val +8888: .arch_extension lsui + ldtp \reg1, \reg2, [\addr, \val]; + + _asm_extable_uaccess 8888b, \l; + .endm #endif diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 490c2ac36ac0..16c4d867f199 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -142,6 +142,7 @@ #define HISI_CPU_PART_TSV110 0xD01 #define HISI_CPU_PART_LINXICORE9100 0xD02 +#define HISI_CPU_PART_HIP11 0xD22 #define HISI_CPU_PART_HIP12 0xD06 #define APPLE_CPU_PART_M1_ICESTORM 0x022 @@ -230,6 +231,7 @@ #define MIDR_FUJITSU_A64FX MIDR_CPU_MODEL(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) #define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) #define MIDR_HISI_LINXICORE9100 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_LINXICORE9100) +#define MIDR_HISI_HIP11 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_HIP11) #define MIDR_HISI_HIP12 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_HIP12) #define MIDR_APPLE_M1_ICESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) #define MIDR_APPLE_M1_FIRESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index dd0877a75922..ed307e1425b4 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -392,12 +392,29 @@ do { \ } while(0) extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT +#define COPY_OPT_THRESHOLD 4096 +static __always_inline bool use_copy_opt(unsigned long n) +{ + if (alternative_has_cap_unlikely(ARM64_HAS_LSUI)) + return true; + return alternative_has_cap_unlikely(ARM64_HAS_COPY_OPT) && n >= COPY_OPT_THRESHOLD; +} +extern unsigned long __must_check __arch_copy_from_user_opt(void *to, + const void __user *from, unsigned long n); +#else +static __always_inline bool use_copy_opt(unsigned long n) { return false; } +#endif #define raw_copy_from_user(to, from, n) \ ({ \ unsigned long __acfu_ret; \ uaccess_ttbr0_enable(); \ - __acfu_ret = __arch_copy_from_user((to), \ - __uaccess_mask_ptr(from), (n)); \ + if (use_copy_opt(n)) \ + __acfu_ret = __arch_copy_from_user_opt((to), \ + __uaccess_mask_ptr(from), (n)); \ + else \ + __acfu_ret = __arch_copy_from_user((to), \ + __uaccess_mask_ptr(from), (n)); \ uaccess_ttbr0_disable(); \ __acfu_ret; \ }) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index c8e3f16387cb..1ed2850ba72d 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2338,6 +2338,37 @@ static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) set_pstate_dit(1); } +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT +static bool copy_opt_disable __ro_after_init; + +static int __init copy_opt_disable_param(char *str) +{ + copy_opt_disable = true; + return 0; +} +early_param("copy_opt_disable", copy_opt_disable_param); + +static bool has_copy_opt(const struct arm64_cpu_capabilities *entry, int scope) +{ + static const struct midr_range copy_opt_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_HISI_LINXICORE9100), + MIDR_ALL_VERSIONS(MIDR_HISI_HIP11), + MIDR_ALL_VERSIONS(MIDR_HISI_HIP12), + { /* sentinel */ } + }; + + if (copy_opt_disable) + return false; + + return is_midr_in_range_list(copy_opt_cpus); // todo +} + +static void cpu_enable_copy_opt(const struct arm64_cpu_capabilities *__unused) +{ + pr_info("copy_from_user: optimized implementation enabled\n"); +} +#endif + static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused) { sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn); @@ -3154,6 +3185,26 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_arch_xcall_xint_support, .cpu_enable = cpu_enable_arch_xcall_xint, }, +#endif + { + .desc = "Unprivileged Load Store Instructions", + .capability = ARM64_HAS_LSUI, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .sys_reg = SYS_ID_AA64ISAR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = 4, + .field_width = 4, + .min_field_value = 1, + .matches = has_cpuid_feature, + }, +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + { + .desc = "Optimized copy_from_user", + .capability = ARM64_HAS_COPY_OPT, + .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, + .matches = has_copy_opt, + .cpu_enable = cpu_enable_copy_opt, + }, #endif {}, }; diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 34e317907524..d090836aa9c0 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -52,6 +52,18 @@ stp \reg1, \reg2, [\ptr], \val .endm + .macro ldp2 reg1, reg2, ptr, val + alternative_if_not ARM64_HAS_LSUI + user_ldpair 9997f, \reg1, \reg2, \ptr, \val + alternative_else + user_ldtpair 9997f, \reg1, \reg2, \ptr, \val + alternative_endif + .endm + + .macro stp2 reg1, reg2, ptr, val + stp \reg1, \reg2, [\ptr, \val] + .endm + end .req x5 srcin .req x15 SYM_FUNC_START(__arch_copy_from_user) @@ -71,3 +83,24 @@ USER(9998f, ldtrb tmp1w, [srcin]) ret SYM_FUNC_END(__arch_copy_from_user) EXPORT_SYMBOL(__arch_copy_from_user) + +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT +SYM_FUNC_START(__arch_copy_from_user_opt) + add end, x0, x2 + mov srcin, x1 + +#include "copy_template_opt.S" + mov x0, #0 // Nothing to copy + ret + + // Exception fixups +9997: cmp dst, dstin + b.ne 9998f + // Before being absolutely sure we couldn't copy anything, try harder +USER(9998f, ldtrb tmp1w, [srcin]) + strb tmp1w, [dst], #1 +9998: sub x0, end, dst // bytes not copied + ret +SYM_FUNC_END(__arch_copy_from_user_opt) +EXPORT_SYMBOL(__arch_copy_from_user_opt) +#endif diff --git a/arch/arm64/lib/copy_template_opt.S b/arch/arm64/lib/copy_template_opt.S new file mode 100644 index 000000000000..df6c2be11dc0 --- /dev/null +++ b/arch/arm64/lib/copy_template_opt.S @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 - Huawei Ltd. + */ + +/* + * Copy a buffer from src to dest (alignment handled by the hardware) + * + * Parameters: + * x0 - dest + * x1 - src + * x2 - n + * Returns: + * x0 - dest + * + * Note: register aliases (dstin, src, count, tmp1, etc.) are defined by + * copy_template.S which is included earlier in copy_from_user.S. + */ + + mov dst, dstin + cmp count, #16 + /*When memory length is less than 16, the accesses are not aligned.*/ + b.lo .Ltiny15_opt + + neg tmp2, src + ands tmp2, tmp2, #15/* Bytes to reach alignment. */ + b.eq .LSrcAligned_opt + sub count, count, tmp2 + /* + * Copy the leading memory data from src to dst in an increasing + * address order.By this way,the risk of overwriting the source + * memory data is eliminated when the distance between src and + * dst is less than 16. The memory accesses here are alignment. + */ + tbz tmp2, #0, 1f + ldrb1 tmp1w, src, #1 + strb1 tmp1w, dst, #1 +1: + tbz tmp2, #1, 2f + ldrh1 tmp1w, src, #2 + strh1 tmp1w, dst, #2 +2: + tbz tmp2, #2, 3f + ldr1 tmp1w, src, #4 + str1 tmp1w, dst, #4 +3: + tbz tmp2, #3, .LSrcAligned_opt + ldr1 tmp1, src, #8 + str1 tmp1, dst, #8 + +.LSrcAligned_opt: + cmp count, #64 + b.ge .Lcpy_over64_opt + /* + * Deal with small copies quickly by dropping straight into the + * exit block. + */ +.Ltail63_opt: + /* + * Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. + */ + ands tmp1, count, #0x30 + b.eq .Ltiny15_opt + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + add src, src, #16 + add dst, dst, #16 +1: + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + add src, src, #16 + add dst, dst, #16 +2: + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + add src, src, #16 + add dst, dst, #16 +.Ltiny15_opt: + /* + * Prefer to break one ldp/stp into several load/store to access + * memory in an increasing address order,rather than to load/store 16 + * bytes from (src-16) to (dst-16) and to backward the src to aligned + * address,which way is used in original cortex memcpy. If keeping + * the original memcpy process here, memmove need to satisfy the + * precondition that src address is at least 16 bytes bigger than dst + * address,otherwise some source data will be overwritten when memove + * call memcpy directly. To make memmove simpler and decouple the + * memcpy's dependency on memmove, withdrew the original process. + */ + tbz count, #3, 1f + ldr1 tmp1, src, #8 + str1 tmp1, dst, #8 +1: + tbz count, #2, 2f + ldr1 tmp1w, src, #4 + str1 tmp1w, dst, #4 +2: + tbz count, #1, 3f + ldrh1 tmp1w, src, #2 + strh1 tmp1w, dst, #2 +3: + tbz count, #0, .Lexitfunc_opt + ldrb1 tmp1w, src, #1 + strb1 tmp1w, dst, #1 + + b .Lexitfunc_opt + +.Lcpy_over64_opt: + subs count, count, #128 + b.ge .Lcpy_body_large_opt + /* + * Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. + */ + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + ldp2 B_l, B_h, src, #16 + ldp2 C_l, C_h, src, #32 + stp2 B_l, B_h, dst, #16 + stp2 C_l, C_h, dst, #32 + ldp2 D_l, D_h, src, #48 + stp2 D_l, D_h, dst, #48 + add src, src, #64 + add dst, dst, #64 + + tst count, #0x3f + b.ne .Ltail63_opt + b .Lexitfunc_opt + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. + */ + .p2align L1_CACHE_SHIFT +.Lcpy_body_large_opt: + + /* pre-get 64 bytes data. */ + ldp2 A_l, A_h, src, #0 + ldp2 B_l, B_h, src, #16 + ldp2 C_l, C_h, src, #32 + ldp2 D_l, D_h, src, #48 + add src, src, #64 +1: + /* + * interlace the load of next 64 bytes data block with store of the last + * loaded 64 bytes data. + */ + stp2 A_l, A_h, dst, #0 + ldp2 A_l, A_h, src, #0 + stp2 B_l, B_h, dst, #16 + ldp2 B_l, B_h, src, #16 + stp2 C_l, C_h, dst, #32 + ldp2 C_l, C_h, src, #32 + stp2 D_l, D_h, dst, #48 + ldp2 D_l, D_h, src, #48 + add dst, dst, #64 + add src, src, #64 + subs count, count, #64 + b.ge 1b + + /* Post-loop: store the last block of data using stp2 */ + /* (without post-increment) */ + stp2 A_l, A_h, dst, #0 + stp2 B_l, B_h, dst, #16 + stp2 C_l, C_h, dst, #32 + stp2 D_l, D_h, dst, #48 + add dst, dst, #64 + + tst count, #0x3f + b.ne .Ltail63_opt +.Lexitfunc_opt: diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index d8f2db273def..f6445266e886 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -114,8 +114,8 @@ HAS_LS64 HAS_LS64_V HAS_HW_XCALL_XINT WORKAROUND_PHYTIUM_FT3386 -KABI_RESERVE_7 -KABI_RESERVE_8 +HAS_COPY_OPT +HAS_LSUI KABI_RESERVE_9 KABI_RESERVE_10 KABI_RESERVE_11 -- 2.53.0