hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9132 ------------------------------------------------------- Introduce CONFIG_ARM64_COPY_FROM_USER_OPT to control compilation of the optimized copy_from_user implementation. Signed-off-by: Qi Xi <xiqi2@huawei.com> --- arch/arm64/Kconfig.turbo | 11 +++++++++++ arch/arm64/configs/openeuler_defconfig | 1 + arch/arm64/include/asm/asm-uaccess.h | 4 ++++ arch/arm64/include/asm/uaccess.h | 11 +++++++++++ arch/arm64/kernel/cpufeature.c | 5 +++++ arch/arm64/lib/copy_from_user.S | 4 ++++ 6 files changed, 36 insertions(+) diff --git a/arch/arm64/Kconfig.turbo b/arch/arm64/Kconfig.turbo index 778ea1025c2c..769fde5fffd7 100644 --- a/arch/arm64/Kconfig.turbo +++ b/arch/arm64/Kconfig.turbo @@ -84,4 +84,15 @@ config DYNAMIC_XCALL and a kernel module which provides customized implementation. +config ARM64_COPY_FROM_USER_OPT + bool "Hisilicon Optimized Copy From User enabled" + depends on ARCH_HISI + default y + help + Enable an optimized copy_from_user implementation for + supported cores (e.g. Hisilicon HIP11, Hisilicon HIP12, + Linxicore9100). This trades a small PAN toggle overhead + for higher throughput on large copies. This can be + disabled at boot via copy_opt_disable. + endmenu # "Turbo features selection" diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 8ecb06017ae9..5b754eed8ca1 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -400,6 +400,7 @@ CONFIG_DEBUG_FEATURE_BYPASS=y CONFIG_SECURITY_FEATURE_BYPASS=y CONFIG_ACTLR_XCALL_XINT=y CONFIG_DYNAMIC_XCALL=y +CONFIG_ARM64_COPY_FROM_USER_OPT=y # end of Turbo features selection # diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index 4edf6c7dc56f..52388026b907 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -95,10 +95,14 @@ alternative_else_nop_endif _asm_extable_uaccess 8888b, \l; .endm +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + .macro user_ldst_pair_index l, inst, reg1, reg2, addr, val 8888: \inst \reg1, \reg2, [\addr, \val]; _asm_extable_uaccess 8888b, \l; .endm +#endif /* CONFIG_ARM64_COPY_FROM_USER_OPT */ + #endif diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 8e2899b36b00..149358ae5506 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -391,6 +391,8 @@ do { \ } while (0); \ } while(0) +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + #define COPY_OPT_THRESHOLD 4096 static __always_inline bool use_copy_opt(unsigned long n) @@ -399,6 +401,15 @@ static __always_inline bool use_copy_opt(unsigned long n) alternative_has_cap_unlikely(ARM64_HAS_COPY_OPT); } +#else /* !CONFIG_ARM64_COPY_FROM_USER_OPT */ + +static __always_inline bool use_copy_opt(unsigned long n) +{ + return false; +} + +#endif /* CONFIG_ARM64_COPY_FROM_USER_OPT */ + extern unsigned long __must_check __arch_copy_from_user_opt(void *to, const void __user *from, unsigned long n); extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 0ad4b3730b64..202a66e2581f 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2576,6 +2576,8 @@ static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__un } #endif +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + static bool copy_opt_disable __ro_after_init; static int __init parse_copy_opt_disable(char *str) @@ -2600,6 +2602,7 @@ static bool has_copy_opt(const struct arm64_cpu_capabilities *cap, int scope) return is_midr_in_range_list(copy_opt_cpus); } +#endif static const struct arm64_cpu_capabilities arm64_features[] = { { @@ -3176,12 +3179,14 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_arch_xcall_xint, }, #endif +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT { .desc = "Hisilicon Optimized Copy From User enabled", .capability = ARM64_HAS_COPY_OPT, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_copy_opt, }, +#endif {}, }; diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 59f1a4f36982..c3f24258803a 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -72,6 +72,8 @@ USER(9998f, ldtrb tmp1w, [srcin]) SYM_FUNC_END(__arch_copy_from_user) EXPORT_SYMBOL(__arch_copy_from_user) +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + .macro ldp2 reg1, reg2, ptr, val user_ldst_pair_index 9997f, ldp, \reg1, \reg2, \ptr, \val .endm @@ -103,3 +105,5 @@ USER(9998f, ldtrb tmp1w, [srcin]) b .L__arch_copy_from_user_opt_exit SYM_FUNC_END(__arch_copy_from_user_opt) EXPORT_SYMBOL(__arch_copy_from_user_opt) + +#endif /* CONFIG_ARM64_COPY_FROM_USER_OPT */ -- 2.33.0