hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9132 ------------------------------------------------------- Introduce CONFIG_ARM64_COPY_FROM_USER_OPT to control compilation of the optimized copy_from_user implementation, and a "copy_opt_disable" cmdline parameter to disable it. Signed-off-by: Qi Xi <xiqi2@huawei.com> --- arch/arm64/Kconfig | 11 +++++++++++ arch/arm64/include/asm/uaccess.h | 11 +++++++++++ arch/arm64/kernel/cpufeature.c | 5 +++++ arch/arm64/lib/copy_from_user.S | 4 ++++ 4 files changed, 31 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index c3b38c890b45..eac62767857b 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2626,6 +2626,17 @@ config CLEAR_USER_WORKAROUND help It has better performance while make sttr instruction 32-aligned in __arch_clear_user(). +config ARM64_COPY_FROM_USER_OPT + bool "Hisilicon Optimized Copy From User enabled" + depends on ARCH_HISI + default y + help + Enable an optimized copy_from_user implementation for + supported cores (e.g. Hisilicon HIP12, Linxicore9100). + This trades a small PAN toggle overhead for higher + throughput on large copies. This can be disabled at + boot via copy_opt=off. + endmenu # "Kernel Features" menu "Boot options" diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 79587588aad4..08a4ceef4a81 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -391,6 +391,8 @@ do { \ } while (0); \ } while(0) +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + #define COPY_OPT_THRESHOLD 4096 extern unsigned long __must_check __arch_copy_from_user_opt(void *to, @@ -402,6 +404,15 @@ static __always_inline bool use_copy_opt(unsigned long n) alternative_has_cap_unlikely(ARM64_HAS_COPY_OPT); } +#else /* !CONFIG_ARM64_COPY_FROM_USER_OPT */ + +static __always_inline bool use_copy_opt(unsigned long n) +{ + return false; +} + +#endif /* CONFIG_ARM64_COPY_FROM_USER_OPT */ + extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); #define raw_copy_from_user(to, from, n) \ ({ \ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 3f8889d5dd8c..659d6b9f12a5 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2576,6 +2576,8 @@ static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__un } #endif +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + static bool copy_opt_disable __ro_after_init; static int __init parse_copy_opt_disable(char *str) @@ -2599,6 +2601,7 @@ static bool has_copy_opt(const struct arm64_cpu_capabilities *cap, int scope) return is_midr_in_range_list(copy_opt_cpus); } +#endif static const struct arm64_cpu_capabilities arm64_features[] = { { @@ -3175,12 +3178,14 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_arch_xcall_xint, }, #endif +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT { .desc = "Hisilicon Optimized Copy From User enabled", .capability = ARM64_HAS_COPY_OPT, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_copy_opt, }, +#endif {}, }; diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 59f1a4f36982..c3f24258803a 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -72,6 +72,8 @@ USER(9998f, ldtrb tmp1w, [srcin]) SYM_FUNC_END(__arch_copy_from_user) EXPORT_SYMBOL(__arch_copy_from_user) +#ifdef CONFIG_ARM64_COPY_FROM_USER_OPT + .macro ldp2 reg1, reg2, ptr, val user_ldst_pair_index 9997f, ldp, \reg1, \reg2, \ptr, \val .endm @@ -103,3 +105,5 @@ USER(9998f, ldtrb tmp1w, [srcin]) b .L__arch_copy_from_user_opt_exit SYM_FUNC_END(__arch_copy_from_user_opt) EXPORT_SYMBOL(__arch_copy_from_user_opt) + +#endif /* CONFIG_ARM64_COPY_FROM_USER_OPT */ -- 2.33.0