hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9132 ------------------------------------------------------- Use __arch_copy_from_user_opt for copies >= 4KB on CPUs with ARM64_HAS_COPY_OPT. The optimized path turns PAN off during the copy, which adds a small overhead. Only enable it for large copies so the overhead is worth the faster throughput. Signed-off-by: Qi Xi <xiqi2@huawei.com> --- arch/arm64/include/asm/uaccess.h | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index dd0877a75922..79587588aad4 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -391,13 +391,29 @@ do { \ } while (0); \ } while(0) +#define COPY_OPT_THRESHOLD 4096 + +extern unsigned long __must_check __arch_copy_from_user_opt(void *to, + const void __user *from, unsigned long n); + +static __always_inline bool use_copy_opt(unsigned long n) +{ + return (n) >= COPY_OPT_THRESHOLD && + alternative_has_cap_unlikely(ARM64_HAS_COPY_OPT); +} + extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); #define raw_copy_from_user(to, from, n) \ ({ \ unsigned long __acfu_ret; \ uaccess_ttbr0_enable(); \ - __acfu_ret = __arch_copy_from_user((to), \ - __uaccess_mask_ptr(from), (n)); \ + if (use_copy_opt(n)) { \ + __acfu_ret = __arch_copy_from_user_opt((to), \ + __uaccess_mask_ptr(from), (n)); \ + } else { \ + __acfu_ret = __arch_copy_from_user((to), \ + __uaccess_mask_ptr(from), (n)); \ + } \ uaccess_ttbr0_disable(); \ __acfu_ret; \ }) -- 2.33.0