Signed-off-by: Qi Xi <xiqi2@huawei.com> arm64: Optimize copy_from/to_user with batched LDP/STP Signed-off-by: Qi Xi <xiqi2@huawei.com> --- arch/arm64/include/asm/asm-uaccess.h | 16 +++ arch/arm64/include/asm/uaccess.h | 25 +++- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/copy_opt.c | 100 ++++++++++++++ arch/arm64/lib/Makefile | 1 + arch/arm64/lib/copy_from_user_opt.S | 75 ++++++++++ arch/arm64/lib/copy_to_user_opt.S | 75 ++++++++++ arch/arm64/lib/copy_user_template.S | 196 +++++++++++++++++++++++++++ 8 files changed, 485 insertions(+), 5 deletions(-) create mode 100644 arch/arm64/kernel/copy_opt.c create mode 100644 arch/arm64/lib/copy_from_user_opt.S create mode 100644 arch/arm64/lib/copy_to_user_opt.S create mode 100644 arch/arm64/lib/copy_user_template.S diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index 46b8d2585980..f51f0d0aadc4 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -92,4 +92,20 @@ alternative_else_nop_endif _asm_mc_extable 8888b,\l; .endm + + .macro user_ldst_index l, inst, reg, addr, val +8888: \inst \reg, [\addr, \val]; + + _asm_extable 8888b,\l; + + _asm_mc_extable 8888b,\l; + .endm + + .macro user_ldst_pair_index l, inst, reg1, reg2, addr, val +8888: \inst \reg1, \reg2, [\addr, \val]; + + _asm_extable 8888b,\l; + + _asm_mc_extable 8888b,\l; + .endm #endif diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 03c2db710f92..d6d21d6bc6d6 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -386,24 +386,41 @@ do { \ goto err_label; \ } while(0) +#define COPY_OPT_THRESHOLD 4096 +extern bool kernel_copy_opt_enabled(void); extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); +extern unsigned long __must_check __arch_copy_from_user_opt(void *to, const void __user *from, unsigned long n); #define raw_copy_from_user(to, from, n) \ ({ \ unsigned long __acfu_ret; \ - uaccess_ttbr0_enable(); \ - __acfu_ret = __arch_copy_from_user((to), \ - __uaccess_mask_ptr(from), (n)); \ + if ((n) >= COPY_OPT_THRESHOLD && kernel_copy_opt_enabled()) { \ + uaccess_enable_privileged(); \ + __acfu_ret = __arch_copy_from_user_opt((to), \ + __uaccess_mask_ptr(from), (n)); \ + uaccess_disable_privileged(); \ + } else { \ + __acfu_ret = __arch_copy_from_user((to), \ + __uaccess_mask_ptr(from), n); \ + } \ uaccess_ttbr0_disable(); \ __acfu_ret; \ }) extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); +extern unsigned long __must_check __arch_copy_to_user_opt(void __user *to, const void *from, unsigned long n); #define raw_copy_to_user(to, from, n) \ ({ \ unsigned long __actu_ret; \ uaccess_ttbr0_enable(); \ - __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to), \ + if ((n) >= COPY_OPT_THRESHOLD && kernel_copy_opt_enabled()) { \ + uaccess_enable_privileged(); \ + __actu_ret = __arch_copy_to_user_opt(__uaccess_mask_ptr(to),\ + (from), (n)); \ + uaccess_disable_privileged(); \ + } else { \ + __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),\ (from), (n)); \ + } \ uaccess_ttbr0_disable(); \ __actu_ret; \ }) diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index a5fb42a6c8d7..04e84437d3e5 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -22,7 +22,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ return_address.o cpuinfo.o cpu_errata.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ - syscall.o proton-pack.o ipi_nmi.o + syscall.o proton-pack.o ipi_nmi.o copy_opt.o targets += efi-entry.o diff --git a/arch/arm64/kernel/copy_opt.c b/arch/arm64/kernel/copy_opt.c new file mode 100644 index 000000000000..3eeae607956d --- /dev/null +++ b/arch/arm64/kernel/copy_opt.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copy to/from user optimization. + * + * Copyright (C) 2026 Huawei Ltd. + */ + +#include <linux/bitmap.h> +#include <linux/bitops.h> +#include <linux/bottom_half.h> +#include <linux/bug.h> +#include <linux/cache.h> +#include <linux/compat.h> +#include <linux/compiler.h> +#include <linux/cpu.h> +#include <linux/cpu_pm.h> +#include <linux/ctype.h> +#include <linux/kernel.h> +#include <linux/linkage.h> +#include <linux/irqflags.h> +#include <linux/init.h> +#include <linux/percpu.h> +#include <linux/prctl.h> +#include <linux/preempt.h> +#include <linux/ptrace.h> +#include <linux/sched/signal.h> +#include <linux/sched/task_stack.h> +#include <linux/signal.h> +#include <linux/slab.h> +#include <linux/stddef.h> +#include <linux/sysctl.h> +#include <linux/swab.h> + +#include <asm/esr.h> +#include <asm/exception.h> +#include <asm/fpsimd.h> +#include <asm/cpufeature.h> +#include <asm/cputype.h> +#include <asm/neon.h> +#include <asm/processor.h> +#include <asm/simd.h> +#include <asm/sigcontext.h> +#include <asm/sysreg.h> +#include <asm/traps.h> +#include <asm/virt.h> + +DEFINE_STATIC_KEY_FALSE(use_copy_opt); +bool kernel_copy_opt_enabled(void) +{ + return static_key_enabled(&use_copy_opt); +} +EXPORT_SYMBOL_GPL(kernel_copy_opt_enabled); + +static int copy_opt_sysctl_handler(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, val; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .mode = table->mode, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!write) { + if (static_key_enabled(&use_copy_opt)) + val = 1; + else + val = 0; + } + ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); + if (write && !ret) { + if (val) + static_branch_enable(&use_copy_opt); + else + static_branch_disable(&use_copy_opt); + } + return 0; +} + +static struct ctl_table copy_opt_sysctl_table[] = { + { + .procname = "copy_opt_user", + .mode = 0644, + .proc_handler = copy_opt_sysctl_handler, + }, + { } +}; + +static int __init copy_opt_sysctl_init(void) +{ + if (!register_sysctl("kernel", copy_opt_sysctl_table)) + return -EINVAL; + return 0; +} +core_initcall(copy_opt_sysctl_init); diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index e2dbef587c9b..c69993a77e78 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_in_user.o copy_page.o \ + copy_to_user_opt.o copy_from_user_opt.o \ clear_page.o csum.o memchr.o memcpy.o memcpy_mc.o memmove.o \ memset.o memcmp.o strcmp.o strncmp.o strlen.o \ strnlen.o strchr.o strrchr.o tishift.o diff --git a/arch/arm64/lib/copy_from_user_opt.S b/arch/arm64/lib/copy_from_user_opt.S new file mode 100644 index 000000000000..94cfc09b89d7 --- /dev/null +++ b/arch/arm64/lib/copy_from_user_opt.S @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 ARM Ltd. + */ + +#include <linux/linkage.h> + +#include <asm/asm-uaccess.h> +#include <asm/assembler.h> +#include <asm/cache.h> + +/* + * Copy from user space to a kernel buffer (alignment handled by the hardware) + * + * Parameters: + * x0 - to + * x1 - from + * x2 - n + * Returns: + * x0 - bytes not copied + */ + + .macro ldrb1 reg, ptr, val + user_ldst 9998f, ldtrb, \reg, \ptr, \val + .endm + + .macro strb1 reg, ptr, val + USER_MC(9998f, strb \reg, [\ptr], \val) + .endm + + .macro ldrh1 reg, ptr, val + user_ldst 9997f, ldtrh, \reg, \ptr, \val + .endm + + .macro strh1 reg, ptr, val + USER_MC(9998f, strh \reg, [\ptr], \val) + .endm + + .macro ldr1 reg, ptr, val + user_ldst 9997f, ldtr, \reg, \ptr, \val + .endm + + .macro str1 reg, ptr, val + USER_MC(9998f, str \reg, [\ptr], \val) + .endm + + .macro ldp2 reg1, reg2, ptr, val + user_ldst_pair_index 9997f, ldp, \reg1, \reg2, \ptr, \val + .endm + + .macro stp2 reg1, reg2, ptr, val + USER_MC(9998f, stp \reg1, \reg2, [\ptr, \val]) + .endm + +end .req x5 +srcin .req x15 +SYM_FUNC_START(__arch_copy_from_user_opt) + add end, x0, x2 + mov srcin, x1 +#include "copy_user_template.S" + mov x0, #0 // Nothing to copy + ret +SYM_FUNC_END(__arch_copy_from_user_opt) +EXPORT_SYMBOL(__arch_copy_from_user_opt) + + .section .fixup,"ax" + .align 2 +9997: cmp dst, dstin + b.ne 9998f + // Before being absolutely sure we couldn't copy anything, try harder +USER(9998f, ldtrb tmp1w, [srcin]) + strb tmp1w, [dst], #1 +9998: sub x0, end, dst // bytes not copied + ret + .previous diff --git a/arch/arm64/lib/copy_to_user_opt.S b/arch/arm64/lib/copy_to_user_opt.S new file mode 100644 index 000000000000..57a47383df20 --- /dev/null +++ b/arch/arm64/lib/copy_to_user_opt.S @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 ARM Ltd. + */ + +#include <linux/linkage.h> + +#include <asm/asm-uaccess.h> +#include <asm/assembler.h> +#include <asm/cache.h> + +/* + * Copy to user space from a kernel buffer (alignment handled by the hardware) + * + * Parameters: + * x0 - to + * x1 - from + * x2 - n + * Returns: + * x0 - bytes not copied + */ + .macro ldrb1 reg, ptr, val + USER_MC(9998f, ldrb \reg, [\ptr], \val) + .endm + + .macro strb1 reg, ptr, val + user_ldst 9998f, sttrb, \reg, \ptr, \val + .endm + + .macro ldrh1 reg, ptr, val + USER_MC(9998f, ldrh \reg, [\ptr], \val) + .endm + + .macro strh1 reg, ptr, val + user_ldst 9997f, sttrh, \reg, \ptr, \val + .endm + + .macro ldr1 reg, ptr, val + USER_MC(9998f, ldr \reg, [\ptr], \val) + .endm + + .macro str1 reg, ptr, val + user_ldst 9997f, sttr, \reg, \ptr, \val + .endm + + .macro ldp2 reg1, reg2, ptr, val + USER_MC(9998f, ldp \reg1, \reg2, [\ptr, \val]) + .endm + + .macro stp2 reg1, reg2, ptr, val + user_ldst_pair_index 9997f, stp, \reg1, \reg2, \ptr, \val + .endm + +end .req x5 +srcin .req x15 +SYM_FUNC_START(__arch_copy_to_user_opt) + add end, x0, x2 + mov srcin, x1 +#include "copy_user_template.S" + mov x0, #0 + ret +SYM_FUNC_END(__arch_copy_to_user_opt) +EXPORT_SYMBOL(__arch_copy_to_user_opt) + + .section .fixup,"ax" + .align 2 +9997: cmp dst, dstin + b.ne 9998f + // Before being absolutely sure we couldn't copy anything, try harder +USER_MC(9998f, ldrb tmp1w, [srcin]) +USER(9998f, sttrb tmp1w, [dst]) + add dst, dst, #1 +9998: sub x0, end, dst // bytes not copied + ret + .previous diff --git a/arch/arm64/lib/copy_user_template.S b/arch/arm64/lib/copy_user_template.S new file mode 100644 index 000000000000..72983eb658b3 --- /dev/null +++ b/arch/arm64/lib/copy_user_template.S @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ + */ + +/* + * Copy a buffer from src to dest (alignment handled by the hardware) + * + * Parameters: + * x0 - dest + * x1 - src + * x2 - n + * Returns: + * x0 - dest + */ +dstin .req x0 +src .req x1 +count .req x2 +tmp1 .req x3 +tmp1w .req w3 +tmp2 .req x4 +tmp2w .req w4 +dst .req x6 + +A_l .req x7 +A_h .req x8 +B_l .req x9 +B_h .req x10 +C_l .req x11 +C_h .req x12 +D_l .req x13 +D_h .req x14 + + mov dst, dstin + cmp count, #16 + /*When memory length is less than 16, the accessed are not aligned.*/ + b.lo .Ltiny15 + + neg tmp2, src + ands tmp2, tmp2, #15/* Bytes to reach alignment. */ + b.eq .LSrcAligned + sub count, count, tmp2 + /* + * Copy the leading memory data from src to dst in an increasing + * address order.By this way,the risk of overwriting the source + * memory data is eliminated when the distance between src and + * dst is less than 16. The memory accesses here are alignment. + */ + tbz tmp2, #0, 1f + ldrb1 tmp1w, src, #1 + strb1 tmp1w, dst, #1 +1: + tbz tmp2, #1, 2f + ldrh1 tmp1w, src, #2 + strh1 tmp1w, dst, #2 +2: + tbz tmp2, #2, 3f + ldr1 tmp1w, src, #4 + str1 tmp1w, dst, #4 +3: + tbz tmp2, #3, .LSrcAligned + ldr1 tmp1, src, #8 + str1 tmp1, dst, #8 + +.LSrcAligned: + cmp count, #64 + b.ge .Lcpy_over64 + /* + * Deal with small copies quickly by dropping straight into the + * exit block. + */ +.Ltail63: + /* + * Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. + */ + ands tmp1, count, #0x30 + b.eq .Ltiny15 + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + add src, src, #16 + add dst, dst, #16 +1: + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + add src, src, #16 + add dst, dst, #16 +2: + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + add src, src, #16 + add dst, dst, #16 +.Ltiny15: + /* + * Prefer to break one ldp/stp into several load/store to access + * memory in an increasing address order,rather than to load/store 16 + * bytes from (src-16) to (dst-16) and to backward the src to aligned + * address,which way is used in original cortex memcpy. If keeping + * the original memcpy process here, memmove need to satisfy the + * precondition that src address is at least 16 bytes bigger than dst + * address,otherwise some source data will be overwritten when memove + * call memcpy directly. To make memmove simpler and decouple the + * memcpy's dependency on memmove, withdrew the original process. + */ + tbz count, #3, 1f + ldr1 tmp1, src, #8 + str1 tmp1, dst, #8 +1: + tbz count, #2, 2f + ldr1 tmp1w, src, #4 + str1 tmp1w, dst, #4 +2: + tbz count, #1, 3f + ldrh1 tmp1w, src, #2 + strh1 tmp1w, dst, #2 +3: + tbz count, #0, .Lexitfunc + ldrb1 tmp1w, src, #1 + strb1 tmp1w, dst, #1 + + b .Lexitfunc + +.Lcpy_over64: + subs count, count, #128 + b.ge .Lcpy_body_large + /* + * Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. + */ + ldp2 A_l, A_h, src, #0 + stp2 A_l, A_h, dst, #0 + ldp2 B_l, B_h, src, #16 + ldp2 C_l, C_h, src, #32 + stp2 B_l, B_h, dst, #16 + stp2 C_l, C_h, dst, #32 + ldp2 D_l, D_h, src, #48 + stp2 D_l, D_h, dst, #48 + add src, src, #64 + add dst, dst, #64 + + tst count, #0x3f + b.ne .Ltail63 + b .Lexitfunc + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. + */ + .p2align L1_CACHE_SHIFT +.Lcpy_body_large: + + /* pre-get 64 bytes data. */ + ldp2 A_l, A_h, src, #0 + ldp2 B_l, B_h, src, #16 + ldp2 C_l, C_h, src, #32 + ldp2 D_l, D_h, src, #48 + add src, src, #64 +1: + /* + * interlace the load of next 64 bytes data block with store of the last + * loaded 64 bytes data. + */ + stp2 A_l, A_h, dst, #0 + ldp2 A_l, A_h, src, #0 + stp2 B_l, B_h, dst, #16 + ldp2 B_l, B_h, src, #16 + stp2 C_l, C_h, dst, #32 + ldp2 C_l, C_h, src, #32 + stp2 D_l, D_h, dst, #48 + ldp2 D_l, D_h, src, #48 + add dst, dst, #64 + add src, src, #64 + subs count, count, #64 + b.ge 1b + + /* Post-loop: store the last block of data using stp2 */ + /* (without post-increment) */ + stp2 A_l, A_h, dst, #0 + stp2 B_l, B_h, dst, #16 + stp2 C_l, C_h, dst, #32 + stp2 D_l, D_h, dst, #48 + add dst, dst, #64 + + tst count, #0x3f + b.ne .Ltail63 +.Lexitfunc: -- 2.43.0