[PATCH OLK-6.6 00/16]fix some issues for arm64 machine check safe
The following optimization are made for feature "arm64: add machine check safe support": 1. Optimize do_sea() processing logic. 2. Add new fixup type of the copy_to/from_user() memory error. 3. Optimize repeated assembly codes in copy_mc_page.S and memcpy_mc.S. 4. The below mailist patch hardening feature is incorporated: https://lore.kernel.org/lkml/20240204080144.7977-1-xueshuai@linux.alibaba.co... Marc Zyngier (1): arm64: Get rid of ARM64_HAS_NO_HW_PREFETCH Shuai Xue (3): ACPI: APEI: send SIGBUS to current task if synchronous memory error not recovered mm: memory-failure: move return value documentation to function declaration ACPI: APEI: handle synchronous exceptions in task work to send correct SIGBUS si_code Tong Tiangen (12): Revert "kasan: fix the compilation error for memcpy_mcs()" Revert "arm64: add machine check safe sysctl interface" Revert "arm64: introduce copy_mc_to_kernel() implementation" Revert "arm64: support copy_mc_[user]_highpage()" Revert "mm/hwpoison: return -EFAULT when copy fail in copy_mc_[user]_highpage()" Revert "arm64: add uaccess to machine check safe" Revert "arm64: add support for machine check error safe" arm64: add support for ARCH_HAS_COPY_MC mm/hwpoison: return -EFAULT when copy fail in copy_mc_[user]_highpage() arm64: support copy_mc_[user]_highpage() arm64: introduce copy_mc_to_kernel() implementation arm64: add machine check safe sysctl interface arch/arm64/include/asm/asm-extable.h | 34 +-- arch/arm64/include/asm/asm-uaccess.h | 4 + arch/arm64/include/asm/assembler.h | 4 - arch/arm64/include/asm/extable.h | 2 +- arch/arm64/include/asm/mte.h | 4 + arch/arm64/include/asm/string.h | 6 +- arch/arm64/include/asm/uaccess.h | 13 +- arch/arm64/kernel/cpufeature.c | 16 -- arch/arm64/lib/Makefile | 4 +- arch/arm64/lib/copy_from_user.S | 10 +- arch/arm64/lib/copy_mc_page.S | 63 +----- arch/arm64/lib/copy_page.S | 61 +----- arch/arm64/lib/copy_page_template.S | 56 +++++ arch/arm64/lib/copy_to_user.S | 10 +- arch/arm64/lib/memcpy_mc.S | 304 ++++++--------------------- arch/arm64/lib/mte.S | 6 +- arch/arm64/mm/copypage.c | 39 ++-- arch/arm64/mm/extable.c | 12 +- arch/arm64/mm/fault.c | 53 ++--- arch/arm64/tools/cpucaps | 1 - arch/x86/kernel/cpu/mce/core.c | 7 - drivers/acpi/apei/ghes.c | 84 +++++--- include/acpi/ghes.h | 3 - mm/kasan/shadow.c | 6 +- mm/memory-failure.c | 22 +- 25 files changed, 282 insertions(+), 542 deletions(-) create mode 100644 arch/arm64/lib/copy_page_template.S -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- This reverts commit 16ccb01403953ea7e921ab1e9f9f52d3cbd5a397. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- mm/kasan/shadow.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 9372f0c6f7ec..ffaf2df1ed6e 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -83,9 +83,9 @@ void *memcpy(void *dest, const void *src, size_t len) #undef memcpy_mcs int memcpy_mcs(void *dest, const void *src, size_t len) { - if (!kasan_check_range(src, len, false, _RET_IP_) || - !kasan_check_range(dest, len, true, _RET_IP_)) - return (int)len; + if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || + !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) + return (unsigned long)len; return __memcpy_mcs(dest, src, len); } -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- This reverts commit fafd6b478a205f6acea391516694890ca4c96dc7. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- Documentation/admin-guide/sysctl/kernel.rst | 14 ---------- arch/arm64/mm/fault.c | 29 --------------------- 2 files changed, 43 deletions(-) diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index 3b8953c49183..cf33de56da27 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -540,20 +540,6 @@ if leaking kernel pointer values to unprivileged users is a concern. When ``kptr_restrict`` is set to 2, kernel pointers printed using %pK will be replaced with 0s regardless of privileges. -machine_check_safe (arm64 only) -================================ - -This indicates whether the Machine Check safe memory copy feature enabled -or not���which only exists on arm64 when ARCH_HAS_COPY_MC enabled. - -The value in this file determines the behaviour of the kernel when -synchronous exception from memory copy. - -= =================================================================== -0 the kernel will panic immediately. -1 the kernel will recover since a memcpy-variant provided which can - safely fail when accessing to hwpoison. -= =================================================================== modprobe ======== diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index e5b38a26bc73..5bf25c6dc27e 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -25,7 +25,6 @@ #include <linux/perf_event.h> #include <linux/preempt.h> #include <linux/hugetlb.h> -#include <linux/sysctl.h> #include <asm/acpi.h> #include <asm/bug.h> @@ -44,31 +43,6 @@ #include <asm/tlbflush.h> #include <asm/traps.h> -static int sysctl_machine_check_safe = IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC); - -#ifdef CONFIG_ARCH_HAS_COPY_MC -static struct ctl_table machine_check_safe_sysctl_table[] = { - { - .procname = "machine_check_safe", - .data = &sysctl_machine_check_safe, - .maxlen = sizeof(sysctl_machine_check_safe), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -}; - -static int __init machine_check_safe_sysctl_init(void) -{ - if (!register_sysctl("kernel", machine_check_safe_sysctl_table)) - return -EINVAL; - return 0; -} - -core_initcall(machine_check_safe_sysctl_init); -#endif - struct fault_info { int (*fn)(unsigned long far, unsigned long esr, struct pt_regs *regs); @@ -764,9 +738,6 @@ static bool arm64_do_kernel_sea(unsigned long addr, unsigned int esr, if (!IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC)) return false; - if (!sysctl_machine_check_safe) - return false; - if (user_mode(regs)) return false; -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- This reverts commit c478eeb8692dbec7afba41989fa1030be141ba9f. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/include/asm/string.h | 5 - arch/arm64/include/asm/uaccess.h | 21 --- arch/arm64/lib/Makefile | 2 +- arch/arm64/lib/memcpy_mc.S | 257 ------------------------------- mm/kasan/shadow.c | 12 -- 5 files changed, 1 insertion(+), 296 deletions(-) delete mode 100644 arch/arm64/lib/memcpy_mc.S diff --git a/arch/arm64/include/asm/string.h b/arch/arm64/include/asm/string.h index 995b63c26e99..3a3264ff47b9 100644 --- a/arch/arm64/include/asm/string.h +++ b/arch/arm64/include/asm/string.h @@ -35,10 +35,6 @@ extern void *memchr(const void *, int, __kernel_size_t); extern void *memcpy(void *, const void *, __kernel_size_t); extern void *__memcpy(void *, const void *, __kernel_size_t); -#define __HAVE_ARCH_MEMCPY_MC -extern int memcpy_mcs(void *, const void *, __kernel_size_t); -extern int __memcpy_mcs(void *, const void *, __kernel_size_t); - #define __HAVE_ARCH_MEMMOVE extern void *memmove(void *, const void *, __kernel_size_t); extern void *__memmove(void *, const void *, __kernel_size_t); @@ -61,7 +57,6 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt); */ #define memcpy(dst, src, len) __memcpy(dst, src, len) -#define memcpy_mcs(dst, src, len) __memcpy_mcs(dst, src, len) #define memmove(dst, src, len) __memmove(dst, src, len) #define memset(s, c, n) __memset(s, c, n) diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 61e28ef2112a..14be5000c5a0 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -425,25 +425,4 @@ static inline size_t probe_subpage_writeable(const char __user *uaddr, #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */ -#ifdef CONFIG_ARCH_HAS_COPY_MC -/** - * copy_mc_to_kernel - memory copy that handles source exceptions - * - * @dst: destination address - * @src: source address - * @len: number of bytes to copy - * - * Return 0 for success, or #size if there was an exception. - */ -static inline unsigned long __must_check -copy_mc_to_kernel(void *to, const void *from, unsigned long size) -{ - int ret; - - ret = memcpy_mcs(to, from, size); - return (ret == -EFAULT) ? size : 0; -} -#define copy_mc_to_kernel copy_mc_to_kernel -#endif - #endif /* __ASM_UACCESS_H */ diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 899d6ae9698c..a2fd865b816d 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -3,7 +3,7 @@ lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ memset.o memcmp.o strcmp.o strncmp.o strlen.o \ - strnlen.o strchr.o strrchr.o tishift.o memcpy_mc.o + strnlen.o strchr.o strrchr.o tishift.o ifeq ($(CONFIG_KERNEL_MODE_NEON), y) obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o diff --git a/arch/arm64/lib/memcpy_mc.S b/arch/arm64/lib/memcpy_mc.S deleted file mode 100644 index 7076b500d154..000000000000 --- a/arch/arm64/lib/memcpy_mc.S +++ /dev/null @@ -1,257 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2012-2021, Arm Limited. - * - * Adapted from the original at: - * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/str... - */ - -#include <linux/linkage.h> -#include <asm/assembler.h> - -/* Assumptions: - * - * ARMv8-a, AArch64, unaligned accesses. - * - */ - -#define L(label) .L ## label - -#define dstin x0 -#define src x1 -#define count x2 -#define dst x3 -#define srcend x4 -#define dstend x5 -#define A_l x6 -#define A_lw w6 -#define A_h x7 -#define B_l x8 -#define B_lw w8 -#define B_h x9 -#define C_l x10 -#define C_lw w10 -#define C_h x11 -#define D_l x12 -#define D_h x13 -#define E_l x14 -#define E_h x15 -#define F_l x16 -#define F_h x17 -#define G_l count -#define G_h dst -#define H_l src -#define H_h srcend -#define tmp1 x14 - -/* This implementation handles overlaps and supports both memcpy and memmove - from a single entry point. It uses unaligned accesses and branchless - sequences to keep the code small, simple and improve performance. - - Copies are split into 3 main cases: small copies of up to 32 bytes, medium - copies of up to 128 bytes, and large copies. The overhead of the overlap - check is negligible since it is only required for large copies. - - Large copies use a software pipelined loop processing 64 bytes per iteration. - The destination pointer is 16-byte aligned to minimize unaligned accesses. - The loop tail is handled by always copying 64 bytes from the end. -*/ - -SYM_FUNC_START(__pi_memcpy_mcs) - add srcend, src, count - add dstend, dstin, count - cmp count, 128 - b.hi L(copy_long) - cmp count, 32 - b.hi L(copy32_128) - - /* Small copies: 0..32 bytes. */ - cmp count, 16 - b.lo L(copy16) - CPY_MC(9998f, ldp A_l, A_h, [src]) - CPY_MC(9998f, ldp D_l, D_h, [srcend, -16]) - CPY_MC(9998f, stp A_l, A_h, [dstin]) - CPY_MC(9998f, stp D_l, D_h, [dstend, -16]) - mov x0, #0 - ret - - /* Copy 8-15 bytes. */ -L(copy16): - tbz count, 3, L(copy8) - CPY_MC(9998f, ldr A_l, [src]) - CPY_MC(9998f, ldr A_h, [srcend, -8]) - CPY_MC(9998f, str A_l, [dstin]) - CPY_MC(9998f, str A_h, [dstend, -8]) - mov x0, #0 - ret - - .p2align 3 - /* Copy 4-7 bytes. */ -L(copy8): - tbz count, 2, L(copy4) - CPY_MC(9998f, ldr A_lw, [src]) - CPY_MC(9998f, ldr B_lw, [srcend, -4]) - CPY_MC(9998f, str A_lw, [dstin]) - CPY_MC(9998f, str B_lw, [dstend, -4]) - mov x0, #0 - ret - - /* Copy 0..3 bytes using a branchless sequence. */ -L(copy4): - cbz count, L(copy0) - lsr tmp1, count, 1 - CPY_MC(9998f, ldrb A_lw, [src]) - CPY_MC(9998f, ldrb C_lw, [srcend, -1]) - CPY_MC(9998f, ldrb B_lw, [src, tmp1]) - CPY_MC(9998f, strb A_lw, [dstin]) - CPY_MC(9998f, strb B_lw, [dstin, tmp1]) - CPY_MC(9998f, strb C_lw, [dstend, -1]) -L(copy0): - mov x0, #0 - ret - - .p2align 4 - /* Medium copies: 33..128 bytes. */ -L(copy32_128): - CPY_MC(9998f, ldp A_l, A_h, [src]) - CPY_MC(9998f, ldp B_l, B_h, [src, 16]) - CPY_MC(9998f, ldp C_l, C_h, [srcend, -32]) - CPY_MC(9998f, ldp D_l, D_h, [srcend, -16]) - cmp count, 64 - b.hi L(copy128) - CPY_MC(9998f, stp A_l, A_h, [dstin]) - CPY_MC(9998f, stp B_l, B_h, [dstin, 16]) - CPY_MC(9998f, stp C_l, C_h, [dstend, -32]) - CPY_MC(9998f, stp D_l, D_h, [dstend, -16]) - mov x0, #0 - ret - - .p2align 4 - /* Copy 65..128 bytes. */ -L(copy128): - CPY_MC(9998f, ldp E_l, E_h, [src, 32]) - CPY_MC(9998f, ldp F_l, F_h, [src, 48]) - cmp count, 96 - b.ls L(copy96) - CPY_MC(9998f, ldp G_l, G_h, [srcend, -64]) - CPY_MC(9998f, ldp H_l, H_h, [srcend, -48]) - CPY_MC(9998f, stp G_l, G_h, [dstend, -64]) - CPY_MC(9998f, stp H_l, H_h, [dstend, -48]) -L(copy96): - CPY_MC(9998f, stp A_l, A_h, [dstin]) - CPY_MC(9998f, stp B_l, B_h, [dstin, 16]) - CPY_MC(9998f, stp E_l, E_h, [dstin, 32]) - CPY_MC(9998f, stp F_l, F_h, [dstin, 48]) - CPY_MC(9998f, stp C_l, C_h, [dstend, -32]) - CPY_MC(9998f, stp D_l, D_h, [dstend, -16]) - mov x0, #0 - ret - - .p2align 4 - /* Copy more than 128 bytes. */ -L(copy_long): - /* Use backwards copy if there is an overlap. */ - sub tmp1, dstin, src - cbz tmp1, L(copy0) - cmp tmp1, count - b.lo L(copy_long_backwards) - - /* Copy 16 bytes and then align dst to 16-byte alignment. */ - - CPY_MC(9998f, ldp D_l, D_h, [src]) - and tmp1, dstin, 15 - bic dst, dstin, 15 - sub src, src, tmp1 - add count, count, tmp1 /* Count is now 16 too large. */ - CPY_MC(9998f, ldp A_l, A_h, [src, 16]) - CPY_MC(9998f, stp D_l, D_h, [dstin]) - CPY_MC(9998f, ldp B_l, B_h, [src, 32]) - CPY_MC(9998f, ldp C_l, C_h, [src, 48]) - CPY_MC(9998f, ldp D_l, D_h, [src, 64]!) - subs count, count, 128 + 16 /* Test and readjust count. */ - b.ls L(copy64_from_end) - -L(loop64): - CPY_MC(9998f, stp A_l, A_h, [dst, 16]) - CPY_MC(9998f, ldp A_l, A_h, [src, 16]) - CPY_MC(9998f, stp B_l, B_h, [dst, 32]) - CPY_MC(9998f, ldp B_l, B_h, [src, 32]) - CPY_MC(9998f, stp C_l, C_h, [dst, 48]) - CPY_MC(9998f, ldp C_l, C_h, [src, 48]) - CPY_MC(9998f, stp D_l, D_h, [dst, 64]!) - CPY_MC(9998f, ldp D_l, D_h, [src, 64]!) - subs count, count, 64 - b.hi L(loop64) - - /* Write the last iteration and copy 64 bytes from the end. */ -L(copy64_from_end): - CPY_MC(9998f, ldp E_l, E_h, [srcend, -64]) - CPY_MC(9998f, stp A_l, A_h, [dst, 16]) - CPY_MC(9998f, ldp A_l, A_h, [srcend, -48]) - CPY_MC(9998f, stp B_l, B_h, [dst, 32]) - CPY_MC(9998f, ldp B_l, B_h, [srcend, -32]) - CPY_MC(9998f, stp C_l, C_h, [dst, 48]) - CPY_MC(9998f, ldp C_l, C_h, [srcend, -16]) - CPY_MC(9998f, stp D_l, D_h, [dst, 64]) - CPY_MC(9998f, stp E_l, E_h, [dstend, -64]) - CPY_MC(9998f, stp A_l, A_h, [dstend, -48]) - CPY_MC(9998f, stp B_l, B_h, [dstend, -32]) - CPY_MC(9998f, stp C_l, C_h, [dstend, -16]) - mov x0, #0 - ret - - .p2align 4 - - /* Large backwards copy for overlapping copies. - Copy 16 bytes and then align dst to 16-byte alignment. */ -L(copy_long_backwards): - CPY_MC(9998f, ldp D_l, D_h, [srcend, -16]) - and tmp1, dstend, 15 - sub srcend, srcend, tmp1 - sub count, count, tmp1 - CPY_MC(9998f, ldp A_l, A_h, [srcend, -16]) - CPY_MC(9998f, stp D_l, D_h, [dstend, -16]) - CPY_MC(9998f, ldp B_l, B_h, [srcend, -32]) - CPY_MC(9998f, ldp C_l, C_h, [srcend, -48]) - CPY_MC(9998f, ldp D_l, D_h, [srcend, -64]!) - sub dstend, dstend, tmp1 - subs count, count, 128 - b.ls L(copy64_from_start) - -L(loop64_backwards): - CPY_MC(9998f, stp A_l, A_h, [dstend, -16]) - CPY_MC(9998f, ldp A_l, A_h, [srcend, -16]) - CPY_MC(9998f, stp B_l, B_h, [dstend, -32]) - CPY_MC(9998f, ldp B_l, B_h, [srcend, -32]) - CPY_MC(9998f, stp C_l, C_h, [dstend, -48]) - CPY_MC(9998f, ldp C_l, C_h, [srcend, -48]) - CPY_MC(9998f, stp D_l, D_h, [dstend, -64]!) - CPY_MC(9998f, ldp D_l, D_h, [srcend, -64]!) - subs count, count, 64 - b.hi L(loop64_backwards) - - /* Write the last iteration and copy 64 bytes from the start. */ -L(copy64_from_start): - CPY_MC(9998f, ldp G_l, G_h, [src, 48]) - CPY_MC(9998f, stp A_l, A_h, [dstend, -16]) - CPY_MC(9998f, ldp A_l, A_h, [src, 32]) - CPY_MC(9998f, stp B_l, B_h, [dstend, -32]) - CPY_MC(9998f, ldp B_l, B_h, [src, 16]) - CPY_MC(9998f, stp C_l, C_h, [dstend, -48]) - CPY_MC(9998f, ldp C_l, C_h, [src]) - CPY_MC(9998f, stp D_l, D_h, [dstend, -64]) - CPY_MC(9998f, stp G_l, G_h, [dstin, 48]) - CPY_MC(9998f, stp A_l, A_h, [dstin, 32]) - CPY_MC(9998f, stp B_l, B_h, [dstin, 16]) - CPY_MC(9998f, stp C_l, C_h, [dstin]) - mov x0, #0 - ret - -9998: mov x0, #-EFAULT - ret -SYM_FUNC_END(__pi_memcpy_mcs) - -SYM_FUNC_ALIAS(__memcpy_mcs, __pi_memcpy_mcs) -EXPORT_SYMBOL(__memcpy_mcs) -SYM_FUNC_ALIAS_WEAK(memcpy_mcs, __memcpy_mcs) -EXPORT_SYMBOL(memcpy_mcs) diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index ffaf2df1ed6e..dd772f9d0f08 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -79,18 +79,6 @@ void *memcpy(void *dest, const void *src, size_t len) } #endif -#ifdef __HAVE_ARCH_MEMCPY_MC -#undef memcpy_mcs -int memcpy_mcs(void *dest, const void *src, size_t len) -{ - if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || - !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) - return (unsigned long)len; - - return __memcpy_mcs(dest, src, len); -} -#endif - void *__asan_memset(void *addr, int c, ssize_t len) { if (!kasan_check_range(addr, len, true, _RET_IP_)) -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- This reverts commit 9e7e36876260c81bb34c1ef7141a50fb22abfd24. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/include/asm/asm-extable.h | 15 ----- arch/arm64/include/asm/assembler.h | 4 -- arch/arm64/include/asm/mte.h | 5 -- arch/arm64/include/asm/page.h | 10 ---- arch/arm64/lib/Makefile | 2 - arch/arm64/lib/copy_mc_page.S | 89 ---------------------------- arch/arm64/lib/mte.S | 27 --------- arch/arm64/mm/copypage.c | 66 ++------------------- arch/arm64/mm/extable.c | 7 +-- include/linux/highmem.h | 8 --- 10 files changed, 9 insertions(+), 224 deletions(-) delete mode 100644 arch/arm64/lib/copy_mc_page.S diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h index 819044fefbe7..980d1dd8e1a3 100644 --- a/arch/arm64/include/asm/asm-extable.h +++ b/arch/arm64/include/asm/asm-extable.h @@ -10,7 +10,6 @@ #define EX_TYPE_UACCESS_ERR_ZERO 2 #define EX_TYPE_KACCESS_ERR_ZERO 3 #define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4 -#define EX_TYPE_COPY_MC_PAGE_ERR_ZERO 5 /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */ #define EX_DATA_REG_ERR_SHIFT 0 @@ -52,16 +51,6 @@ #define _ASM_EXTABLE_UACCESS(insn, fixup) \ _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, wzr, wzr) -#define _ASM_EXTABLE_COPY_MC_PAGE_ERR_ZERO(insn, fixup, err, zero) \ - __ASM_EXTABLE_RAW(insn, fixup, \ - EX_TYPE_COPY_MC_PAGE_ERR_ZERO, \ - ( \ - EX_DATA_REG(ERR, err) | \ - EX_DATA_REG(ZERO, zero) \ - )) - -#define _ASM_EXTABLE_COPY_MC_PAGE(insn, fixup) \ - _ASM_EXTABLE_COPY_MC_PAGE_ERR_ZERO(insn, fixup, wzr, wzr) /* * Create an exception table entry for uaccess `insn`, which will branch to `fixup` * when an unhandled fault is taken. @@ -70,10 +59,6 @@ _ASM_EXTABLE_UACCESS(\insn, \fixup) .endm - .macro _asm_extable_copy_mc_page, insn, fixup - _ASM_EXTABLE_COPY_MC_PAGE(\insn, \fixup) - .endm - /* * Create an exception table entry for `insn` if `fixup` is provided. Otherwise * do nothing. diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 7712c532ce1e..c738877b8c72 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -172,10 +172,6 @@ lr .req x30 // link register #define CPU_LE(code...) code #endif -#define CPY_MC(l, x...) \ -9999: x; \ - _asm_extable_copy_mc_page 9999b, l - /* * Define a macro that constructs a 64-bit value by concatenating two * 32-bit registers. Note that on big endian systems the order of the diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 9cdded082dd4..91fbd5c8a391 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -92,7 +92,6 @@ static inline bool try_page_mte_tagging(struct page *page) void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t pte, unsigned int nr_pages); void mte_copy_page_tags(void *kto, const void *kfrom); -int mte_copy_mc_page_tags(void *kto, const void *kfrom); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_cpu_setup(void); @@ -129,10 +128,6 @@ static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } -static inline int mte_copy_mc_page_tags(void *kto, const void *kfrom) -{ - return 0; -} static inline void mte_thread_init_user(void) { } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 304cc86b8a10..2312e6ee595f 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -29,16 +29,6 @@ void copy_user_highpage(struct page *to, struct page *from, void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE -#ifdef CONFIG_ARCH_HAS_COPY_MC -int copy_mc_page(void *to, const void *from); -int copy_mc_highpage(struct page *to, struct page *from); -#define __HAVE_ARCH_COPY_MC_HIGHPAGE - -int copy_mc_user_highpage(struct page *to, struct page *from, - unsigned long vaddr, struct vm_area_struct *vma); -#define __HAVE_ARCH_COPY_MC_USER_HIGHPAGE -#endif - struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, unsigned long vaddr); #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index a2fd865b816d..29490be2546b 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -15,8 +15,6 @@ endif lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o -lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc_page.o - obj-$(CONFIG_CRC32) += crc32.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o diff --git a/arch/arm64/lib/copy_mc_page.S b/arch/arm64/lib/copy_mc_page.S deleted file mode 100644 index 656d831ef4b8..000000000000 --- a/arch/arm64/lib/copy_mc_page.S +++ /dev/null @@ -1,89 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 ARM Ltd. - */ - -#include <linux/linkage.h> -#include <linux/const.h> -#include <asm/assembler.h> -#include <asm/page.h> -#include <asm/cpufeature.h> -#include <asm/alternative.h> -#include <asm/asm-extable.h> - -/* - * Copy a page from src to dest (both are page aligned) with machine check - * - * Parameters: - * x0 - dest - * x1 - src - * Returns: - * x0 - Return 0 if copy success, or -EFAULT if anything goes wrong - * while copying. - */ -SYM_FUNC_START(__pi_copy_mc_page) -alternative_if ARM64_HAS_NO_HW_PREFETCH - // Prefetch three cache lines ahead. - prfm pldl1strm, [x1, #128] - prfm pldl1strm, [x1, #256] - prfm pldl1strm, [x1, #384] -alternative_else_nop_endif - -CPY_MC(9998f, ldp x2, x3, [x1]) -CPY_MC(9998f, ldp x4, x5, [x1, #16]) -CPY_MC(9998f, ldp x6, x7, [x1, #32]) -CPY_MC(9998f, ldp x8, x9, [x1, #48]) -CPY_MC(9998f, ldp x10, x11, [x1, #64]) -CPY_MC(9998f, ldp x12, x13, [x1, #80]) -CPY_MC(9998f, ldp x14, x15, [x1, #96]) -CPY_MC(9998f, ldp x16, x17, [x1, #112]) - - add x0, x0, #256 - add x1, x1, #128 -1: - tst x0, #(PAGE_SIZE - 1) - -alternative_if ARM64_HAS_NO_HW_PREFETCH - prfm pldl1strm, [x1, #384] -alternative_else_nop_endif - -CPY_MC(9998f, stnp x2, x3, [x0, #-256]) -CPY_MC(9998f, ldp x2, x3, [x1]) -CPY_MC(9998f, stnp x4, x5, [x0, #16 - 256]) -CPY_MC(9998f, ldp x4, x5, [x1, #16]) -CPY_MC(9998f, stnp x6, x7, [x0, #32 - 256]) -CPY_MC(9998f, ldp x6, x7, [x1, #32]) -CPY_MC(9998f, stnp x8, x9, [x0, #48 - 256]) -CPY_MC(9998f, ldp x8, x9, [x1, #48]) -CPY_MC(9998f, stnp x10, x11, [x0, #64 - 256]) -CPY_MC(9998f, ldp x10, x11, [x1, #64]) -CPY_MC(9998f, stnp x12, x13, [x0, #80 - 256]) -CPY_MC(9998f, ldp x12, x13, [x1, #80]) -CPY_MC(9998f, stnp x14, x15, [x0, #96 - 256]) -CPY_MC(9998f, ldp x14, x15, [x1, #96]) -CPY_MC(9998f, stnp x16, x17, [x0, #112 - 256]) -CPY_MC(9998f, ldp x16, x17, [x1, #112]) - - add x0, x0, #128 - add x1, x1, #128 - - b.ne 1b - -CPY_MC(9998f, stnp x2, x3, [x0, #-256]) -CPY_MC(9998f, stnp x4, x5, [x0, #16 - 256]) -CPY_MC(9998f, stnp x6, x7, [x0, #32 - 256]) -CPY_MC(9998f, stnp x8, x9, [x0, #48 - 256]) -CPY_MC(9998f, stnp x10, x11, [x0, #64 - 256]) -CPY_MC(9998f, stnp x12, x13, [x0, #80 - 256]) -CPY_MC(9998f, stnp x14, x15, [x0, #96 - 256]) -CPY_MC(9998f, stnp x16, x17, [x0, #112 - 256]) - - mov x0, #0 - ret - -9998: mov x0, #-EFAULT - ret - -SYM_FUNC_END(__pi_copy_mc_page) -SYM_FUNC_ALIAS(copy_mc_page, __pi_copy_mc_page) -EXPORT_SYMBOL(copy_mc_page) diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 2b748e83f6cf..5018ac03b6bf 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -80,33 +80,6 @@ SYM_FUNC_START(mte_copy_page_tags) ret SYM_FUNC_END(mte_copy_page_tags) -/* - * Copy the tags from the source page to the destination one wiht machine check safe - * x0 - address of the destination page - * x1 - address of the source page - * Returns: - * x0 - Return 0 if copy success, or - * -EFAULT if anything goes wrong while copying. - */ -SYM_FUNC_START(mte_copy_mc_page_tags) - mov x2, x0 - mov x3, x1 - multitag_transfer_size x5, x6 -1: -CPY_MC(2f, ldgm x4, [x3]) -CPY_MC(2f, stgm x4, [x2]) - add x2, x2, x5 - add x3, x3, x5 - tst x2, #(PAGE_SIZE - 1) - b.ne 1b - - mov x0, #0 - ret - -2: mov x0, #-EFAULT - ret -SYM_FUNC_END(mte_copy_mc_page_tags) - /* * Read tags from a user buffer (one tag per byte) and set the corresponding * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c index 9765e40cde6c..a7bb20055ce0 100644 --- a/arch/arm64/mm/copypage.c +++ b/arch/arm64/mm/copypage.c @@ -14,25 +14,6 @@ #include <asm/cpufeature.h> #include <asm/mte.h> -static int do_mte(struct page *to, struct page *from, void *kto, void *kfrom, bool mc) -{ - int ret = 0; - - if (system_supports_mte() && page_mte_tagged(from)) { - /* It's a new page, shouldn't have been tagged yet */ - WARN_ON_ONCE(!try_page_mte_tagging(to)); - if (mc) - ret = mte_copy_mc_page_tags(kto, kfrom); - else - mte_copy_page_tags(kto, kfrom); - - if (!ret) - set_page_mte_tagged(to); - } - - return ret; -} - void copy_highpage(struct page *to, struct page *from) { void *kto = page_address(to); @@ -43,7 +24,12 @@ void copy_highpage(struct page *to, struct page *from) if (kasan_hw_tags_enabled()) page_kasan_tag_reset(to); - do_mte(to, from, kto, kfrom, false); + if (system_supports_mte() && page_mte_tagged(from)) { + /* It's a new page, shouldn't have been tagged yet */ + WARN_ON_ONCE(!try_page_mte_tagging(to)); + mte_copy_page_tags(kto, kfrom); + set_page_mte_tagged(to); + } } EXPORT_SYMBOL(copy_highpage); @@ -54,43 +40,3 @@ void copy_user_highpage(struct page *to, struct page *from, flush_dcache_page(to); } EXPORT_SYMBOL_GPL(copy_user_highpage); - -#ifdef CONFIG_ARCH_HAS_COPY_MC -/* - * Return -EFAULT if anything goes wrong while copying page or mte. - */ -int copy_mc_highpage(struct page *to, struct page *from) -{ - void *kto = page_address(to); - void *kfrom = page_address(from); - int ret; - - ret = copy_mc_page(kto, kfrom); - if (ret) - return -EFAULT; - - if (kasan_hw_tags_enabled()) - page_kasan_tag_reset(to); - - ret = do_mte(to, from, kto, kfrom, true); - if (ret) - return -EFAULT; - - return 0; -} -EXPORT_SYMBOL(copy_mc_highpage); - -int copy_mc_user_highpage(struct page *to, struct page *from, - unsigned long vaddr, struct vm_area_struct *vma) -{ - int ret; - - ret = copy_mc_highpage(to, from); - - if (!ret) - flush_dcache_page(to); - - return ret; -} -EXPORT_SYMBOL_GPL(copy_mc_user_highpage); -#endif diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index bdc81518d207..28ec35e3d210 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -16,7 +16,7 @@ get_ex_fixup(const struct exception_table_entry *ex) return ((unsigned long)&ex->fixup + ex->fixup); } -static bool ex_handler_fixup_err_zero(const struct exception_table_entry *ex, +static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, struct pt_regs *regs) { int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); @@ -69,7 +69,7 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_bpf(ex, regs); case EX_TYPE_UACCESS_ERR_ZERO: case EX_TYPE_KACCESS_ERR_ZERO: - return ex_handler_fixup_err_zero(ex, regs); + return ex_handler_uaccess_err_zero(ex, regs); case EX_TYPE_LOAD_UNALIGNED_ZEROPAD: return ex_handler_load_unaligned_zeropad(ex, regs); } @@ -87,8 +87,7 @@ bool fixup_exception_mc(struct pt_regs *regs) switch (ex->type) { case EX_TYPE_UACCESS_ERR_ZERO: - case EX_TYPE_COPY_MC_PAGE_ERR_ZERO: - return ex_handler_fixup_err_zero(ex, regs); + return ex_handler_uaccess_err_zero(ex, regs); } return false; diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 254e1f69a6f5..b02a8570609d 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -332,7 +332,6 @@ static inline void copy_highpage(struct page *to, struct page *from) #endif #ifdef copy_mc_to_kernel -#ifndef __HAVE_ARCH_COPY_MC_USER_HIGHPAGE /* * If architecture supports machine check exception handling, define the * #MC versions of copy_user_highpage and copy_highpage. They copy a memory @@ -355,9 +354,7 @@ static inline int copy_mc_user_highpage(struct page *to, struct page *from, return ret ? -EFAULT : 0; } -#endif -#ifndef __HAVE_ARCH_COPY_MC_HIGHPAGE static inline int copy_mc_highpage(struct page *to, struct page *from) { unsigned long ret; @@ -373,25 +370,20 @@ static inline int copy_mc_highpage(struct page *to, struct page *from) return ret ? -EFAULT : 0; } -#endif #else -#ifndef __HAVE_ARCH_COPY_MC_USER_HIGHPAGE static inline int copy_mc_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { copy_user_highpage(to, from, vaddr, vma); return 0; } -#endif -#ifndef __HAVE_ARCH_COPY_MC_HIGHPAGE static inline int copy_mc_highpage(struct page *to, struct page *from) { copy_highpage(to, from); return 0; } #endif -#endif static inline void memcpy_page(struct page *dst_page, size_t dst_off, struct page *src_page, size_t src_off, -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- This reverts commit 681be06895261494fa655b5edc5cb6cb924014ba. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- include/linux/highmem.h | 8 ++++---- mm/khugepaged.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/highmem.h b/include/linux/highmem.h index b02a8570609d..75607d4ba26c 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -335,8 +335,8 @@ static inline void copy_highpage(struct page *to, struct page *from) /* * If architecture supports machine check exception handling, define the * #MC versions of copy_user_highpage and copy_highpage. They copy a memory - * page with #MC in source page (@from) handled, and return -EFAULT if there - * was a #MC, otherwise 0 for success. + * page with #MC in source page (@from) handled, and return the number + * of bytes not copied if there was a #MC, otherwise 0 for success. */ static inline int copy_mc_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) @@ -352,7 +352,7 @@ static inline int copy_mc_user_highpage(struct page *to, struct page *from, kunmap_local(vto); kunmap_local(vfrom); - return ret ? -EFAULT : 0; + return ret; } static inline int copy_mc_highpage(struct page *to, struct page *from) @@ -368,7 +368,7 @@ static inline int copy_mc_highpage(struct page *to, struct page *from) kunmap_local(vto); kunmap_local(vfrom); - return ret ? -EFAULT : 0; + return ret; } #else static inline int copy_mc_user_highpage(struct page *to, struct page *from, diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 0b8636537383..107608bc2ada 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -804,7 +804,7 @@ static int __collapse_huge_page_copy(pte_t *pte, continue; } src_page = pte_page(pteval); - if (copy_mc_user_highpage(page, src_page, _address, vma)) { + if (copy_mc_user_highpage(page, src_page, _address, vma) > 0) { result = SCAN_COPY_MC; break; } @@ -2071,7 +2071,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, clear_highpage(hpage + (index % HPAGE_PMD_NR)); index++; } - if (copy_mc_highpage(hpage + (page->index % HPAGE_PMD_NR), page)) { + if (copy_mc_highpage(hpage + (page->index % HPAGE_PMD_NR), page) > 0) { result = SCAN_COPY_MC; goto rollback; } -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- This reverts commit 1a1eddb97e28a08e2d427ded2bccbf5e28b9a7e6. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/lib/copy_from_user.S | 10 +++++----- arch/arm64/lib/copy_to_user.S | 10 +++++----- arch/arm64/mm/extable.c | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 1bf676e9201d..34e317907524 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -25,7 +25,7 @@ .endm .macro strb1 reg, ptr, val - USER(9998f, strb \reg, [\ptr], \val) + strb \reg, [\ptr], \val .endm .macro ldrh1 reg, ptr, val @@ -33,7 +33,7 @@ .endm .macro strh1 reg, ptr, val - USER(9998f, strh \reg, [\ptr], \val) + strh \reg, [\ptr], \val .endm .macro ldr1 reg, ptr, val @@ -41,7 +41,7 @@ .endm .macro str1 reg, ptr, val - USER(9998f, str \reg, [\ptr], \val) + str \reg, [\ptr], \val .endm .macro ldp1 reg1, reg2, ptr, val @@ -49,7 +49,7 @@ .endm .macro stp1 reg1, reg2, ptr, val - USER(9998f, stp \reg1, \reg2, [\ptr], \val) + stp \reg1, \reg2, [\ptr], \val .endm end .req x5 @@ -66,7 +66,7 @@ SYM_FUNC_START(__arch_copy_from_user) b.ne 9998f // Before being absolutely sure we couldn't copy anything, try harder USER(9998f, ldtrb tmp1w, [srcin]) -USER(9998f, strb tmp1w, [dst], #1) + strb tmp1w, [dst], #1 9998: sub x0, end, dst // bytes not copied ret SYM_FUNC_END(__arch_copy_from_user) diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index cc031bd87455..802231772608 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -20,7 +20,7 @@ * x0 - bytes not copied */ .macro ldrb1 reg, ptr, val - USER(9998f, ldrb \reg, [\ptr], \val) + ldrb \reg, [\ptr], \val .endm .macro strb1 reg, ptr, val @@ -28,7 +28,7 @@ .endm .macro ldrh1 reg, ptr, val - USER(9998f, ldrh \reg, [\ptr], \val) + ldrh \reg, [\ptr], \val .endm .macro strh1 reg, ptr, val @@ -36,7 +36,7 @@ .endm .macro ldr1 reg, ptr, val - USER(9998f, ldr \reg, [\ptr], \val) + ldr \reg, [\ptr], \val .endm .macro str1 reg, ptr, val @@ -44,7 +44,7 @@ .endm .macro ldp1 reg1, reg2, ptr, val - USER(9998f, ldp \reg1, \reg2, [\ptr], \val) + ldp \reg1, \reg2, [\ptr], \val .endm .macro stp1 reg1, reg2, ptr, val @@ -64,7 +64,7 @@ SYM_FUNC_START(__arch_copy_to_user) 9997: cmp dst, dstin b.ne 9998f // Before being absolutely sure we couldn't copy anything, try harder -USER(9998f, ldrb tmp1w, [srcin]) + ldrb tmp1w, [srcin] USER(9998f, sttrb tmp1w, [dst]) add dst, dst, #1 9998: sub x0, end, dst // bytes not copied diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index 28ec35e3d210..478e639f8680 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -85,10 +85,10 @@ bool fixup_exception_mc(struct pt_regs *regs) if (!ex) return false; - switch (ex->type) { - case EX_TYPE_UACCESS_ERR_ZERO: - return ex_handler_uaccess_err_zero(ex, regs); - } + /* + * This is not complete, More Machine check safe extable type can + * be processed here. + */ return false; } -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- This reverts commit 975e216e36aa3f7aba9d8e1e1d6cbba7cf692ad7. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/Kconfig | 1 - arch/arm64/include/asm/extable.h | 1 - arch/arm64/mm/extable.c | 16 ---------------- arch/arm64/mm/fault.c | 29 +---------------------------- 4 files changed, 1 insertion(+), 46 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 890461d3f5e5..f657a02693cc 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -20,7 +20,6 @@ config ARM64 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE - select ARCH_HAS_COPY_MC if ACPI_APEI_GHES select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEBUG_VM_PGTABLE diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h index f80ebd0addfd..72b0e71cc3de 100644 --- a/arch/arm64/include/asm/extable.h +++ b/arch/arm64/include/asm/extable.h @@ -46,5 +46,4 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, #endif /* !CONFIG_BPF_JIT */ bool fixup_exception(struct pt_regs *regs); -bool fixup_exception_mc(struct pt_regs *regs); #endif diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index 478e639f8680..228d681a8715 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -76,19 +76,3 @@ bool fixup_exception(struct pt_regs *regs) BUG(); } - -bool fixup_exception_mc(struct pt_regs *regs) -{ - const struct exception_table_entry *ex; - - ex = search_exception_tables(instruction_pointer(regs)); - if (!ex) - return false; - - /* - * This is not complete, More Machine check safe extable type can - * be processed here. - */ - - return false; -} diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 5bf25c6dc27e..9bae7fbad7a7 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -732,31 +732,6 @@ static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs) return 1; /* "fault" */ } -static bool arm64_do_kernel_sea(unsigned long addr, unsigned int esr, - struct pt_regs *regs, int sig, int code) -{ - if (!IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC)) - return false; - - if (user_mode(regs)) - return false; - - if (apei_claim_sea(regs) < 0) - return false; - - if (!fixup_exception_mc(regs)) - return false; - - if (current->flags & PF_KTHREAD) - return true; - - set_thread_esr(0, esr); - arm64_force_sig_fault(sig, code, addr, - "Uncorrected memory error on access to user memory\n"); - - return true; -} - static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs) { const struct fault_info *inf; @@ -782,9 +757,7 @@ static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs) */ siaddr = untagged_addr(far); } - - if (!arm64_do_kernel_sea(siaddr, esr, regs, inf->sig, inf->code)) - arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr); + arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr); return 0; } -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- For the arm64 kernel, when it processes hardware memory errors for synchronize notifications(do_sea()), if the errors is consumed within the kernel, the current processing is panic. However, it is not optimal. Take copy_from/to_user for example, If ld* triggers a memory error, even in kernel mode, only the associated process is affected. Killing the user process and isolating the corrupt page is a better choice. New fixup type EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE is added to identify insn that can recover from memory errors triggered by access to kernel memory. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/asm-extable.h | 31 +++++++++++++++++++++++----- arch/arm64/include/asm/asm-uaccess.h | 4 ++++ arch/arm64/include/asm/extable.h | 1 + arch/arm64/lib/copy_to_user.S | 10 ++++----- arch/arm64/mm/extable.c | 19 +++++++++++++++++ arch/arm64/mm/fault.c | 27 +++++++++++++++++------- 7 files changed, 75 insertions(+), 18 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index f657a02693cc..890461d3f5e5 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -20,6 +20,7 @@ config ARM64 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE + select ARCH_HAS_COPY_MC if ACPI_APEI_GHES select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEBUG_VM_PGTABLE diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h index 980d1dd8e1a3..9c0664fe1eb1 100644 --- a/arch/arm64/include/asm/asm-extable.h +++ b/arch/arm64/include/asm/asm-extable.h @@ -5,11 +5,13 @@ #include <linux/bits.h> #include <asm/gpr-num.h> -#define EX_TYPE_NONE 0 -#define EX_TYPE_BPF 1 -#define EX_TYPE_UACCESS_ERR_ZERO 2 -#define EX_TYPE_KACCESS_ERR_ZERO 3 -#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4 +#define EX_TYPE_NONE 0 +#define EX_TYPE_BPF 1 +#define EX_TYPE_UACCESS_ERR_ZERO 2 +#define EX_TYPE_KACCESS_ERR_ZERO 3 +#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4 +/* kernel access memory error safe */ +#define EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE 5 /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */ #define EX_DATA_REG_ERR_SHIFT 0 @@ -51,6 +53,17 @@ #define _ASM_EXTABLE_UACCESS(insn, fixup) \ _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, wzr, wzr) +#define _ASM_EXTABLE_KACCESS_ERR_ZERO_ME_SAFE(insn, fixup, err, zero) \ + __ASM_EXTABLE_RAW(insn, fixup, \ + EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE, \ + ( \ + EX_DATA_REG(ERR, err) | \ + EX_DATA_REG(ZERO, zero) \ + )) + +#define _ASM_EXTABLE_KACCESS_ME_SAFE(insn, fixup) \ + _ASM_EXTABLE_KACCESS_ERR_ZERO_ME_SAFE(insn, fixup, wzr, wzr) + /* * Create an exception table entry for uaccess `insn`, which will branch to `fixup` * when an unhandled fault is taken. @@ -69,6 +82,14 @@ .endif .endm +/* + * Create an exception table entry for kaccess me(memory error) safe `insn`, which + * will branch to `fixup` when an unhandled fault is taken. + */ + .macro _asm_extable_kaccess_me_safe, insn, fixup + _ASM_EXTABLE_KACCESS_ME_SAFE(\insn, \fixup) + .endm + #else /* __ASSEMBLY__ */ #include <linux/stringify.h> diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index 5b6efe8abeeb..7bbebfa5b710 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -57,6 +57,10 @@ alternative_else_nop_endif .endm #endif +#define KERNEL_ME_SAFE(l, x...) \ +9999: x; \ + _asm_extable_kaccess_me_safe 9999b, l + #define USER(l, x...) \ 9999: x; \ _asm_extable_uaccess 9999b, l diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h index 72b0e71cc3de..bc49443bc502 100644 --- a/arch/arm64/include/asm/extable.h +++ b/arch/arm64/include/asm/extable.h @@ -46,4 +46,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, #endif /* !CONFIG_BPF_JIT */ bool fixup_exception(struct pt_regs *regs); +bool fixup_exception_me(struct pt_regs *regs); #endif diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index 802231772608..2ac716c0d6d8 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -20,7 +20,7 @@ * x0 - bytes not copied */ .macro ldrb1 reg, ptr, val - ldrb \reg, [\ptr], \val + KERNEL_ME_SAFE(9998f, ldrb \reg, [\ptr], \val) .endm .macro strb1 reg, ptr, val @@ -28,7 +28,7 @@ .endm .macro ldrh1 reg, ptr, val - ldrh \reg, [\ptr], \val + KERNEL_ME_SAFE(9998f, ldrh \reg, [\ptr], \val) .endm .macro strh1 reg, ptr, val @@ -36,7 +36,7 @@ .endm .macro ldr1 reg, ptr, val - ldr \reg, [\ptr], \val + KERNEL_ME_SAFE(9998f, ldr \reg, [\ptr], \val) .endm .macro str1 reg, ptr, val @@ -44,7 +44,7 @@ .endm .macro ldp1 reg1, reg2, ptr, val - ldp \reg1, \reg2, [\ptr], \val + KERNEL_ME_SAFE(9998f, ldp \reg1, \reg2, [\ptr], \val) .endm .macro stp1 reg1, reg2, ptr, val @@ -64,7 +64,7 @@ SYM_FUNC_START(__arch_copy_to_user) 9997: cmp dst, dstin b.ne 9998f // Before being absolutely sure we couldn't copy anything, try harder - ldrb tmp1w, [srcin] +KERNEL_ME_SAFE(9998f, ldrb tmp1w, [srcin]) USER(9998f, sttrb tmp1w, [dst]) add dst, dst, #1 9998: sub x0, end, dst // bytes not copied diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index 228d681a8715..8c690ae61944 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -72,7 +72,26 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_uaccess_err_zero(ex, regs); case EX_TYPE_LOAD_UNALIGNED_ZEROPAD: return ex_handler_load_unaligned_zeropad(ex, regs); + case EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE: + return false; } BUG(); } + +bool fixup_exception_me(struct pt_regs *regs) +{ + const struct exception_table_entry *ex; + + ex = search_exception_tables(instruction_pointer(regs)); + if (!ex) + return false; + + switch (ex->type) { + case EX_TYPE_UACCESS_ERR_ZERO: + case EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE: + return ex_handler_uaccess_err_zero(ex, regs); + } + + return false; +} diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 9bae7fbad7a7..cbe5600912a9 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -732,21 +732,32 @@ static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs) return 1; /* "fault" */ } +/* + * APEI claimed this as a firmware-first notification. + * Some processing deferred to task_work before ret_to_user(). + */ +static bool do_apei_claim_sea(struct pt_regs *regs) +{ + if (user_mode(regs)) { + if (!apei_claim_sea(regs)) + return true; + } else if (IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC)) { + if (fixup_exception_me(regs) && !apei_claim_sea(regs)) + return true; + } + + return false; +} + static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs) { const struct fault_info *inf; unsigned long siaddr; - inf = esr_to_fault_info(esr); - - if (user_mode(regs) && apei_claim_sea(regs) == 0) { - /* - * APEI claimed this as a firmware-first notification. - * Some processing deferred to task_work before ret_to_user(). - */ + if (do_apei_claim_sea(regs)) return 0; - } + inf = esr_to_fault_info(esr); if (esr & ESR_ELx_FnV) { siaddr = 0; } else { -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- If hardware errors are encountered during page copying, returning the bytes not copied is not meaningful, and the caller cannot do any processing on the remaining data. Returning -EFAULT is more reasonable, which represents a hardware error encountered during the copying. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- include/linux/highmem.h | 8 ++++---- mm/khugepaged.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 75607d4ba26c..b02a8570609d 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -335,8 +335,8 @@ static inline void copy_highpage(struct page *to, struct page *from) /* * If architecture supports machine check exception handling, define the * #MC versions of copy_user_highpage and copy_highpage. They copy a memory - * page with #MC in source page (@from) handled, and return the number - * of bytes not copied if there was a #MC, otherwise 0 for success. + * page with #MC in source page (@from) handled, and return -EFAULT if there + * was a #MC, otherwise 0 for success. */ static inline int copy_mc_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) @@ -352,7 +352,7 @@ static inline int copy_mc_user_highpage(struct page *to, struct page *from, kunmap_local(vto); kunmap_local(vfrom); - return ret; + return ret ? -EFAULT : 0; } static inline int copy_mc_highpage(struct page *to, struct page *from) @@ -368,7 +368,7 @@ static inline int copy_mc_highpage(struct page *to, struct page *from) kunmap_local(vto); kunmap_local(vfrom); - return ret; + return ret ? -EFAULT : 0; } #else static inline int copy_mc_user_highpage(struct page *to, struct page *from, diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 107608bc2ada..0b8636537383 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -804,7 +804,7 @@ static int __collapse_huge_page_copy(pte_t *pte, continue; } src_page = pte_page(pteval); - if (copy_mc_user_highpage(page, src_page, _address, vma) > 0) { + if (copy_mc_user_highpage(page, src_page, _address, vma)) { result = SCAN_COPY_MC; break; } @@ -2071,7 +2071,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, clear_highpage(hpage + (index % HPAGE_PMD_NR)); index++; } - if (copy_mc_highpage(hpage + (page->index % HPAGE_PMD_NR), page) > 0) { + if (copy_mc_highpage(hpage + (page->index % HPAGE_PMD_NR), page)) { result = SCAN_COPY_MC; goto rollback; } -- 2.25.1
From: Marc Zyngier <maz@kernel.org> mainline inclusion from mainline-v6.8-rc1 commit 103423ad7e56d6c756738823c332c414b07899e6 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ---------------------------------------------------------------------- Back in 2016, it was argued that implementations lacking a HW prefetcher could be helped by sprinkling a number of PRFM instructions in strategic locations. In 2023, the one platform that presumably needed this hack is no longer in active use (let alone maintained), and an quick experiment shows dropping this hack only leads to a 0.4% drop on a full kernel compilation (tested on a MT30-GS0 48 CPU system). Given that this is pretty much in the noise department and that it may give odd ideas to other implementers, drop the hack for good. Suggested-by: Will Deacon <will@kernel.org> Suggested-by: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Marc Zyngier <maz@kernel.org> Acked-by: Catalin Marinas <catalin.marinas@arm.com> Link: https://lore.kernel.org/r/20231122133754.1240687-1-maz@kernel.org Signed-off-by: Will Deacon <will@kernel.org> conflicts: arch/arm64/kernel/cpufeature.c arch/arm64/tools/cpucaps Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/kernel/cpufeature.c | 16 ---------------- arch/arm64/lib/copy_page.S | 11 ----------- arch/arm64/tools/cpucaps | 1 - 3 files changed, 28 deletions(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 316c57b19c93..c0d5985012d7 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1582,16 +1582,6 @@ static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, return has_sre; } -static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused) -{ - u32 midr = read_cpuid_id(); - - /* Cavium ThunderX pass 1.x and 2.x */ - return midr_is_cpu_model_range(midr, MIDR_THUNDERX, - MIDR_CPU_VAR_REV(0, 0), - MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK)); -} - static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused) { u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); @@ -2401,12 +2391,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, ATOMIC, IMP) }, #endif /* CONFIG_ARM64_LSE_ATOMICS */ - { - .desc = "Software prefetching using PRFM", - .capability = ARM64_HAS_NO_HW_PREFETCH, - .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, - .matches = has_no_hw_prefetch, - }, { .desc = "Virtualization Host Extensions", .capability = ARM64_HAS_VIRT_HOST_EXTN, diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S index c336d2ffdec5..6a56d7cf309d 100644 --- a/arch/arm64/lib/copy_page.S +++ b/arch/arm64/lib/copy_page.S @@ -18,13 +18,6 @@ * x1 - src */ SYM_FUNC_START(__pi_copy_page) -alternative_if ARM64_HAS_NO_HW_PREFETCH - // Prefetch three cache lines ahead. - prfm pldl1strm, [x1, #128] - prfm pldl1strm, [x1, #256] - prfm pldl1strm, [x1, #384] -alternative_else_nop_endif - ldp x2, x3, [x1] ldp x4, x5, [x1, #16] ldp x6, x7, [x1, #32] @@ -39,10 +32,6 @@ alternative_else_nop_endif 1: tst x0, #(PAGE_SIZE - 1) -alternative_if ARM64_HAS_NO_HW_PREFETCH - prfm pldl1strm, [x1, #384] -alternative_else_nop_endif - stnp x2, x3, [x0, #-256] ldp x2, x3, [x1] stnp x4, x5, [x0, #16 - 256] diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index f5e1e494ba84..2d3df8c73158 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -41,7 +41,6 @@ HAS_MOPS HAS_NESTED_VIRT HAS_NMI HAS_NO_FPSIMD -HAS_NO_HW_PREFETCH HAS_PAN HAS_S1PIE HAS_RAS_EXTN -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- Currently, many scenarios that can tolerate memory errors when copying page have been supported in the kernel[1~5], all of which are implemented by copy_mc_[user]_highpage(). arm64 should also support this mechanism. Due to mte, arm64 needs to have its own copy_mc_[user]_highpage() architecture implementation, macros __HAVE_ARCH_COPY_MC_HIGHPAGE and __HAVE_ARCH_COPY_MC_USER_HIGHPAGE have been added to control it. Add new helper copy_mc_page() which provide a page copy implementation with hardware memory error safe. The code logic of copy_mc_page() is the same as copy_page(), the main difference is that the ldp insn of copy_mc_page() contains the fixup type EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE, therefore, the main logic is extracted to copy_page_template.S. [1] commit d302c2398ba2 ("mm, hwpoison: when copy-on-write hits poison, take page offline") [2] commit 1cb9dc4b475c ("mm: hwpoison: support recovery from HugePage copy-on-write faults") [3] commit 6b970599e807 ("mm: hwpoison: support recovery from ksm_might_need_to_copy()") [4] commit 98c76c9f1ef7 ("mm/khugepaged: recover from poisoned anonymous memory") [5] commit 12904d953364 ("mm/khugepaged: recover from poisoned file-backed memory") Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/include/asm/mte.h | 9 +++++ arch/arm64/include/asm/page.h | 10 ++++++ arch/arm64/lib/Makefile | 2 ++ arch/arm64/lib/copy_mc_page.S | 40 +++++++++++++++++++++ arch/arm64/lib/copy_page.S | 50 +++----------------------- arch/arm64/lib/copy_page_template.S | 56 +++++++++++++++++++++++++++++ arch/arm64/lib/mte.S | 29 +++++++++++++++ arch/arm64/mm/copypage.c | 45 +++++++++++++++++++++++ include/linux/highmem.h | 8 +++++ 9 files changed, 204 insertions(+), 45 deletions(-) create mode 100644 arch/arm64/lib/copy_mc_page.S create mode 100644 arch/arm64/lib/copy_page_template.S diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 91fbd5c8a391..dc68337c2623 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -92,6 +92,11 @@ static inline bool try_page_mte_tagging(struct page *page) void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t pte, unsigned int nr_pages); void mte_copy_page_tags(void *kto, const void *kfrom); + +#ifdef CONFIG_ARCH_HAS_COPY_MC +int mte_copy_mc_page_tags(void *kto, const void *kfrom); +#endif + void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_cpu_setup(void); @@ -128,6 +133,10 @@ static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } +static inline int mte_copy_mc_page_tags(void *kto, const void *kfrom) +{ + return 0; +} static inline void mte_thread_init_user(void) { } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 2312e6ee595f..304cc86b8a10 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -29,6 +29,16 @@ void copy_user_highpage(struct page *to, struct page *from, void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE +#ifdef CONFIG_ARCH_HAS_COPY_MC +int copy_mc_page(void *to, const void *from); +int copy_mc_highpage(struct page *to, struct page *from); +#define __HAVE_ARCH_COPY_MC_HIGHPAGE + +int copy_mc_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +#define __HAVE_ARCH_COPY_MC_USER_HIGHPAGE +#endif + struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, unsigned long vaddr); #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 29490be2546b..a2fd865b816d 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -15,6 +15,8 @@ endif lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o +lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc_page.o + obj-$(CONFIG_CRC32) += crc32.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o diff --git a/arch/arm64/lib/copy_mc_page.S b/arch/arm64/lib/copy_mc_page.S new file mode 100644 index 000000000000..a5b53a2bcac8 --- /dev/null +++ b/arch/arm64/lib/copy_mc_page.S @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 ARM Ltd. + */ + +#include <linux/linkage.h> +#include <linux/const.h> +#include <asm/assembler.h> +#include <asm/page.h> +#include <asm/cpufeature.h> +#include <asm/alternative.h> +#include <asm/asm-extable.h> +#include <asm/asm-uaccess.h> + +/* + * Copy a page from src to dest (both are page aligned) with memory error safe + * + * Parameters: + * x0 - dest + * x1 - src + * Returns: + * x0 - Return 0 if copy success, or -EFAULT if anything goes wrong + * while copying. + */ + .macro ldp1 reg1, reg2, ptr, val + KERNEL_ME_SAFE(9998f, ldp \reg1, \reg2, [\ptr, \val]) + .endm + +SYM_FUNC_START(__pi_copy_mc_page) +#include "copy_page_template.S" + + mov x0, #0 + ret + +9998: mov x0, #-EFAULT + ret + +SYM_FUNC_END(__pi_copy_mc_page) +SYM_FUNC_ALIAS(copy_mc_page, __pi_copy_mc_page) +EXPORT_SYMBOL(copy_mc_page) diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S index 6a56d7cf309d..5499f507bb75 100644 --- a/arch/arm64/lib/copy_page.S +++ b/arch/arm64/lib/copy_page.S @@ -17,52 +17,12 @@ * x0 - dest * x1 - src */ -SYM_FUNC_START(__pi_copy_page) - ldp x2, x3, [x1] - ldp x4, x5, [x1, #16] - ldp x6, x7, [x1, #32] - ldp x8, x9, [x1, #48] - ldp x10, x11, [x1, #64] - ldp x12, x13, [x1, #80] - ldp x14, x15, [x1, #96] - ldp x16, x17, [x1, #112] - - add x0, x0, #256 - add x1, x1, #128 -1: - tst x0, #(PAGE_SIZE - 1) - - stnp x2, x3, [x0, #-256] - ldp x2, x3, [x1] - stnp x4, x5, [x0, #16 - 256] - ldp x4, x5, [x1, #16] - stnp x6, x7, [x0, #32 - 256] - ldp x6, x7, [x1, #32] - stnp x8, x9, [x0, #48 - 256] - ldp x8, x9, [x1, #48] - stnp x10, x11, [x0, #64 - 256] - ldp x10, x11, [x1, #64] - stnp x12, x13, [x0, #80 - 256] - ldp x12, x13, [x1, #80] - stnp x14, x15, [x0, #96 - 256] - ldp x14, x15, [x1, #96] - stnp x16, x17, [x0, #112 - 256] - ldp x16, x17, [x1, #112] - - add x0, x0, #128 - add x1, x1, #128 - - b.ne 1b - - stnp x2, x3, [x0, #-256] - stnp x4, x5, [x0, #16 - 256] - stnp x6, x7, [x0, #32 - 256] - stnp x8, x9, [x0, #48 - 256] - stnp x10, x11, [x0, #64 - 256] - stnp x12, x13, [x0, #80 - 256] - stnp x14, x15, [x0, #96 - 256] - stnp x16, x17, [x0, #112 - 256] + .macro ldp1 reg1, reg2, ptr, val + ldp \reg1, \reg2, [\ptr, \val] + .endm +SYM_FUNC_START(__pi_copy_page) +#include "copy_page_template.S" ret SYM_FUNC_END(__pi_copy_page) SYM_FUNC_ALIAS(copy_page, __pi_copy_page) diff --git a/arch/arm64/lib/copy_page_template.S b/arch/arm64/lib/copy_page_template.S new file mode 100644 index 000000000000..b3ddec2c7a27 --- /dev/null +++ b/arch/arm64/lib/copy_page_template.S @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 ARM Ltd. + */ + +/* + * Copy a page from src to dest (both are page aligned) + * + * Parameters: + * x0 - dest + * x1 - src + */ + ldp1 x2, x3, x1, #0 + ldp1 x4, x5, x1, #16 + ldp1 x6, x7, x1, #32 + ldp1 x8, x9, x1, #48 + ldp1 x10, x11, x1, #64 + ldp1 x12, x13, x1, #80 + ldp1 x14, x15, x1, #96 + ldp1 x16, x17, x1, #112 + + add x0, x0, #256 + add x1, x1, #128 +1: + tst x0, #(PAGE_SIZE - 1) + + stnp x2, x3, [x0, #-256] + ldp1 x2, x3, x1, #0 + stnp x4, x5, [x0, #16 - 256] + ldp1 x4, x5, x1, #16 + stnp x6, x7, [x0, #32 - 256] + ldp1 x6, x7, x1, #32 + stnp x8, x9, [x0, #48 - 256] + ldp1 x8, x9, x1, #48 + stnp x10, x11, [x0, #64 - 256] + ldp1 x10, x11, x1, #64 + stnp x12, x13, [x0, #80 - 256] + ldp1 x12, x13, x1, #80 + stnp x14, x15, [x0, #96 - 256] + ldp1 x14, x15, x1, #96 + stnp x16, x17, [x0, #112 - 256] + ldp1 x16, x17, x1, #112 + + add x0, x0, #128 + add x1, x1, #128 + + b.ne 1b + + stnp x2, x3, [x0, #-256] + stnp x4, x5, [x0, #16 - 256] + stnp x6, x7, [x0, #32 - 256] + stnp x8, x9, [x0, #48 - 256] + stnp x10, x11, [x0, #64 - 256] + stnp x12, x13, [x0, #80 - 256] + stnp x14, x15, [x0, #96 - 256] + stnp x16, x17, [x0, #112 - 256] diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 5018ac03b6bf..50ef24318281 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -80,6 +80,35 @@ SYM_FUNC_START(mte_copy_page_tags) ret SYM_FUNC_END(mte_copy_page_tags) +#ifdef CONFIG_ARCH_HAS_COPY_MC +/* + * Copy the tags from the source page to the destination one wiht machine check safe + * x0 - address of the destination page + * x1 - address of the source page + * Returns: + * x0 - Return 0 if copy success, or + * -EFAULT if anything goes wrong while copying. + */ +SYM_FUNC_START(mte_copy_mc_page_tags) + mov x2, x0 + mov x3, x1 + multitag_transfer_size x5, x6 +1: +KERNEL_ME_SAFE(2f, ldgm x4, [x3]) + stgm x4, [x2] + add x2, x2, x5 + add x3, x3, x5 + tst x2, #(PAGE_SIZE - 1) + b.ne 1b + + mov x0, #0 + ret + +2: mov x0, #-EFAULT + ret +SYM_FUNC_END(mte_copy_mc_page_tags) +#endif + /* * Read tags from a user buffer (one tag per byte) and set the corresponding * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c index a7bb20055ce0..ff0d9ceea2a4 100644 --- a/arch/arm64/mm/copypage.c +++ b/arch/arm64/mm/copypage.c @@ -40,3 +40,48 @@ void copy_user_highpage(struct page *to, struct page *from, flush_dcache_page(to); } EXPORT_SYMBOL_GPL(copy_user_highpage); + +#ifdef CONFIG_ARCH_HAS_COPY_MC +/* + * Return -EFAULT if anything goes wrong while copying page or mte. + */ +int copy_mc_highpage(struct page *to, struct page *from) +{ + void *kto = page_address(to); + void *kfrom = page_address(from); + int ret; + + ret = copy_mc_page(kto, kfrom); + if (ret) + return -EFAULT; + + if (kasan_hw_tags_enabled()) + page_kasan_tag_reset(to); + + if (system_supports_mte() && page_mte_tagged(from)) { + /* It's a new page, shouldn't have been tagged yet */ + WARN_ON_ONCE(!try_page_mte_tagging(to)); + ret = mte_copy_mc_page_tags(kto, kfrom); + if (ret) + return -EFAULT; + + set_page_mte_tagged(to); + } + + return 0; +} +EXPORT_SYMBOL(copy_mc_highpage); + +int copy_mc_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma) +{ + int ret; + + ret = copy_mc_highpage(to, from); + if (!ret) + flush_dcache_page(to); + + return ret; +} +EXPORT_SYMBOL_GPL(copy_mc_user_highpage); +#endif diff --git a/include/linux/highmem.h b/include/linux/highmem.h index b02a8570609d..254e1f69a6f5 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -332,6 +332,7 @@ static inline void copy_highpage(struct page *to, struct page *from) #endif #ifdef copy_mc_to_kernel +#ifndef __HAVE_ARCH_COPY_MC_USER_HIGHPAGE /* * If architecture supports machine check exception handling, define the * #MC versions of copy_user_highpage and copy_highpage. They copy a memory @@ -354,7 +355,9 @@ static inline int copy_mc_user_highpage(struct page *to, struct page *from, return ret ? -EFAULT : 0; } +#endif +#ifndef __HAVE_ARCH_COPY_MC_HIGHPAGE static inline int copy_mc_highpage(struct page *to, struct page *from) { unsigned long ret; @@ -370,20 +373,25 @@ static inline int copy_mc_highpage(struct page *to, struct page *from) return ret ? -EFAULT : 0; } +#endif #else +#ifndef __HAVE_ARCH_COPY_MC_USER_HIGHPAGE static inline int copy_mc_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { copy_user_highpage(to, from, vaddr, vma); return 0; } +#endif +#ifndef __HAVE_ARCH_COPY_MC_HIGHPAGE static inline int copy_mc_highpage(struct page *to, struct page *from) { copy_highpage(to, from); return 0; } #endif +#endif static inline void memcpy_page(struct page *dst_page, size_t dst_off, struct page *src_page, size_t src_off, -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- The copy_mc_to_kernel() helper is memory copy implementation that handles source exceptions. It can be used in memory copy scenarios that tolerate hardware memory errors(e.g: pmem_read/dax_copy_to_iter). Currnently, only x86 and ppc suuport this helper, after arm64 support ARCH_HAS_COPY_MC , we introduce copy_mc_to_kernel() implementation. Also add memcpy_mc() for memory copy that handles source exceptions. Because there is no GPR is available for saving "bytes not copied" in memcpy(), the mempcy_mc() is referenced to the implementation of copy_from_user(). Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/arm64/include/asm/string.h | 5 +++ arch/arm64/include/asm/uaccess.h | 18 ++++++++ arch/arm64/lib/Makefile | 2 +- arch/arm64/lib/memcpy_mc.S | 73 ++++++++++++++++++++++++++++++++ mm/kasan/shadow.c | 12 ++++++ 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/lib/memcpy_mc.S diff --git a/arch/arm64/include/asm/string.h b/arch/arm64/include/asm/string.h index 3a3264ff47b9..23eca4fb24fa 100644 --- a/arch/arm64/include/asm/string.h +++ b/arch/arm64/include/asm/string.h @@ -35,6 +35,10 @@ extern void *memchr(const void *, int, __kernel_size_t); extern void *memcpy(void *, const void *, __kernel_size_t); extern void *__memcpy(void *, const void *, __kernel_size_t); +#define __HAVE_ARCH_MEMCPY_MC +extern int memcpy_mc(void *, const void *, __kernel_size_t); +extern int __memcpy_mc(void *, const void *, __kernel_size_t); + #define __HAVE_ARCH_MEMMOVE extern void *memmove(void *, const void *, __kernel_size_t); extern void *__memmove(void *, const void *, __kernel_size_t); @@ -57,6 +61,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt); */ #define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memcpy_mc(dst, src, len) __memcpy_mc(dst, src, len) #define memmove(dst, src, len) __memmove(dst, src, len) #define memset(s, c, n) __memset(s, c, n) diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 14be5000c5a0..07c1aeaeb094 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -425,4 +425,22 @@ static inline size_t probe_subpage_writeable(const char __user *uaddr, #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */ +#ifdef CONFIG_ARCH_HAS_COPY_MC +/** + * copy_mc_to_kernel - memory copy that handles source exceptions + * + * @to: destination address + * @from: source address + * @size: number of bytes to copy + * + * Return 0 for success, or bytes not copied. + */ +static inline unsigned long __must_check +copy_mc_to_kernel(void *to, const void *from, unsigned long size) +{ + return memcpy_mc(to, from, size); +} +#define copy_mc_to_kernel copy_mc_to_kernel +#endif + #endif /* __ASM_UACCESS_H */ diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index a2fd865b816d..3b6914bb8ec7 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -15,7 +15,7 @@ endif lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o -lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc_page.o +lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc_page.o memcpy_mc.o obj-$(CONFIG_CRC32) += crc32.o diff --git a/arch/arm64/lib/memcpy_mc.S b/arch/arm64/lib/memcpy_mc.S new file mode 100644 index 000000000000..1798090eba06 --- /dev/null +++ b/arch/arm64/lib/memcpy_mc.S @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> +#include <asm/cache.h> +#include <asm/asm-uaccess.h> + +/* + * Copy a buffer from src to dest (alignment handled by the hardware) + * + * Parameters: + * x0 - dest + * x1 - src + * x2 - n + * Returns: + * x0 - dest + */ + .macro ldrb1 reg, ptr, val + KERNEL_ME_SAFE(9997f, ldrb \reg, [\ptr], \val) + .endm + + .macro strb1 reg, ptr, val + strb \reg, [\ptr], \val + .endm + + .macro ldrh1 reg, ptr, val + KERNEL_ME_SAFE(9997f, ldrh \reg, [\ptr], \val) + .endm + + .macro strh1 reg, ptr, val + strh \reg, [\ptr], \val + .endm + + .macro ldr1 reg, ptr, val + KERNEL_ME_SAFE(9997f, ldr \reg, [\ptr], \val) + .endm + + .macro str1 reg, ptr, val + str \reg, [\ptr], \val + .endm + + .macro ldp1 reg1, reg2, ptr, val + KERNEL_ME_SAFE(9997f, ldp \reg1, \reg2, [\ptr], \val) + .endm + + .macro stp1 reg1, reg2, ptr, val + stp \reg1, \reg2, [\ptr], \val + .endm + +end .req x5 +SYM_FUNC_START(__memcpy_mc) + add end, x0, x2 +#include "copy_template.S" + mov x0, #0 // Nothing to copy + ret + + // Exception fixups +9997: sub x0, end, dst // bytes not copied + ret +SYM_FUNC_END(__memcpy_mc) +EXPORT_SYMBOL(__memcpy_mc) +SYM_FUNC_ALIAS_WEAK(memcpy_mc, __memcpy_mc) +EXPORT_SYMBOL(memcpy_mc) diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index dd772f9d0f08..ac8d19bdcc95 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -79,6 +79,18 @@ void *memcpy(void *dest, const void *src, size_t len) } #endif +#ifdef __HAVE_ARCH_MEMCPY_MC +#undef memcpy_mc +int memcpy_mc(void *dest, const void *src, size_t len) +{ + if (!kasan_check_range(src, len, false, _RET_IP_) || + !kasan_check_range(dest, len, true, _RET_IP_)) + return (int)len; + + return __memcpy_mc(dest, src, len); +} +#endif + void *__asan_memset(void *addr, int c, ssize_t len) { if (!kasan_check_range(addr, len, true, _RET_IP_)) -- 2.25.1
hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA ------------------------------- Add /proc/sys/kernel/machine_check_safe_enable. Set 1(default value) to enable machine check safe support. Set 0(default) to disable machine check safe support. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- Documentation/admin-guide/sysctl/kernel.rst | 14 ++++++++++ arch/arm64/mm/fault.c | 30 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index cf33de56da27..3b8953c49183 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -540,6 +540,20 @@ if leaking kernel pointer values to unprivileged users is a concern. When ``kptr_restrict`` is set to 2, kernel pointers printed using %pK will be replaced with 0s regardless of privileges. +machine_check_safe (arm64 only) +================================ + +This indicates whether the Machine Check safe memory copy feature enabled +or not���which only exists on arm64 when ARCH_HAS_COPY_MC enabled. + +The value in this file determines the behaviour of the kernel when +synchronous exception from memory copy. + += =================================================================== +0 the kernel will panic immediately. +1 the kernel will recover since a memcpy-variant provided which can + safely fail when accessing to hwpoison. += =================================================================== modprobe ======== diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index cbe5600912a9..5d040bbfd247 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -25,6 +25,7 @@ #include <linux/perf_event.h> #include <linux/preempt.h> #include <linux/hugetlb.h> +#include <linux/sysctl.h> #include <asm/acpi.h> #include <asm/bug.h> @@ -43,6 +44,31 @@ #include <asm/tlbflush.h> #include <asm/traps.h> +static int sysctl_machine_check_safe = IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC); + +#ifdef CONFIG_ARCH_HAS_COPY_MC +static struct ctl_table machine_check_safe_sysctl_table[] = { + { + .procname = "machine_check_safe", + .data = &sysctl_machine_check_safe, + .maxlen = sizeof(sysctl_machine_check_safe), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +}; + +static int __init machine_check_safe_sysctl_init(void) +{ + if (!register_sysctl("kernel", machine_check_safe_sysctl_table)) + return -EINVAL; + return 0; +} + +core_initcall(machine_check_safe_sysctl_init); +#endif + struct fault_info { int (*fn)(unsigned long far, unsigned long esr, struct pt_regs *regs); @@ -742,7 +768,9 @@ static bool do_apei_claim_sea(struct pt_regs *regs) if (!apei_claim_sea(regs)) return true; } else if (IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC)) { - if (fixup_exception_me(regs) && !apei_claim_sea(regs)) + if (sysctl_machine_check_safe && + fixup_exception_me(regs) && + !apei_claim_sea(regs)) return true; } -- 2.25.1
From: Shuai Xue <xueshuai@linux.alibaba.com> maillist inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA Reference: https://lore.kernel.org/lkml/20240204080144.7977-2-xueshuai@linux.alibaba.co... -------------------------------- Synchronous error was detected as a result of user-space process accessing a 2-bit uncorrected error. The CPU will take a synchronous error exception such as Synchronous External Abort (SEA) on Arm64. The kernel will queue a memory_failure() work which poisons the related page, unmaps the page, and then sends a SIGBUS to the process, so that a system wide panic can be avoided. However, no memory_failure() work will be queued when abnormal synchronous errors occur. These errors can include situations such as invalid PA, unexpected severity, no memory failure config support, invalid GUID section, etc. In such case, the user-space process will trigger SEA again. This loop can potentially exceed the platform firmware threshold or even trigger a kernel hard lockup, leading to a system reboot. Fix it by performing a force kill if no memory_failure() work is queued for synchronous errors. Signed-off-by: Shuai Xue <xueshuai@linux.alibaba.com> conflicts: drivers/acpi/apei/ghes.c Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- drivers/acpi/apei/ghes.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 7e0770722bc5..2136d6072fa0 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -726,6 +726,15 @@ static bool ghes_do_proc(struct ghes *ghes, #endif } + /* + * If no memory failure work is queued for abnormal synchronous + * errors, do a force kill. + */ + if (sync && !queued) { + pr_err("Sending SIGBUS to current task due to memory error not recovered"); + force_sig(SIGBUS); + } + return queued; } -- 2.25.1
From: Shuai Xue <xueshuai@linux.alibaba.com> maillist inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA Reference: https://lore.kernel.org/lkml/20240204080144.7977-3-xueshuai@linux.alibaba.co... -------------------------------- Part of return value comments for memory_failure() were originally documented at the call site. Move those comments to the function declaration to improve code readability and to provide developers with immediate access to function usage and return information. Signed-off-by: Shuai Xue <xueshuai@linux.alibaba.com> conflicts: arch/x86/kernel/cpu/mce/core.c Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- arch/x86/kernel/cpu/mce/core.c | 7 ------- mm/memory-failure.c | 9 ++++++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 211f83fbcb1b..ea56b6406e0b 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1353,13 +1353,6 @@ static void kill_me_maybe(struct callback_head *cb) return; } - /* - * -EHWPOISON from memory_failure() means that it already sent SIGBUS - * to the current process with the proper error info, - * -EOPNOTSUPP means hwpoison_filter() filtered the error event, - * - * In both cases, no further processing is required. - */ if (ret == -EHWPOISON || ret == -EOPNOTSUPP) return; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index c9372bae3524..40f6b4d540b2 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -2171,9 +2171,12 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, * Must run in process context (e.g. a work queue) with interrupts * enabled and no spinlocks held. * - * Return: 0 for successfully handled the memory error, - * -EOPNOTSUPP for hwpoison_filter() filtered the error event, - * < 0(except -EOPNOTSUPP) on failure. + * Return values: + * 0 - success + * -EOPNOTSUPP - hwpoison_filter() filtered the error event. + * -EHWPOISON - sent SIGBUS to the current process with the proper + * error info by kill_accessing_process(). + * other negative values - failure */ int memory_failure(unsigned long pfn, int flags) { -- 2.25.1
From: Shuai Xue <xueshuai@linux.alibaba.com> maillist inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I9GCZS CVE: NA Reference: https://lore.kernel.org/lkml/20240204080144.7977-4-xueshuai@linux.alibaba.co... -------------------------------- Hardware errors could be signaled by asynchronous interrupt, e.g. when an error is detected by a background scrubber, or signaled by synchronous exception, e.g. when a CPU tries to access a poisoned cache line. Since commit a70297d22132 ("ACPI: APEI: set memory failure flags as MF_ACTION_REQUIRED on synchronous events")', the flag MF_ACTION_REQUIRED could be used to determine whether a synchronous exception occurs on ARM64 platform. When a synchronous exception is detected, the kernel should terminate the current process which accessing the poisoned page. This is done by sending a SIGBUS signal with an error code BUS_MCEERR_AR, indicating an action-required machine check error on read. However, the memory failure recovery is incorrectly sending a SIGBUS with wrong error code BUS_MCEERR_AO for synchronous errors in early kill mode, even MF_ACTION_REQUIRED is set. The main problem is that synchronous errors are queued as a memory_failure() work, and are executed within a kernel thread context, not the user-space process that encountered the corrupted memory on ARM64 platform. As a result, when kill_proc() is called to terminate the process, it sends the incorrect SIGBUS error code because the context in which it operates is not the one where the error was triggered. To this end, queue memory_failure() as a task_work so that it runs in the context of the process that is actually consuming the poisoned data, and it will send SIBBUS with si_code BUS_MCEERR_AR. Signed-off-by: Shuai Xue <xueshuai@linux.alibaba.com> Tested-by: Ma Wupeng <mawupeng1@huawei.com> Reviewed-by: Kefeng Wang <wangkefeng.wang@huawei.com> Reviewed-by: Xiaofei Tan <tanxiaofei@huawei.com> Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com> conflicts: drivers/acpi/apei/ghes.c Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> --- drivers/acpi/apei/ghes.c | 77 +++++++++++++++++++++++----------------- include/acpi/ghes.h | 3 -- mm/memory-failure.c | 13 ------- 3 files changed, 44 insertions(+), 49 deletions(-) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 2136d6072fa0..eab8cb40b601 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -464,28 +464,41 @@ static void ghes_clear_estatus(struct ghes *ghes, } /* - * Called as task_work before returning to user-space. - * Ensure any queued work has been done before we return to the context that - * triggered the notification. + * struct sync_task_work - for synchronous RAS event + * + * @twork: callback_head for task work + * @pfn: page frame number of corrupted page + * @flags: fine tune action taken + * + * Structure to pass task work to be handled before + * ret_to_user via task_work_add(). */ -static void ghes_kick_task_work(struct callback_head *head) +struct sync_task_work { + struct callback_head twork; + u64 pfn; + int flags; +}; + +static void memory_failure_cb(struct callback_head *twork) { - struct acpi_hest_generic_status *estatus; - struct ghes_estatus_node *estatus_node; - u32 node_len; + int ret; + struct sync_task_work *twcb = + container_of(twork, struct sync_task_work, twork); - estatus_node = container_of(head, struct ghes_estatus_node, task_work); - if (IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE)) - memory_failure_queue_kick(estatus_node->task_work_cpu); + ret = memory_failure(twcb->pfn, twcb->flags); + gen_pool_free(ghes_estatus_pool, (unsigned long)twcb, sizeof(*twcb)); - estatus = GHES_ESTATUS_FROM_NODE(estatus_node); - node_len = GHES_ESTATUS_NODE_LEN(cper_estatus_len(estatus)); - gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node, node_len); + if (!ret || ret == -EHWPOISON || ret == -EOPNOTSUPP) + return; + + pr_err("Sending SIGBUS to current task due to memory error not recovered"); + force_sig(SIGBUS); } static bool ghes_do_memory_failure(u64 physical_addr, int flags) { unsigned long pfn; + struct sync_task_work *twcb; if (!IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE)) return false; @@ -498,6 +511,18 @@ static bool ghes_do_memory_failure(u64 physical_addr, int flags) return false; } + if (flags == MF_ACTION_REQUIRED && current->mm) { + twcb = (void *)gen_pool_alloc(ghes_estatus_pool, sizeof(*twcb)); + if (!twcb) + return false; + + twcb->pfn = pfn; + twcb->flags = flags; + init_task_work(&twcb->twork, memory_failure_cb); + task_work_add(current, &twcb->twork, TWA_RESUME); + return true; + } + memory_failure_queue(pfn, flags); return true; } @@ -676,7 +701,7 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } -static bool ghes_do_proc(struct ghes *ghes, +static void ghes_do_proc(struct ghes *ghes, const struct acpi_hest_generic_status *estatus) { int sev, sec_sev; @@ -734,8 +759,6 @@ static bool ghes_do_proc(struct ghes *ghes, pr_err("Sending SIGBUS to current task due to memory error not recovered"); force_sig(SIGBUS); } - - return queued; } static void __ghes_print_estatus(const char *pfx, @@ -1037,9 +1060,7 @@ static void ghes_proc_in_irq(struct irq_work *irq_work) struct ghes_estatus_node *estatus_node; struct acpi_hest_generic *generic; struct acpi_hest_generic_status *estatus; - bool task_work_pending; u32 len, node_len; - int ret; llnode = llist_del_all(&ghes_estatus_llist); /* @@ -1054,25 +1075,16 @@ static void ghes_proc_in_irq(struct irq_work *irq_work) estatus = GHES_ESTATUS_FROM_NODE(estatus_node); len = cper_estatus_len(estatus); node_len = GHES_ESTATUS_NODE_LEN(len); - task_work_pending = ghes_do_proc(estatus_node->ghes, estatus); + + ghes_do_proc(estatus_node->ghes, estatus); + if (!ghes_estatus_cached(estatus)) { generic = estatus_node->generic; if (ghes_print_estatus(NULL, generic, estatus)) ghes_estatus_cache_add(generic, estatus); } - - if (task_work_pending && current->mm) { - estatus_node->task_work.func = ghes_kick_task_work; - estatus_node->task_work_cpu = smp_processor_id(); - ret = task_work_add(current, &estatus_node->task_work, - TWA_RESUME); - if (ret) - estatus_node->task_work.func = NULL; - } - - if (!estatus_node->task_work.func) - gen_pool_free(ghes_estatus_pool, - (unsigned long)estatus_node, node_len); + gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node, + node_len); llnode = next; } @@ -1133,7 +1145,6 @@ static int ghes_in_nmi_queue_one_entry(struct ghes *ghes, estatus_node->ghes = ghes; estatus_node->generic = ghes->generic; - estatus_node->task_work.func = NULL; estatus = GHES_ESTATUS_FROM_NODE(estatus_node); if (__ghes_read_estatus(estatus, buf_paddr, fixmap_idx, len)) { diff --git a/include/acpi/ghes.h b/include/acpi/ghes.h index be1dd4c1a917..ebd21b05fe6e 100644 --- a/include/acpi/ghes.h +++ b/include/acpi/ghes.h @@ -35,9 +35,6 @@ struct ghes_estatus_node { struct llist_node llnode; struct acpi_hest_generic *generic; struct ghes *ghes; - - int task_work_cpu; - struct callback_head task_work; }; struct ghes_estatus_cache { diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 40f6b4d540b2..d710f6c55080 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -2462,19 +2462,6 @@ static void memory_failure_work_func(struct work_struct *work) } } -/* - * Process memory_failure work queued on the specified CPU. - * Used to avoid return-to-userspace racing with the memory_failure workqueue. - */ -void memory_failure_queue_kick(int cpu) -{ - struct memory_failure_cpu *mf_cpu; - - mf_cpu = &per_cpu(memory_failure_cpu, cpu); - cancel_work_sync(&mf_cpu->work); - memory_failure_work_func(&mf_cpu->work); -} - static int __init memory_failure_init(void) { struct memory_failure_cpu *mf_cpu; -- 2.25.1
反馈： 您发送到kernel@openeuler.org的补丁/补丁集，已成功转换为PR！ PR链接地址： https://gitee.com/openeuler/kernel/pulls/6105 邮件列表地址：https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/C... FeedBack: The patch(es) which you have sent to kernel@openeuler.org mailing list has been converted to a pull request successfully! Pull request link: https://gitee.com/openeuler/kernel/pulls/6105 Mailing list address: https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/C...
participants (2)
-
patchwork bot -
Tong Tiangen