This patch add uce kernel recovery path support in get_user.
get_user is implemented through macros and the uce kernel recovery use kallsyms_lookup_size_offset to confirm the location of triggering uce which based on function, Therefore, in order to make get_user support uce kernel recovery, it needs to be implemented with function.
Signed-off-by: Tong Tiangen tongtiangen@huawei.com --- arch/arm64/include/asm/exception.h | 5 +++ arch/arm64/include/asm/uaccess.h | 67 ++++++++++++++++++++++++++++++ arch/arm64/lib/Makefile | 2 + arch/arm64/lib/get_user.c | 20 +++++++++ arch/arm64/mm/fault.c | 14 +++++-- include/linux/sched.h | 1 + kernel/sysctl.c | 4 +- mm/internal.h | 1 + 8 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 arch/arm64/lib/get_user.c
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index d0c8a1fda453..91f1a336b692 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -42,16 +42,21 @@ static inline u32 disr_to_esr(u64 disr) }
#ifdef CONFIG_UCE_KERNEL_RECOVERY +/* Need set task state when trigger uce */ +#define KR_SET_TASK_STATE 0x00000001 + struct uce_kernel_recovery_info { int (*fn)(void); const char *name; unsigned long addr; unsigned long size; + unsigned int flags; };
extern int copy_page_cow_sea_fallback(void); extern int copy_generic_read_sea_fallback(void); extern int copy_from_user_sea_fallback(void); +extern int get_user_sea_fallback(void); #endif
#endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 19a7d62593ed..e9d3f4c88a76 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -331,7 +331,74 @@ do { \ __gu_err; \ })
+#ifdef CONFIG_UCE_KERNEL_RECOVERY +extern void get_user_func(long *p, const long __user *addr, int size, int *err); +extern int is_get_user_kernel_recovery_enable(void); + +#define __get_user_uce_err(x, ptr, size, err) \ +do { \ + unsigned long __gu_val; \ + __chk_user_ptr(ptr); \ + uaccess_enable_not_uao(); \ + switch ((size)) { \ + case 1: \ + __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ + break; \ + case 2: \ + __get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ + break; \ + case 4: \ + __get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ + break; \ + case 8: \ + __get_user_asm("ldr", "ldtr", "%x", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ + break; \ + default: \ + __gu_val = 0; (err) = -EFAULT; \ + break; \ + } \ + uaccess_disable_not_uao(); \ + (x) = (__force __typeof__(*(ptr)))__gu_val; \ +} while (0) + +#define __get_user_uce_check(x, ptr, size, err) \ +({ \ + __typeof__(*(ptr)) __user *__p = (ptr); \ + might_fault(); \ + if (access_ok(__p, sizeof(*__p))) { \ + __p = uaccess_mask_ptr(__p); \ + __get_user_uce_err((x), __p, (size), (err)); \ + } else { \ + (x) = 0; (err) = -EFAULT; \ + } \ +}) + +/* + * uce kernel recovery use kallsyms_lookup_size_offset to confirm the + * location of triggering uce which based on function, so here needs to + * be implemented based on function. + */ +#define get_user(x, ptr) \ +({ \ + int __gu_err = 0; \ + \ + if (!is_get_user_kernel_recovery_enable()) { \ + __get_user_check((x), (ptr), __gu_err); \ + } else { \ + long __t; \ + const long *__s = (const long *)(ptr); \ + get_user_func(&__t, __s, sizeof(*(ptr)), &__gu_err); \ + (x) = (__typeof__(x))__t; \ + } \ + __gu_err; \ +}) +#else #define get_user __get_user +#endif
#define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ asm volatile( \ diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index a7606007a749..9b92d0c19695 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -5,6 +5,8 @@ lib-y := clear_user.o delay.o copy_from_user.o \ memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ strchr.o strrchr.o tishift.o csum.o
+lib-$(CONFIG_UCE_KERNEL_RECOVERY) += get_user.o + ifeq ($(CONFIG_KERNEL_MODE_NEON), y) obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o CFLAGS_REMOVE_xor-neon.o += -mgeneral-regs-only diff --git a/arch/arm64/lib/get_user.c b/arch/arm64/lib/get_user.c new file mode 100644 index 000000000000..818ccf91c210 --- /dev/null +++ b/arch/arm64/lib/get_user.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/uaccess.h> +#include <linux/errno.h> + +/* get user space to a kernel buffer */ +noinline void get_user_func(long *p, const long __user *addr, + int size, int *err) +{ + asm volatile(".global get_user_sea_fallback\n" + "get_user_sea_fallback:\n"); + + if (unlikely(current->flags & PF_UCE_KERNEL_RECOVERY)) { + current->flags &= ~PF_UCE_KERNEL_RECOVERY; + *err = -EFAULT; + return; + } + + __get_user_uce_check(*p, addr, size, *err); +} +EXPORT_SYMBOL(get_user_func); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 50e37f4097cc..ae020feea280 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -669,9 +669,10 @@ int kernel_access_sea_recovery; * kernel_access_sea_recovery. */ static struct uce_kernel_recovery_info reco_info[] = { - {copy_page_cow_sea_fallback, "copy_page_cow", (unsigned long)copy_page_cow, 0}, - {copy_generic_read_sea_fallback, "__arch_copy_to_user_generic_read", (unsigned long)__arch_copy_to_user_generic_read, 0}, - {copy_from_user_sea_fallback, "__arch_copy_from_user", (unsigned long)__arch_copy_from_user, 0}, + {copy_page_cow_sea_fallback, "copy_page_cow", (unsigned long)copy_page_cow, 0, 0}, + {copy_generic_read_sea_fallback, "__arch_copy_to_user_generic_read", (unsigned long)__arch_copy_to_user_generic_read, 0, 0}, + {copy_from_user_sea_fallback, "__arch_copy_from_user", (unsigned long)__arch_copy_from_user, 0, 0}, + {get_user_sea_fallback, "get_user_sea_fallback", (unsigned long)get_user_sea_fallback, 0, KR_SET_TASK_STATE}, };
static int __init kernel_access_sea_recovery_init(void) @@ -723,6 +724,11 @@ int is_pagecache_reading_kernel_recovery_enable(void) return kernel_access_sea_recovery & 0x2; }
+int is_get_user_kernel_recovery_enable(void) +{ + return kernel_access_sea_recovery & 0x8; +} +EXPORT_SYMBOL(is_get_user_kernel_recovery_enable); /* * what is kernel recovery? * If the process's private data is accessed in the kernel mode to trigger @@ -855,6 +861,8 @@ static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
current->thread.fault_address = regs->pc; current->thread.fault_code = esr; + if (reco_info[idx].flags & KR_SET_TASK_STATE) + current->flags |= PF_UCE_KERNEL_RECOVERY; regs->pc = (unsigned long)reco_info[idx].fn; arm64_force_sig_info(&info, "Uncorrected hardware memory use with kernel recovery in kernel-access\n", diff --git a/include/linux/sched.h b/include/linux/sched.h index 945a57ecd9a5..4b21adcfddfb 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1441,6 +1441,7 @@ extern struct pid *cad_pid; #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ #define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */ +#define PF_UCE_KERNEL_RECOVERY 0x02000000 /* Task in uce kernel recovery state */ #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ #define PF_IO_WORKER 0x20000000 /* Task is an IO worker */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 702376d7a796..454037b155af 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -130,7 +130,7 @@ static int __maybe_unused two = 2; static int __maybe_unused three = 3; static int __maybe_unused four = 4; static int __maybe_unused five = 5; -static int __maybe_unused seven = 7; +static int __maybe_unused uce_kernel_recovery_max = 15; static unsigned long zero_ul; static unsigned long one_ul = 1; static unsigned long long_max = LONG_MAX; @@ -1281,7 +1281,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = &zero, - .extra2 = &seven, + .extra2 = &uce_kernel_recovery_max, },
#endif diff --git a/mm/internal.h b/mm/internal.h index 5a142fa64908..c94de10189eb 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -595,6 +595,7 @@ extern struct page *alloc_new_node_page(struct page *page, unsigned long node); #ifdef CONFIG_UCE_KERNEL_RECOVERY extern int is_cow_kernel_recovery_enable(void); extern int is_pagecache_reading_kernel_recovery_enable(void); +extern int is_get_user_kernel_recovery_enable(void); #endif
#endif /* __MM_INTERNAL_H */