[PATCH 13/28] arm64: initialize and switch ptrauth kernel keys

2 Sep 2020

From: Kristina Martsenko kristina.martsenko@arm.com
mainline inclusion
from v5.7-rc1
commit 33e45234987e
category: feature
bugzilla: 27615
CVE: NA
-------------------------------------------------
Set up keys to use pointer authentication within the kernel. The kernel
will be compiled with APIAKey instructions, the other keys are currently
unused. Each task is given its own APIAKey, which is initialized during
fork. The key is changed during context switch and on kernel entry from
EL0.
The keys for idle threads need to be set before calling any C functions,
because it is not possible to enter and exit a function with different
keys.
Reviewed-by: Kees Cook keescook@chromium.org
Reviewed-by: Catalin Marinas catalin.marinas@arm.com
Reviewed-by: Vincenzo Frascino Vincenzo.Frascino@arm.com
Signed-off-by: Kristina Martsenko kristina.martsenko@arm.com
[Amit: Modified secondary cores key structure, comments]
Signed-off-by: Amit Daniel Kachhap amit.kachhap@arm.com
Signed-off-by: Catalin Marinas catalin.marinas@arm.com
Conflicts:
    arch/arm64/kernel/entry.S
    arch/arm64/kernel/smp.c
[Zheng Zengkai: fix conflicts caused by skipping the following commit.
a532508 arm64: Handle erratum 1418040 as a superset of erratum 1188873
0f80cad arm64: Restrict ARM64_ERRATUM_1188873 mitigation to AArch32
5b1cfe3 arm64: smp: Don't enter kernel with NULL stack pointer or
task struct]
Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com
Reviewed-by: Hanjun Guo guohanjun@huawei.com
Signed-off-by: Yang Yingliang yangyingliang@huawei.com
---
 arch/arm64/include/asm/asm_pointer_auth.h | 14 ++++++++++++++
 arch/arm64/include/asm/pointer_auth.h     | 13 +++++++++++++
 arch/arm64/include/asm/processor.h        |  1 +
 arch/arm64/include/asm/smp.h              |  4 ++++
 arch/arm64/kernel/asm-offsets.c           |  5 +++++
 arch/arm64/kernel/entry.S                 |  3 +++
 arch/arm64/kernel/process.c               |  2 ++
 arch/arm64/kernel/smp.c                   |  8 ++++++++
 arch/arm64/mm/proc.S                      | 12 ++++++++++++
 9 files changed, 62 insertions(+)

diff --git a/arch/arm64/include/asm/asm_pointer_auth.h b/arch/arm64/include/asm/asm_pointer_auth.h
index 3482348ec07ff..d3f4aee42851d 100644
--- a/arch/arm64/include/asm/asm_pointer_auth.h
+++ b/arch/arm64/include/asm/asm_pointer_auth.h
@@ -39,11 +39,25 @@ alternative_if ARM64_HAS_GENERIC_AUTH
 alternative_else_nop_endif
    .endm
+	.macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3
+alternative_if ARM64_HAS_ADDRESS_AUTH
+	mov	\tmp1, #THREAD_KEYS_KERNEL
+	add	\tmp1, \tsk, \tmp1
+	ldp	\tmp2, \tmp3, [\tmp1, #PTRAUTH_KERNEL_KEY_APIA]
+	msr_s	SYS_APIAKEYLO_EL1, \tmp2
+	msr_s	SYS_APIAKEYHI_EL1, \tmp3
+	isb
+alternative_else_nop_endif
+	.endm
+
 #else /* CONFIG_ARM64_PTR_AUTH */
.macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
    .endm
+	.macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3
+	.endm
+
 #endif /* CONFIG_ARM64_PTR_AUTH */
#endif /* __ASM_ASM_POINTER_AUTH_H */
diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h
index 404276bebbbec..5848e739b0cc0 100644
--- a/arch/arm64/include/asm/pointer_auth.h
+++ b/arch/arm64/include/asm/pointer_auth.h
@@ -30,6 +30,10 @@ struct ptrauth_keys_user {
    struct ptrauth_key apga;
 };
+struct ptrauth_keys_kernel {
+	struct ptrauth_key apia;
+};
+
 static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys)
 {
    if (system_supports_address_auth()) {
@@ -50,6 +54,12 @@ do {								\
    write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1);	\
 } while (0)
+static inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys)
+{
+	if (system_supports_address_auth())
+		get_random_bytes(&keys->apia, sizeof(keys->apia));
+}
+
 extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg);
/*
@@ -66,11 +76,14 @@ static inline unsigned long ptrauth_strip_insn_pac(unsigned long ptr)
#define ptrauth_thread_init_user(tsk)					\
    ptrauth_keys_init_user(&(tsk)->thread.keys_user)
+#define ptrauth_thread_init_kernel(tsk)					\
+	ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel)
#else /* CONFIG_ARM64_PTR_AUTH */
 #define ptrauth_prctl_reset_keys(tsk, arg)	(-EINVAL)
 #define ptrauth_strip_insn_pac(lr)	(lr)
 #define ptrauth_thread_init_user(tsk)
+#define ptrauth_thread_init_kernel(tsk)
 #endif /* CONFIG_ARM64_PTR_AUTH */
#endif /* __ASM_POINTER_AUTH_H */
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index e25269c47d70c..3aa17494ee1ee 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -145,6 +145,7 @@ struct thread_struct {
    struct debug_info	debug;		/* debugging */
 #ifdef CONFIG_ARM64_PTR_AUTH
    struct ptrauth_keys_user	keys_user;
+	struct ptrauth_keys_kernel	keys_kernel;
 #endif
 };
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index fc93ea608d1af..0faa0f62f4033 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -42,6 +42,7 @@
 #include <linux/threads.h>
 #include <linux/cpumask.h>
 #include <linux/thread_info.h>
+#include <asm/pointer_auth.h>
DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
@@ -93,6 +94,9 @@ asmlinkage void secondary_start_kernel(void);
 struct secondary_data {
    void *stack;
    struct task_struct *task;
+#ifdef CONFIG_ARM64_PTR_AUTH
+	struct ptrauth_keys_kernel ptrauth_key;
+#endif
    long status;
 };
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index c6df491da71bd..35f1b16956a88 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -50,6 +50,7 @@ int main(void)
   DEFINE(THREAD_CPU_CONTEXT,	offsetof(struct task_struct, thread.cpu_context));
 #ifdef CONFIG_ARM64_PTR_AUTH
   DEFINE(THREAD_KEYS_USER,	offsetof(struct task_struct, thread.keys_user));
+  DEFINE(THREAD_KEYS_KERNEL,	offsetof(struct task_struct, thread.keys_kernel));
 #endif
   BLANK();
   DEFINE(S_X0,			offsetof(struct pt_regs, regs[0]));
@@ -137,6 +138,9 @@ int main(void)
   BLANK();
   DEFINE(CPU_BOOT_STACK,	offsetof(struct secondary_data, stack));
   DEFINE(CPU_BOOT_TASK,		offsetof(struct secondary_data, task));
+#ifdef CONFIG_ARM64_PTR_AUTH
+  DEFINE(CPU_BOOT_PTRAUTH_KEY,	offsetof(struct secondary_data, ptrauth_key));
+#endif
   BLANK();
 #ifdef CONFIG_KVM_ARM_HOST
   DEFINE(VCPU_CONTEXT,		offsetof(struct kvm_vcpu, arch.ctxt));
@@ -181,6 +185,7 @@ int main(void)
   DEFINE(PTRAUTH_USER_KEY_APDA,		offsetof(struct ptrauth_keys_user, apda));
   DEFINE(PTRAUTH_USER_KEY_APDB,		offsetof(struct ptrauth_keys_user, apdb));
   DEFINE(PTRAUTH_USER_KEY_APGA,		offsetof(struct ptrauth_keys_user, apga));
+  DEFINE(PTRAUTH_KERNEL_KEY_APIA,	offsetof(struct ptrauth_keys_kernel, apia));
   BLANK();
 #endif
   return 0;
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 65457545b3b61..2eb1b657de2fb 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -184,6 +184,7 @@ alternative_cb_end
apply_ssbd 1, x22, x23
+	ptrauth_keys_install_kernel tsk, x20, x22, x23
    .else
    add	x21, sp, #S_FRAME_SIZE
    get_thread_info tsk
@@ -340,6 +341,7 @@ alternative_if ARM64_WORKAROUND_845719
 alternative_else_nop_endif
 #endif
 3:
+	/* No kernel C function calls after this as user keys are set. */
    ptrauth_keys_install_user tsk, x0, x1, x2
apply_ssbd 0, x0, x1
@@ -1161,6 +1163,7 @@ ENTRY(cpu_switch_to)
    ldr	lr, [x8]
    mov	sp, x9
    msr	sp_el0, x1
+	ptrauth_keys_install_kernel x1, x8, x9, x10
    ret
 ENDPROC(cpu_switch_to)
 NOKPROBE(cpu_switch_to)
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 878e5bf8a43e8..a5f219c1296fa 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -381,6 +381,8 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,
     */
    fpsimd_flush_task_state(p);
+	ptrauth_thread_init_kernel(p);
+
    if (likely(!(p->flags & PF_KTHREAD))) {
    	*childregs = *current_pt_regs();
    	childregs->regs[0] = 0;
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 5235b9aa05241..ce0e7055dbf7e 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -125,6 +125,10 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
     */
    secondary_data.task = idle;
    secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
+#if defined(CONFIG_ARM64_PTR_AUTH)
+	secondary_data.ptrauth_key.apia.lo = idle->thread.keys_kernel.apia.lo;
+	secondary_data.ptrauth_key.apia.hi = idle->thread.keys_kernel.apia.hi;
+#endif
    update_cpu_boot_status(CPU_MMU_OFF);
    __flush_dcache_area(&secondary_data, sizeof(secondary_data));
@@ -155,6 +159,10 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
secondary_data.task = NULL;
    secondary_data.stack = NULL;
+#if defined(CONFIG_ARM64_PTR_AUTH)
+	secondary_data.ptrauth_key.apia.lo = 0;
+	secondary_data.ptrauth_key.apia.hi = 0;
+#endif
    status = READ_ONCE(secondary_data.status);
    if (ret && status) {
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 5a7c3a2b8e27c..df178bed25edf 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -477,6 +477,10 @@ ENTRY(__cpu_setup)
    ubfx	x2, x2, #ID_AA64ISAR1_APA_SHIFT, #8
    cbz	x2, 3f
+	/*
+	 * The primary cpu keys are reset here and can be
+	 * re-initialised with some proper values later.
+	 */
    msr_s	SYS_APIAKEYLO_EL1, xzr
    msr_s	SYS_APIAKEYHI_EL1, xzr
@@ -489,6 +493,14 @@ alternative_if_not ARM64_HAS_ADDRESS_AUTH
    b	3f
 alternative_else_nop_endif
+	/* Install ptrauth key for secondary cpus */
+	adr_l	x2, secondary_data
+	ldr	x3, [x2, #CPU_BOOT_TASK]	// get secondary_data.task
+	cbz	x3, 2f				// check for slow booting cpus
+	ldp	x3, x4, [x2, #CPU_BOOT_PTRAUTH_KEY]
+	msr_s	SYS_APIAKEYLO_EL1, x3
+	msr_s	SYS_APIAKEYHI_EL1, x4
+
 2:	/* Enable ptrauth instructions */
    ldr	x2, =SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \
    	     SCTLR_ELx_ENDA | SCTLR_ELx_ENDB
-- 
2.25.1

    

2024

2023

2022

2021

2020

2019

[PATCH 13/28] arm64: initialize and switch ptrauth kernel keys