From: Mark Rutland mark.rutland@arm.com
mainline inclusion from v4.20-rc3 commit 750319756256 category: feature bugzilla: 27615 CVE: NA
-------------------------------------------------
This patch adds basic support for pointer authentication, allowing userspace to make use of APIAKey, APIBKey, APDAKey, APDBKey, and APGAKey. The kernel maintains key values for each process (shared by all threads within), which are initialised to random values at exec() time.
The ID_AA64ISAR1_EL1.{APA,API,GPA,GPI} fields are exposed to userspace, to describe that pointer authentication instructions are available and that the kernel is managing the keys. Two new hwcaps are added for the same reason: PACA (for address authentication) and PACG (for generic authentication).
Reviewed-by: Richard Henderson richard.henderson@linaro.org Signed-off-by: Mark Rutland mark.rutland@arm.com Signed-off-by: Kristina Martsenko kristina.martsenko@arm.com Tested-by: Adam Wallis awallis@codeaurora.org Cc: Catalin Marinas catalin.marinas@arm.com Cc: Ramana Radhakrishnan ramana.radhakrishnan@arm.com Cc: Suzuki K Poulose suzuki.poulose@arm.com Cc: Will Deacon will.deacon@arm.com [will: Fix sizeof() usage and unroll address key initialisation] Signed-off-by: Will Deacon will.deacon@arm.com
Conflicts: arch/arm64/include/asm/thread_info.h arch/arm64/kernel/process.c [Zheng Zengkai: adjust context and fix conflicts caused by skipping the following commit. 3962446 arm64: preempt: Provide our own implementation of asm/preempt.h]
Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com Reviewed-by: Hanjun Guo guohanjun@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- arch/arm64/include/asm/pointer_auth.h | 79 +++++++++++++++++++++++++++ arch/arm64/include/asm/thread_info.h | 4 ++ arch/arm64/include/uapi/asm/hwcap.h | 2 + arch/arm64/kernel/cpufeature.c | 13 +++++ arch/arm64/kernel/cpuinfo.c | 2 + arch/arm64/kernel/process.c | 4 ++ 6 files changed, 104 insertions(+) create mode 100644 arch/arm64/include/asm/pointer_auth.h
diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h new file mode 100644 index 0000000000000..91c4185dda5bc --- /dev/null +++ b/arch/arm64/include/asm/pointer_auth.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __ASM_POINTER_AUTH_H +#define __ASM_POINTER_AUTH_H + +#include <linux/random.h> + +#include <asm/cpufeature.h> +#include <asm/sysreg.h> + +#ifdef CONFIG_ARM64_PTR_AUTH +/* + * Each key is a 128-bit quantity which is split across a pair of 64-bit + * registers (Lo and Hi). + */ +struct ptrauth_key { + unsigned long lo, hi; +}; + +/* + * We give each process its own keys, which are shared by all threads. The keys + * are inherited upon fork(), and reinitialised upon exec*(). + */ +struct ptrauth_keys { + struct ptrauth_key apia; + struct ptrauth_key apib; + struct ptrauth_key apda; + struct ptrauth_key apdb; + struct ptrauth_key apga; +}; + +static inline void ptrauth_keys_init(struct ptrauth_keys *keys) +{ + if (system_supports_address_auth()) { + get_random_bytes(&keys->apia, sizeof(keys->apia)); + get_random_bytes(&keys->apib, sizeof(keys->apib)); + get_random_bytes(&keys->apda, sizeof(keys->apda)); + get_random_bytes(&keys->apdb, sizeof(keys->apdb)); + } + + if (system_supports_generic_auth()) + get_random_bytes(&keys->apga, sizeof(keys->apga)); +} + +#define __ptrauth_key_install(k, v) \ +do { \ + struct ptrauth_key __pki_v = (v); \ + write_sysreg_s(__pki_v.lo, SYS_ ## k ## KEYLO_EL1); \ + write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \ +} while (0) + +static inline void ptrauth_keys_switch(struct ptrauth_keys *keys) +{ + if (system_supports_address_auth()) { + __ptrauth_key_install(APIA, keys->apia); + __ptrauth_key_install(APIB, keys->apib); + __ptrauth_key_install(APDA, keys->apda); + __ptrauth_key_install(APDB, keys->apdb); + } + + if (system_supports_generic_auth()) + __ptrauth_key_install(APGA, keys->apga); +} + +#define ptrauth_thread_init_user(tsk) \ +do { \ + struct task_struct *__ptiu_tsk = (tsk); \ + ptrauth_keys_init(&__ptiu_tsk->thread_info.keys_user); \ + ptrauth_keys_switch(&__ptiu_tsk->thread_info.keys_user); \ +} while (0) + +#define ptrauth_thread_switch(tsk) \ + ptrauth_keys_switch(&(tsk)->thread_info.keys_user) + +#else /* CONFIG_ARM64_PTR_AUTH */ +#define ptrauth_thread_init_user(tsk) +#define ptrauth_thread_switch(tsk) +#endif /* CONFIG_ARM64_PTR_AUTH */ + +#endif /* __ASM_POINTER_AUTH_H */ diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 01f063a81d648..3422c50e4f449 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -28,6 +28,7 @@ struct task_struct;
#include <asm/memory.h> +#include <asm/pointer_auth.h> #include <asm/stack_pointer.h> #include <asm/types.h>
@@ -43,6 +44,9 @@ struct thread_info { u64 ttbr0; /* saved TTBR0_EL1 */ #endif int preempt_count; /* 0 => preemptable, <0 => bug */ +#ifdef CONFIG_ARM64_PTR_AUTH + struct ptrauth_keys keys_user; +#endif };
#define thread_saved_pc(tsk) \ diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 7784f7cba16cf..5f0750c2199cc 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -50,5 +50,7 @@ #define HWCAP_FLAGM (1 << 27) #define HWCAP_SSBS (1 << 28) #define HWCAP_SB (1 << 29) +#define HWCAP_PACA (1 << 30) +#define HWCAP_PACG (1UL << 31)
#endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6fb1b9dbe71d5..fe93b9ec32189 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1283,6 +1283,12 @@ static bool can_clearpage_use_stnp(const struct arm64_cpu_capabilities *entry, }
#ifdef CONFIG_ARM64_PTR_AUTH +static void cpu_enable_address_auth(struct arm64_cpu_capabilities const *cap) +{ + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | + SCTLR_ELx_ENDA | SCTLR_ELx_ENDB); +} + static bool has_address_auth(const struct arm64_cpu_capabilities *entry, int __unused) { @@ -1588,6 +1594,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_HAS_ADDRESS_AUTH, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_address_auth, + .cpu_enable = cpu_enable_address_auth, }, { .desc = "Generic authentication (architected algorithm)", @@ -1675,6 +1682,12 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE), #endif HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, HWCAP_SSBS), +#ifdef CONFIG_ARM64_PTR_AUTH + { .desc = "HWCAP_PACA", .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_address_auth, + .hwcap_type = CAP_HWCAP, .hwcap = HWCAP_PACA }, + { .desc = "HWCAP_PACG", .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_generic_auth, + .hwcap_type = CAP_HWCAP, .hwcap = HWCAP_PACG }, +#endif {}, };
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 557ed4e805e8c..a11a66c1199dc 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -83,6 +83,8 @@ static const char *const hwcap_str[] = { "flagm", "ssbs", "sb", + "paca", + "pacg", NULL };
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index b9dded33e7b30..1073b5620c67b 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -57,6 +57,7 @@ #include <asm/fpsimd.h> #include <asm/mmu_context.h> #include <asm/processor.h> +#include <asm/pointer_auth.h> #include <asm/stacktrace.h> #include <asm/mpam_sched.h>
@@ -514,6 +515,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, contextidr_thread_switch(next); entry_task_switch(next); uao_thread_switch(next); + ptrauth_thread_switch(next); ssbs_thread_switch(next);
/* @@ -584,6 +586,8 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) void arch_setup_new_exec(void) { current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0; + + ptrauth_thread_init_user(current); }
#ifdef CONFIG_GCC_PLUGIN_STACKLEAK