Introduce a high-performance, zero-syscall user-mode virtual exception model via the new EL0_FGT feature. This architecture allows privileged software to intercept EL0 SVC and ERET instructions and dynamically route them straight to a user-space vector (VBAR_EL0) without the overhead of entering the EL1 kernel. Implement the core infrastructure for this hardware capability: - Define the following system registers within the implementation-defined architecture space: - VBAR_EL0 - ELR_EL0 - SPSR_EL0 - FGDT0/1_EL1 - Wire up the hardware capability matching and enabling sequence via the arm64 cpufeature framework (detecting via AIDR_EL1) - Statically lock down two operational profiles during the CPU bring-up phase: - FGDT_PROFILE_APP_AREA. For Sanbox app use. - FGDT_PROFILE_SANDBOX_AREA. For Sanbox sentry use. - Inject micro-architectural context switch isolation into __switch_to() leveraging ACTLR_EL1[20] as the global master switch per process. - Hook up control triggers via the prctl(PR_SET_SANDBOX_MODE) interface. This provides an ultra-low-overhead isolation boundary for user-space secure sandboxing and rapid syscall emulation frameworks. Clear the custom tindex flag (PSTATE bit 14) during signal delivery in setup_return() to allow unintercepted system calls within user-space signal handlers, while ensuring proper state restoration upon sigreturn. The lifecycle of the user-space sandbox and signal handling is as follows: 1. Normal Sandbox Operations (tindex=1): Standard EL0 SVCs are intercepted by the hardware emulation layer and redirected to the user-space sandbox handler (tindex=0) without trapping into EL1. The handler returns to the sandbox APP via ERET (restoring tindex to 1) completely bypassing SPSR_EL1. 2. Signal Delivery Phase (tindex=1 -> tindex=0): Asynchronous traps force an entry into EL1. The kernel pushes the original PSTATE (with tindex=1) onto the user stack within the sigframe. Then, setup_return() forcibly clears Bit 14 in regs->pstate. 3. Signal Execution & Trampoline (tindex=0): The CPU returns to EL0 to execute the signal handler. With tindex=0, any subsequent SVCs from the handler or the sigtramp page bypass sandbox redirection and trap into the kernel normally. 4. Signal Restoration Phase (EL1 -> EL0): During sigreturn, restore_sigframe() pops the archived PSTATE (with tindex=1) back into regs->pstate (SPSR_EL1). Upon kernel ERET, the emulation layer decodes SPSR_EL1 bit 14 to seamlessly restore tindex=1 for the sandboxed APP. Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- v3: - Save and restore tindex using a bit in pstate when EL0 -> EL1 or EL1 -> EL0. - Enable and vbar_el0 also treat as thread level. v2: - prctl(PR_SET_SANDBOX_MODE) support 2 params for vbar_base. - Clear el0 fgt context in arch_setup_new_exe(). - Child process inherits el0 fgt context from parent. - --- arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.turbo | 12 +++ arch/arm64/include/asm/actlr.h | 2 + arch/arm64/include/asm/cpufeature.h | 5 ++ arch/arm64/include/asm/el0_fgt.h | 79 +++++++++++++++++ arch/arm64/include/asm/processor.h | 12 +++ arch/arm64/include/asm/sysreg.h | 8 ++ arch/arm64/include/uapi/asm/ptrace.h | 1 + arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/cpufeature.c | 61 ++++++++++++++ arch/arm64/kernel/el0_fgt.c | 122 +++++++++++++++++++++++++++ arch/arm64/kernel/process.c | 20 ++++- arch/arm64/kernel/ptrace.c | 4 +- arch/arm64/kernel/signal.c | 3 + arch/arm64/tools/cpucaps | 1 + include/uapi/linux/prctl.h | 3 + kernel/fork.c | 1 + kernel/sys.c | 9 ++ 18 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 arch/arm64/include/asm/el0_fgt.h create mode 100644 arch/arm64/kernel/el0_fgt.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 954cc42589f3..3facfca71a9d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -199,6 +199,7 @@ config ARM64 select HAVE_ARM_SMCCC select HAVE_ASM_MODVERSIONS select HAVE_EBPF_JIT + select HAVE_EXIT_THREAD select HAVE_C_RECORDMCOUNT select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL diff --git a/arch/arm64/Kconfig.turbo b/arch/arm64/Kconfig.turbo index d6a18d0c4513..b8bf685b4406 100644 --- a/arch/arm64/Kconfig.turbo +++ b/arch/arm64/Kconfig.turbo @@ -112,4 +112,16 @@ config SMT_QOS SMT running LS task could occupy more uarch resources to reach better IPC. +config ARM64_EL0_FGT + bool "Support for EL0 Fine-Grained Traps (FEAT_EL0_FGT)" + depends on ARM64 + default y + help + This option enables support for EL0 Fine-Grained Traps (EL0_FGT). + It allows the kernel to intercept EL0 SVC and ERET instructions, + routing them directly to VBAR_EL0 based on ACTLR_EL1[20] control, + bypassing the traditional EL1 kernel exception entry. + + If unsure, say Y. + endmenu # "Turbo features selection" diff --git a/arch/arm64/include/asm/actlr.h b/arch/arm64/include/asm/actlr.h index 3a44b6876e81..2b7b4bc2feda 100644 --- a/arch/arm64/include/asm/actlr.h +++ b/arch/arm64/include/asm/actlr.h @@ -12,4 +12,6 @@ #define ACTLR_ELx_XINT_SHIFT (21) #define ACTLR_ELx_XINT (UL(1) << ACTLR_ELx_XINT_SHIFT) +#define ACTLR_EL1_FGT_SHIFT (22) +#define ACTLR_EL1_FGT (UL(1) << ACTLR_EL1_FGT_SHIFT) #endif /* __ASM_ACTLR_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index fed81fd3baf2..2bca7ecf092a 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -856,6 +856,11 @@ static __always_inline bool system_uses_xint(void) return IS_ENABLED(CONFIG_FAST_IRQ) && cpus_have_const_cap(ARM64_HAS_XINT); } +static __always_inline bool system_uses_el0_fgt(void) +{ + return IS_ENABLED(CONFIG_ARM64_EL0_FGT) && cpus_have_const_cap(ARM64_HAS_EL0_FGT); +} + static __always_inline bool system_uses_irq_prio_masking(void) { return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && diff --git a/arch/arm64/include/asm/el0_fgt.h b/arch/arm64/include/asm/el0_fgt.h new file mode 100644 index 000000000000..bc0899c867e0 --- /dev/null +++ b/arch/arm64/include/asm/el0_fgt.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * EL0 FGT + * + * Copyright (C) 2026- Huawei, Inc. + * Author: Jinjie Ruan <ruanjinjie@huawei.com> + * + */ +#ifndef __ASM_EL0_FGT_H +#define __ASM_EL0_FGT_H + +#include <asm/actlr.h> + +#define AIDR_EL1_FGT_SHIFT 0 +#define AIDR_EL1_FGT (UL(1) << AIDR_EL1_FGT_SHIFT) + +/* + * Bit definitions for FGDT<n>_EL1 registers. + * Defines how SVC and ERET instructions behave at EL0. + */ +#define FGDT_EL1_SVC_TRAP_EN (1ULL << 0) /* Bit[0]: 1 = Redirect SVC to VBAR_EL0 */ +#define FGDT_EL1_ERET_TRAP_EN (1ULL << 1) /* Bit[1]: 1 = Redirect ERET to ELR_EL0 */ + +/* App Area Profile: Intercept SVC, let ERET behave natively */ +#define FGDT_PROFILE_APP_AREA (FGDT_EL1_SVC_TRAP_EN) + +/* Sandbox Area Profile: Let SVC pass through (for standard syscalls), intercept ERET */ +#define FGDT_PROFILE_SANDBOX_AREA (FGDT_EL1_ERET_TRAP_EN) + +#ifdef CONFIG_ARM64_EL0_FGT +struct el0_fgt_thead_ctx { + u64 sandbox_elr_el0; + u64 sandbox_spsr_el0; + unsigned long vbar_el0; + u32 enabled; +}; + +static inline void el0_fgt_exit_thread(struct task_struct *tsk) +{ + if (!system_uses_el0_fgt()) + return; + + kfree(tsk->thread.fgt_ctx); + tsk->thread.fgt_ctx = NULL; +} + +static inline void el0_fgt_init_user(void) +{ + u64 actlr; + + if (!system_uses_el0_fgt()) + return; + + kfree(current->thread.fgt_ctx); + current->thread.fgt_ctx = NULL; + + write_sysreg_s(0, SYS_VBAR_EL0_FGT); + + actlr = read_sysreg(actlr_el1); + actlr &= ~ACTLR_EL1_FGT; + write_sysreg(actlr, actlr_el1); + isb(); +} + +extern int arch_set_sandbox_mode(unsigned long arg2, unsigned long arg3); +extern int arch_get_sandbox_mode(void); +extern void fgt_thread_switch(struct task_struct *prev, struct task_struct *next); +#else +static inline int arch_set_sandbox_mode(unsigned long arg2, unsigned long arg3) +{ + return -EINVAL; +} +static inline int arch_get_sandbox_mode(void) { return -EINVAL; } +static inline void fgt_thread_switch(struct task_struct *prev, struct task_struct *next) { } +static inline void el0_fgt_exit_thread(struct task_struct *tsk) { } +static inline void el0_fgt_init_user(void) { } +#endif /* CONFIG_ARM64_EL0_FGT */ + +#endif /* __ASM_EL0_FGT_H */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 9e688b1b13d4..07f934024a9a 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -188,7 +188,11 @@ struct thread_struct { u64 sctlr_user; u64 svcr; u64 tpidr2_el0; +#ifdef CONFIG_ARM64_EL0_FGT + void *fgt_ctx; +#else KABI_RESERVE(1) +#endif KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) @@ -308,8 +312,16 @@ static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) static inline void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { +#ifdef CONFIG_ARM64_EL0_FGT + unsigned long old_pstate = regs->pstate; +#endif + start_thread_common(regs, pc); regs->pstate = PSR_MODE_EL0t; +#ifdef CONFIG_ARM64_EL0_FGT + if (system_uses_el0_fgt() && old_pstate & PSR_TINDEX_BIT) + regs->pstate |= PSR_TINDEX_BIT; +#endif spectre_v4_enable_task_mitigation(current); regs->sp = sp; } diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 0d36f20ecd33..43613420285d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -352,6 +352,12 @@ #define SYS_MAIR_EL1 sys_reg(3, 0, 10, 2, 0) #define SYS_AMAIR_EL1 sys_reg(3, 0, 10, 3, 0) +#define SYS_ELR_EL0_FGT sys_reg(3, 0, 11, 1, 1) +#define SYS_SPSR_EL0_FGT sys_reg(3, 0, 11, 1, 2) + +#define SYS_FGDT0_EL1_FGT sys_reg(3, 0, 11, 2, 0) +#define SYS_FGDT1_EL1_FGT sys_reg(3, 0, 11, 2, 1) + #define SYS_VBAR_EL1 sys_reg(3, 0, 12, 0, 0) #define SYS_DISR_EL1 sys_reg(3, 0, 12, 1, 1) @@ -406,6 +412,8 @@ #define SYS_PMUSERENR_EL0 sys_reg(3, 3, 9, 14, 0) #define SYS_PMOVSSET_EL0 sys_reg(3, 3, 9, 14, 3) +#define SYS_VBAR_EL0_FGT sys_reg(3, 3, 11, 1, 0) + #define SYS_TPIDR_EL0 sys_reg(3, 3, 13, 0, 2) #define SYS_TPIDRRO_EL0 sys_reg(3, 3, 13, 0, 3) #define SYS_TPIDR2_EL0 sys_reg(3, 3, 13, 0, 5) diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index 7fa2f7036aa7..55188a6b301f 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -48,6 +48,7 @@ #define PSR_D_BIT 0x00000200 #define PSR_BTYPE_MASK 0x00000c00 #define PSR_SSBS_BIT 0x00001000 +#define PSR_TINDEX_BIT 0x00004000 #define PSR_PAN_BIT 0x00400000 #define PSR_UAO_BIT 0x00800000 #define PSR_DIT_BIT 0x01000000 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 51219e5721ee..4cc08bf8ca0b 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -89,6 +89,7 @@ obj-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_host.o CFLAGS_patch-scs.o += -mbranch-protection=none obj-$(CONFIG_SCHED_PARAL) += prefer_numa.o obj-$(CONFIG_BPF_RVI) += bpf-rvi.o +obj-$(CONFIG_ARM64_EL0_FGT) += el0_fgt.o # Force dependency (vdso*-wrap.S includes vdso.so through incbin) $(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9db021642fff..ec290bb9f43c 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2580,6 +2580,58 @@ static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__un } #endif +#ifdef CONFIG_ARM64_EL0_FGT +#include <asm/el0_fgt.h> + +static bool has_el0_fgt(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 aidr = read_sysreg_s(SYS_AIDR_EL1); + + return !!(aidr & AIDR_EL1_FGT); +} + +static void cpu_enable_el0_fgt(const struct arm64_cpu_capabilities *__unused) +{ + u64 val_fgdt0, val_fgdt1, val_actlr; + u64 actlr; + + /* + * Configure FGDT0_EL1 for TIndex=0 (Sandbox Area). + * Profile enables intercepting ERET and reverting to original PC. + */ + write_sysreg_s(FGDT_PROFILE_SANDBOX_AREA, SYS_FGDT0_EL1_FGT); + + /* + * Configure FGDT1_EL1 for TIndex=1 (App Area). + * Profile enables intercepting SVC and routing to VBAR_EL0. + */ + write_sysreg_s(FGDT_PROFILE_APP_AREA, SYS_FGDT1_EL1_FGT); + + /* + * Disable the core-level master switch by default; + * controlled via process scheduler later + */ + actlr = read_sysreg(actlr_el1); + actlr &= ~ACTLR_EL1_FGT; + write_sysreg(actlr, actlr_el1); + + /* Synchronize the current CPU pipeline state machine */ + isb(); + + val_fgdt0 = read_sysreg_s(SYS_FGDT0_EL1_FGT); + val_fgdt1 = read_sysreg_s(SYS_FGDT1_EL1_FGT); + val_actlr = read_sysreg(actlr_el1); + + pr_info("CPU%d enable EL0_FGT\n", smp_processor_id()); + pr_info("[FGT_DEBUG] FGDT0_EL1_FGT Readback: 0x%llx (Expected: 0x%llx)\n", + val_fgdt0, (u64)FGDT_PROFILE_SANDBOX_AREA); + pr_info("[FGT_DEBUG] FGDT1_EL1_FGT Readback: 0x%llx (Expected: 0x%llx)\n", + val_fgdt1, (u64)FGDT_PROFILE_APP_AREA); + pr_info("[FGT_DEBUG] ACTLR_EL1 Readback: 0x%llx\n", val_actlr); +} + +#endif /* CONFIG_ARM64_EL0_FGT */ + #ifdef CONFIG_ARM64_COPY_FROM_USER_OPT static bool copy_opt_disable __ro_after_init; @@ -3190,6 +3242,15 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_copy_opt, }, +#endif +#ifdef CONFIG_ARM64_EL0_FGT + { + .desc = "EL0 Fine-Grained Traps", + .capability = ARM64_HAS_EL0_FGT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_el0_fgt, + .cpu_enable = cpu_enable_el0_fgt, + }, #endif {}, }; diff --git a/arch/arm64/kernel/el0_fgt.c b/arch/arm64/kernel/el0_fgt.c new file mode 100644 index 000000000000..38ce3f7efee2 --- /dev/null +++ b/arch/arm64/kernel/el0_fgt.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define pr_fmt(fmt) "EL0_FGT: " fmt + +#include <linux/cpufeature.h> +#include <linux/sched.h> +#include <linux/sched/task_stack.h> +#include <linux/slab.h> + +#include <asm/actlr.h> +#include <asm/el0_fgt.h> +#include <asm/processor.h> +#include <asm/sysreg.h> +#include <asm/uaccess.h> + +/* + * Core handler for prctl(PR_SET_SANDBOX_MODE) + */ +int arch_set_sandbox_mode(unsigned long enable, unsigned long vbar_base) +{ + struct el0_fgt_thead_ctx *fgt = current->thread.fgt_ctx; + struct pt_regs *regs = task_pt_regs(current); + u64 actlr; + + if (!system_uses_el0_fgt()) + return -EINVAL; + + if (enable != 0 && enable != 1) + return -EINVAL; + + if (enable) { + if ((vbar_base & 0x7ff) || !access_ok((void __user *)vbar_base, 1)) + return -EINVAL; + + if (!fgt) { + fgt = kzalloc(sizeof(struct el0_fgt_thead_ctx), GFP_KERNEL); + if (!fgt) + return -ENOMEM; + current->thread.fgt_ctx = fgt; + } + + fgt->enabled = 1; + fgt->vbar_el0 = vbar_base; + write_sysreg_s(vbar_base, SYS_VBAR_EL0_FGT); + pr_info("fgt enabled is %d, VBAR_EL0: 0x%lx\n", fgt->enabled, fgt->vbar_el0); + + if (regs) + regs->pstate |= PSR_TINDEX_BIT; + } else { + if (fgt) { + fgt->enabled = 0; + fgt->vbar_el0 = 0; + } + + write_sysreg_s(0, SYS_VBAR_EL0_FGT); + + if (regs) + regs->pstate &= ~PSR_TINDEX_BIT; + } + + actlr = read_sysreg(actlr_el1); + if (enable) + actlr |= ACTLR_EL1_FGT; + else + actlr &= ~ACTLR_EL1_FGT; + + write_sysreg(actlr, actlr_el1); + isb(); + + return 0; +} + +int arch_get_sandbox_mode(void) +{ + struct el0_fgt_thead_ctx *fgt = current->thread.fgt_ctx; + + if (!system_uses_el0_fgt() || !fgt) + return -EINVAL; + + return fgt->enabled; +} + +void fgt_thread_switch(struct task_struct *prev, struct task_struct *next) +{ + struct el0_fgt_thead_ctx *prev_fgt, *next_fgt; + bool prev_active = false; + bool next_active = false; + u64 actlr; + + if (!system_uses_el0_fgt()) + return; + + prev_fgt = (struct el0_fgt_thead_ctx *)prev->thread.fgt_ctx; + next_fgt = (struct el0_fgt_thead_ctx *)next->thread.fgt_ctx; + prev_active = prev_fgt && prev_fgt->enabled; + next_active = next_fgt && next_fgt->enabled; + + /* 切出和切入的两个线程都没有开启沙盒 */ + if (!prev_active && !next_active) + return; + + /* 线程级别状态切换：保存与恢复各自的 ELR_EL0_FGT 和 SPSR_EL0_FGT */ + if (prev_active) { + prev_fgt->sandbox_elr_el0 = read_sysreg_s(SYS_ELR_EL0_FGT); + prev_fgt->sandbox_spsr_el0 = read_sysreg_s(SYS_SPSR_EL0_FGT); + } + + actlr = read_sysreg(actlr_el1); + + /* 跨进程切入：恢复新沙盒进程的上下文 */ + if (next_active) { + write_sysreg_s(next_fgt->sandbox_elr_el0, SYS_ELR_EL0_FGT); + write_sysreg_s(next_fgt->sandbox_spsr_el0, SYS_SPSR_EL0_FGT); + write_sysreg_s(next_fgt->vbar_el0, SYS_VBAR_EL0_FGT); + actlr |= ACTLR_EL1_FGT; + } else { /* 跨进程切入普通进程：清空 VBAR_EL0 并关闭 FGT 硬件 */ + write_sysreg_s(0, SYS_VBAR_EL0_FGT); + actlr &= ~ACTLR_EL1_FGT; + } + + write_sysreg(actlr, actlr_el1); + isb(); +} diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index e9e5ce956f15..1ad66c2b301a 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -56,6 +56,7 @@ #include <asm/switch_to.h> #include <asm/system_misc.h> #include <asm/xcall.h> +#include <asm/el0_fgt.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> @@ -179,7 +180,7 @@ static void print_pstate(struct pt_regs *regs) const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >> PSR_BTYPE_SHIFT]; - printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n", + printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cTINDEX %cSSBS BTYPE=%s)\n", pstate, pstate & PSR_N_BIT ? 'N' : 'n', pstate & PSR_Z_BIT ? 'Z' : 'z', @@ -193,6 +194,7 @@ static void print_pstate(struct pt_regs *regs) pstate & PSR_UAO_BIT ? '+' : '-', pstate & PSR_TCO_BIT ? '+' : '-', pstate & PSR_DIT_BIT ? '+' : '-', + pstate & PSR_TINDEX_BIT ? '+' : '-', pstate & PSR_SSBS_BIT ? '+' : '-', btype_str); } @@ -294,6 +296,10 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) /* We rely on the above assignment to initialize dst's thread_flags: */ BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK)); +#ifdef CONFIG_ARM64_EL0_FGT + dst->thread.fgt_ctx = NULL; +#endif + /* * Detach src's sve_state (if any) from dst so that it does not * get erroneously used or freed prematurely. dst's copies @@ -351,6 +357,10 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) struct pt_regs *childregs = task_pt_regs(p); memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); +#ifdef CONFIG_ARM64_EL0_FGT + if (system_uses_el0_fgt()) + p->thread.fgt_ctx = NULL; +#endif /* * In case p was allocated the same task_struct pointer as some @@ -417,6 +427,11 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) return 0; } +void exit_thread(struct task_struct *tsk) +{ + el0_fgt_exit_thread(tsk); +} + void tls_preserve_current_state(void) { *task_user_tls(current) = read_sysreg(tpidr_el0); @@ -563,6 +578,8 @@ struct task_struct *__switch_to(struct task_struct *prev, */ mpam_thread_switch(next); + fgt_thread_switch(prev, next); + /* the actual thread switch */ last = cpu_switch_to(prev, next); @@ -659,6 +676,7 @@ void arch_setup_new_exec(void) current->mm->context.flags = mmflags; ptrauth_thread_init_user(); mte_thread_init_user(); + el0_fgt_init_user(); erratum_1418040_new_exec(); if (task_spec_ssb_noexec(current)) { diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 6ddee2b5cf50..65c95907bee3 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -2259,7 +2259,7 @@ void syscall_trace_exit(struct pt_regs *regs) */ #define SPSR_EL1_AARCH64_RES0_BITS \ (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \ - GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5)) + GENMASK_ULL(20, 15) | BIT_ULL(13) | GENMASK_ULL(5, 5)) #define SPSR_EL1_AARCH32_RES0_BITS \ (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20)) @@ -2308,7 +2308,7 @@ static int valid_native_regs(struct user_pt_regs *regs) } /* Force PSR to a valid 64-bit EL0t */ - regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; + regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT | PSR_TINDEX_BIT; return 0; } diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index e83c9d9d56f8..3ee2474a8c08 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -909,6 +909,9 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, sme_smstop(); } + if (system_uses_el0_fgt()) + regs->pstate &= ~PSR_TINDEX_BIT; + if (ka->sa.sa_flags & SA_RESTORER) sigtramp = ka->sa.sa_restorer; else diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index ce0c60d48dff..4b211cba61d4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -24,6 +24,7 @@ HAS_DIT HAS_E0PD HAS_ECV HAS_ECV_CNTPOFF +HAS_EL0_FGT HAS_EPAN HAS_EVT HAS_FGT diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 370ed14b1ae0..5975f34999ac 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -306,4 +306,7 @@ struct prctl_mm_map { # define PR_RISCV_V_VSTATE_CTRL_NEXT_MASK 0xc # define PR_RISCV_V_VSTATE_CTRL_MASK 0x1f +#define PR_SET_SANDBOX_MODE 71 /* Enable/disable EL0_FGT */ +#define PR_GET_SANDBOX_MODE 72 /* Get current EL0_FGT status */ + #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 04d934eabbc7..3d6ddbb25f81 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -117,6 +117,7 @@ #ifdef CONFIG_FAST_SYSCALL #include <asm/xcall.h> #endif +#include <asm/el0_fgt.h> #include <trace/events/sched.h> diff --git a/kernel/sys.c b/kernel/sys.c index 47cb10a16b00..69ad8e39e710 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -74,6 +74,7 @@ #include <linux/uaccess.h> #include <asm/io.h> #include <asm/unistd.h> +#include <asm/el0_fgt.h> #include "uid16.h" @@ -2767,6 +2768,14 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, case PR_RISCV_V_GET_CONTROL: error = RISCV_V_GET_CONTROL(); break; +#ifdef CONFIG_ARM64_EL0_FGT + case PR_SET_SANDBOX_MODE: + error = arch_set_sandbox_mode(arg2, arg3); + break; + case PR_GET_SANDBOX_MODE: + error = arch_get_sandbox_mode(); + break; +#endif default: error = -EINVAL; break; -- 2.34.1