Introduce a high-performance, zero-syscall user-mode virtual exception model via the new EL0_FGT feature. This architecture allows privileged software to intercept EL0 SVC and ERET instructions and dynamically route them straight to a user-space vector (VBAR_EL0) without the overhead of entering the EL1 kernel. Implement the core infrastructure for this hardware capability: - Define the following system registers within the implementation-defined architecture space: - VBAR_EL0 - ELR_EL0 - SPSR_EL0 - FGDT0/1_EL1 - Wire up the hardware capability matching and enabling sequence via the arm64 cpufeature framework (detecting via AIDR_EL1) - Statically lock down two operational profiles during the CPU bring-up phase: - FGDT_PROFILE_APP_AREA. For Sanbox app use. - FGDT_PROFILE_SANDBOX_AREA. For Sanbox sentry use. - Inject micro-architectural context switch isolation into __switch_to() leveraging ACTLR_EL1[20] as the global master switch per process. - Hook up control triggers via the prctl(PR_SET_SANDBOX_MODE) interface. This provides an ultra-low-overhead isolation boundary for user-space secure sandboxing and rapid syscall emulation frameworks. Clear the custom tindex flag (PSTATE bit 14) during signal delivery in setup_return() to allow unintercepted system calls within user-space signal handlers, while ensuring proper state restoration upon sigreturn. The lifecycle of the user-space sandbox and signal handling is as follows: 1. Normal Sandbox Operations (tindex=1): Standard EL0 SVCs are intercepted by the hardware emulation layer and redirected to the user-space sandbox handler (tindex=0) without trapping into EL1. The handler returns to the sandbox APP via ERET (restoring tindex to 1) completely bypassing SPSR_EL1. 2. Signal Delivery Phase (tindex=1 -> tindex=0): Asynchronous traps force an entry into EL1. The kernel pushes the original PSTATE (with tindex=1) onto the user stack within the sigframe. Then, setup_return() forcibly clears Bit 14 in regs->pstate. 3. Signal Execution & Trampoline (tindex=0): The CPU returns to EL0 to execute the signal handler. With tindex=0, any subsequent SVCs from the handler or the sigtramp page bypass sandbox redirection and trap into the kernel normally. 4. Signal Restoration Phase (EL1 -> EL0): During sigreturn, restore_sigframe() pops the archived PSTATE (with tindex=1) back into regs->pstate (SPSR_EL1). Upon kernel ERET, the emulation layer decodes SPSR_EL1 bit 14 to seamlessly restore tindex=1 for the sandboxed APP. Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- v3: - Save and restore tindex using a bit in pstate when EL0 -> EL1 or EL1 -> EL0. - Enable and vbar_el0 also treat as thread level. v2: - prctl(PR_SET_SANDBOX_MODE) support 2 params for vbar_base. - Clear el0 fgt context in arch_setup_new_exe(). - Child process inherits el0 fgt context from parent. - --- arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.turbo | 12 +++ arch/arm64/include/asm/actlr.h | 2 + arch/arm64/include/asm/cpufeature.h | 5 ++ arch/arm64/include/asm/el0_fgt.h | 79 +++++++++++++++++ arch/arm64/include/asm/processor.h | 12 +++ arch/arm64/include/asm/sysreg.h | 8 ++ arch/arm64/include/uapi/asm/ptrace.h | 1 + arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/cpufeature.c | 61 ++++++++++++++ arch/arm64/kernel/el0_fgt.c | 122 +++++++++++++++++++++++++++ arch/arm64/kernel/process.c | 20 ++++- arch/arm64/kernel/ptrace.c | 4 +- arch/arm64/kernel/signal.c | 3 + arch/arm64/tools/cpucaps | 1 + include/uapi/linux/prctl.h | 3 + kernel/fork.c | 1 + kernel/sys.c | 9 ++ 18 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 arch/arm64/include/asm/el0_fgt.h create mode 100644 arch/arm64/kernel/el0_fgt.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 954cc42589f3..3facfca71a9d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -199,6 +199,7 @@ config ARM64 select HAVE_ARM_SMCCC select HAVE_ASM_MODVERSIONS select HAVE_EBPF_JIT + select HAVE_EXIT_THREAD select HAVE_C_RECORDMCOUNT select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL diff --git a/arch/arm64/Kconfig.turbo b/arch/arm64/Kconfig.turbo index d6a18d0c4513..b8bf685b4406 100644 --- a/arch/arm64/Kconfig.turbo +++ b/arch/arm64/Kconfig.turbo @@ -112,4 +112,16 @@ config SMT_QOS SMT running LS task could occupy more uarch resources to reach better IPC. +config ARM64_EL0_FGT + bool "Support for EL0 Fine-Grained Traps (FEAT_EL0_FGT)" + depends on ARM64 + default y + help + This option enables support for EL0 Fine-Grained Traps (EL0_FGT). + It allows the kernel to intercept EL0 SVC and ERET instructions, + routing them directly to VBAR_EL0 based on ACTLR_EL1[20] control, + bypassing the traditional EL1 kernel exception entry. + + If unsure, say Y. + endmenu # "Turbo features selection" diff --git a/arch/arm64/include/asm/actlr.h b/arch/arm64/include/asm/actlr.h index 3a44b6876e81..2b7b4bc2feda 100644 --- a/arch/arm64/include/asm/actlr.h +++ b/arch/arm64/include/asm/actlr.h @@ -12,4 +12,6 @@ #define ACTLR_ELx_XINT_SHIFT (21) #define ACTLR_ELx_XINT (UL(1) << ACTLR_ELx_XINT_SHIFT) +#define ACTLR_EL1_FGT_SHIFT (22) +#define ACTLR_EL1_FGT (UL(1) << ACTLR_EL1_FGT_SHIFT) #endif /* __ASM_ACTLR_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index fed81fd3baf2..2bca7ecf092a 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -856,6 +856,11 @@ static __always_inline bool system_uses_xint(void) return IS_ENABLED(CONFIG_FAST_IRQ) && cpus_have_const_cap(ARM64_HAS_XINT); } +static __always_inline bool system_uses_el0_fgt(void) +{ + return IS_ENABLED(CONFIG_ARM64_EL0_FGT) && cpus_have_const_cap(ARM64_HAS_EL0_FGT); +} + static __always_inline bool system_uses_irq_prio_masking(void) { return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && diff --git a/arch/arm64/include/asm/el0_fgt.h b/arch/arm64/include/asm/el0_fgt.h new file mode 100644 index 000000000000..bc0899c867e0 --- /dev/null +++ b/arch/arm64/include/asm/el0_fgt.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * EL0 FGT + * + * Copyright (C) 2026- Huawei, Inc. + * Author: Jinjie Ruan <ruanjinjie@huawei.com> + * + */ +#ifndef __ASM_EL0_FGT_H +#define __ASM_EL0_FGT_H + +#include <asm/actlr.h> + +#define AIDR_EL1_FGT_SHIFT 0 +#define AIDR_EL1_FGT (UL(1) << AIDR_EL1_FGT_SHIFT) + +/* + * Bit definitions for FGDT<n>_EL1 registers. + * Defines how SVC and ERET instructions behave at EL0. + */ +#define FGDT_EL1_SVC_TRAP_EN (1ULL << 0) /* Bit[0]: 1 = Redirect SVC to VBAR_EL0 */ +#define FGDT_EL1_ERET_TRAP_EN (1ULL << 1) /* Bit[1]: 1 = Redirect ERET to ELR_EL0 */ + +/* App Area Profile: Intercept SVC, let ERET behave natively */ +#define FGDT_PROFILE_APP_AREA (FGDT_EL1_SVC_TRAP_EN) + +/* Sandbox Area Profile: Let SVC pass through (for standard syscalls), intercept ERET */ +#define FGDT_PROFILE_SANDBOX_AREA (FGDT_EL1_ERET_TRAP_EN) + +#ifdef CONFIG_ARM64_EL0_FGT +struct el0_fgt_thead_ctx { + u64 sandbox_elr_el0; + u64 sandbox_spsr_el0; + unsigned long vbar_el0; + u32 enabled; +}; + +static inline void el0_fgt_exit_thread(struct task_struct *tsk) +{ + if (!system_uses_el0_fgt()) + return; + + kfree(tsk->thread.fgt_ctx); + tsk->thread.fgt_ctx = NULL; +} + +static inline void el0_fgt_init_user(void) +{ + u64 actlr; + + if (!system_uses_el0_fgt()) + return; + + kfree(current->thread.fgt_ctx); + current->thread.fgt_ctx = NULL; + + write_sysreg_s(0, SYS_VBAR_EL0_FGT); + + actlr = read_sysreg(actlr_el1); + actlr &= ~ACTLR_EL1_FGT; + write_sysreg(actlr, actlr_el1); + isb(); +} + +extern int arch_set_sandbox_mode(unsigned long arg2, unsigned long arg3); +extern int arch_get_sandbox_mode(void); +extern void fgt_thread_switch(struct task_struct *prev, struct task_struct *next); +#else +static inline int arch_set_sandbox_mode(unsigned long arg2, unsigned long arg3) +{ + return -EINVAL; +} +static inline int arch_get_sandbox_mode(void) { return -EINVAL; } +static inline void fgt_thread_switch(struct task_struct *prev, struct task_struct *next) { } +static inline void el0_fgt_exit_thread(struct task_struct *tsk) { } +static inline void el0_fgt_init_user(void) { } +#endif /* CONFIG_ARM64_EL0_FGT */ + +#endif /* __ASM_EL0_FGT_H */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 9e688b1b13d4..07f934024a9a 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -188,7 +188,11 @@ struct thread_struct { u64 sctlr_user; u64 svcr; u64 tpidr2_el0; +#ifdef CONFIG_ARM64_EL0_FGT + void *fgt_ctx; +#else KABI_RESERVE(1) +#endif KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) @@ -308,8 +312,16 @@ static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) static inline void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { +#ifdef CONFIG_ARM64_EL0_FGT + unsigned long old_pstate = regs->pstate; +#endif + start_thread_common(regs, pc); regs->pstate = PSR_MODE_EL0t; +#ifdef CONFIG_ARM64_EL0_FGT + if (system_uses_el0_fgt() && old_pstate & PSR_TINDEX_BIT) + regs->pstate |= PSR_TINDEX_BIT; +#endif spectre_v4_enable_task_mitigation(current); regs->sp = sp; } diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 0d36f20ecd33..43613420285d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -352,6 +352,12 @@ #define SYS_MAIR_EL1 sys_reg(3, 0, 10, 2, 0) #define SYS_AMAIR_EL1 sys_reg(3, 0, 10, 3, 0) +#define SYS_ELR_EL0_FGT sys_reg(3, 0, 11, 1, 1) +#define SYS_SPSR_EL0_FGT sys_reg(3, 0, 11, 1, 2) + +#define SYS_FGDT0_EL1_FGT sys_reg(3, 0, 11, 2, 0) +#define SYS_FGDT1_EL1_FGT sys_reg(3, 0, 11, 2, 1) + #define SYS_VBAR_EL1 sys_reg(3, 0, 12, 0, 0) #define SYS_DISR_EL1 sys_reg(3, 0, 12, 1, 1) @@ -406,6 +412,8 @@ #define SYS_PMUSERENR_EL0 sys_reg(3, 3, 9, 14, 0) #define SYS_PMOVSSET_EL0 sys_reg(3, 3, 9, 14, 3) +#define SYS_VBAR_EL0_FGT sys_reg(3, 3, 11, 1, 0) + #define SYS_TPIDR_EL0 sys_reg(3, 3, 13, 0, 2) #define SYS_TPIDRRO_EL0 sys_reg(3, 3, 13, 0, 3) #define SYS_TPIDR2_EL0 sys_reg(3, 3, 13, 0, 5) diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index 7fa2f7036aa7..55188a6b301f 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -48,6 +48,7 @@ #define PSR_D_BIT 0x00000200 #define PSR_BTYPE_MASK 0x00000c00 #define PSR_SSBS_BIT 0x00001000 +#define PSR_TINDEX_BIT 0x00004000 #define PSR_PAN_BIT 0x00400000 #define PSR_UAO_BIT 0x00800000 #define PSR_DIT_BIT 0x01000000 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 51219e5721ee..4cc08bf8ca0b 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -89,6 +89,7 @@ obj-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_host.o CFLAGS_patch-scs.o += -mbranch-protection=none obj-$(CONFIG_SCHED_PARAL) += prefer_numa.o obj-$(CONFIG_BPF_RVI) += bpf-rvi.o +obj-$(CONFIG_ARM64_EL0_FGT) += el0_fgt.o # Force dependency (vdso*-wrap.S includes vdso.so through incbin) $(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9db021642fff..ec290bb9f43c 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2580,6 +2580,58 @@ static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__un } #endif +#ifdef CONFIG_ARM64_EL0_FGT +#include <asm/el0_fgt.h> + +static bool has_el0_fgt(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 aidr = read_sysreg_s(SYS_AIDR_EL1); + + return !!(aidr & AIDR_EL1_FGT); +} + +static void cpu_enable_el0_fgt(const struct arm64_cpu_capabilities *__unused) +{ + u64 val_fgdt0, val_fgdt1, val_actlr; + u64 actlr; + + /* + * Configure FGDT0_EL1 for TIndex=0 (Sandbox Area). + * Profile enables intercepting ERET and reverting to original PC. + */ + write_sysreg_s(FGDT_PROFILE_SANDBOX_AREA, SYS_FGDT0_EL1_FGT); + + /* + * Configure FGDT1_EL1 for TIndex=1 (App Area). + * Profile enables intercepting SVC and routing to VBAR_EL0. + */ + write_sysreg_s(FGDT_PROFILE_APP_AREA, SYS_FGDT1_EL1_FGT); + + /* + * Disable the core-level master switch by default; + * controlled via process scheduler later + */ + actlr = read_sysreg(actlr_el1); + actlr &= ~ACTLR_EL1_FGT; + write_sysreg(actlr, actlr_el1); + + /* Synchronize the current CPU pipeline state machine */ + isb(); + + val_fgdt0 = read_sysreg_s(SYS_FGDT0_EL1_FGT); + val_fgdt1 = read_sysreg_s(SYS_FGDT1_EL1_FGT); + val_actlr = read_sysreg(actlr_el1); + + pr_info("CPU%d enable EL0_FGT\n", smp_processor_id()); + pr_info("[FGT_DEBUG] FGDT0_EL1_FGT Readback: 0x%llx (Expected: 0x%llx)\n", + val_fgdt0, (u64)FGDT_PROFILE_SANDBOX_AREA); + pr_info("[FGT_DEBUG] FGDT1_EL1_FGT Readback: 0x%llx (Expected: 0x%llx)\n", + val_fgdt1, (u64)FGDT_PROFILE_APP_AREA); + pr_info("[FGT_DEBUG] ACTLR_EL1 Readback: 0x%llx\n", val_actlr); +} + +#endif /* CONFIG_ARM64_EL0_FGT */ + #ifdef CONFIG_ARM64_COPY_FROM_USER_OPT static bool copy_opt_disable __ro_after_init; @@ -3190,6 +3242,15 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_copy_opt, }, +#endif +#ifdef CONFIG_ARM64_EL0_FGT + { + .desc = "EL0 Fine-Grained Traps", + .capability = ARM64_HAS_EL0_FGT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_el0_fgt, + .cpu_enable = cpu_enable_el0_fgt, + }, #endif {}, }; diff --git a/arch/arm64/kernel/el0_fgt.c b/arch/arm64/kernel/el0_fgt.c new file mode 100644 index 000000000000..38ce3f7efee2 --- /dev/null +++ b/arch/arm64/kernel/el0_fgt.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define pr_fmt(fmt) "EL0_FGT: " fmt + +#include <linux/cpufeature.h> +#include <linux/sched.h> +#include <linux/sched/task_stack.h> +#include <linux/slab.h> + +#include <asm/actlr.h> +#include <asm/el0_fgt.h> +#include <asm/processor.h> +#include <asm/sysreg.h> +#include <asm/uaccess.h> + +/* + * Core handler for prctl(PR_SET_SANDBOX_MODE) + */ +int arch_set_sandbox_mode(unsigned long enable, unsigned long vbar_base) +{ + struct el0_fgt_thead_ctx *fgt = current->thread.fgt_ctx; + struct pt_regs *regs = task_pt_regs(current); + u64 actlr; + + if (!system_uses_el0_fgt()) + return -EINVAL; + + if (enable != 0 && enable != 1) + return -EINVAL; + + if (enable) { + if ((vbar_base & 0x7ff) || !access_ok((void __user *)vbar_base, 1)) + return -EINVAL; + + if (!fgt) { + fgt = kzalloc(sizeof(struct el0_fgt_thead_ctx), GFP_KERNEL); + if (!fgt) + return -ENOMEM; + current->thread.fgt_ctx = fgt; + } + + fgt->enabled = 1; + fgt->vbar_el0 = vbar_base; + write_sysreg_s(vbar_base, SYS_VBAR_EL0_FGT); + pr_info("fgt enabled is %d, VBAR_EL0: 0x%lx\n", fgt->enabled, fgt->vbar_el0); + + if (regs) + regs->pstate |= PSR_TINDEX_BIT; + } else { + if (fgt) { + fgt->enabled = 0; + fgt->vbar_el0 = 0; + } + + write_sysreg_s(0, SYS_VBAR_EL0_FGT); + + if (regs) + regs->pstate &= ~PSR_TINDEX_BIT; + } + + actlr = read_sysreg(actlr_el1); + if (enable) + actlr |= ACTLR_EL1_FGT; + else + actlr &= ~ACTLR_EL1_FGT; + + write_sysreg(actlr, actlr_el1); + isb(); + + return 0; +} + +int arch_get_sandbox_mode(void) +{ + struct el0_fgt_thead_ctx *fgt = current->thread.fgt_ctx; + + if (!system_uses_el0_fgt() || !fgt) + return -EINVAL; + + return fgt->enabled; +} + +void fgt_thread_switch(struct task_struct *prev, struct task_struct *next) +{ + struct el0_fgt_thead_ctx *prev_fgt, *next_fgt; + bool prev_active = false; + bool next_active = false; + u64 actlr; + + if (!system_uses_el0_fgt()) + return; + + prev_fgt = (struct el0_fgt_thead_ctx *)prev->thread.fgt_ctx; + next_fgt = (struct el0_fgt_thead_ctx *)next->thread.fgt_ctx; + prev_active = prev_fgt && prev_fgt->enabled; + next_active = next_fgt && next_fgt->enabled; + + /* 切出和切入的两个线程都没有开启沙盒 */ + if (!prev_active && !next_active) + return; + + /* 线程级别状态切换：保存与恢复各自的 ELR_EL0_FGT 和 SPSR_EL0_FGT */ + if (prev_active) { + prev_fgt->sandbox_elr_el0 = read_sysreg_s(SYS_ELR_EL0_FGT); + prev_fgt->sandbox_spsr_el0 = read_sysreg_s(SYS_SPSR_EL0_FGT); + } + + actlr = read_sysreg(actlr_el1); + + /* 跨进程切入：恢复新沙盒进程的上下文 */ + if (next_active) { + write_sysreg_s(next_fgt->sandbox_elr_el0, SYS_ELR_EL0_FGT); + write_sysreg_s(next_fgt->sandbox_spsr_el0, SYS_SPSR_EL0_FGT); + write_sysreg_s(next_fgt->vbar_el0, SYS_VBAR_EL0_FGT); + actlr |= ACTLR_EL1_FGT; + } else { /* 跨进程切入普通进程：清空 VBAR_EL0 并关闭 FGT 硬件 */ + write_sysreg_s(0, SYS_VBAR_EL0_FGT); + actlr &= ~ACTLR_EL1_FGT; + } + + write_sysreg(actlr, actlr_el1); + isb(); +} diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index e9e5ce956f15..1ad66c2b301a 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -56,6 +56,7 @@ #include <asm/switch_to.h> #include <asm/system_misc.h> #include <asm/xcall.h> +#include <asm/el0_fgt.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> @@ -179,7 +180,7 @@ static void print_pstate(struct pt_regs *regs) const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >> PSR_BTYPE_SHIFT]; - printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n", + printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cTINDEX %cSSBS BTYPE=%s)\n", pstate, pstate & PSR_N_BIT ? 'N' : 'n', pstate & PSR_Z_BIT ? 'Z' : 'z', @@ -193,6 +194,7 @@ static void print_pstate(struct pt_regs *regs) pstate & PSR_UAO_BIT ? '+' : '-', pstate & PSR_TCO_BIT ? '+' : '-', pstate & PSR_DIT_BIT ? '+' : '-', + pstate & PSR_TINDEX_BIT ? '+' : '-', pstate & PSR_SSBS_BIT ? '+' : '-', btype_str); } @@ -294,6 +296,10 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) /* We rely on the above assignment to initialize dst's thread_flags: */ BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK)); +#ifdef CONFIG_ARM64_EL0_FGT + dst->thread.fgt_ctx = NULL; +#endif + /* * Detach src's sve_state (if any) from dst so that it does not * get erroneously used or freed prematurely. dst's copies @@ -351,6 +357,10 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) struct pt_regs *childregs = task_pt_regs(p); memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); +#ifdef CONFIG_ARM64_EL0_FGT + if (system_uses_el0_fgt()) + p->thread.fgt_ctx = NULL; +#endif /* * In case p was allocated the same task_struct pointer as some @@ -417,6 +427,11 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) return 0; } +void exit_thread(struct task_struct *tsk) +{ + el0_fgt_exit_thread(tsk); +} + void tls_preserve_current_state(void) { *task_user_tls(current) = read_sysreg(tpidr_el0); @@ -563,6 +578,8 @@ struct task_struct *__switch_to(struct task_struct *prev, */ mpam_thread_switch(next); + fgt_thread_switch(prev, next); + /* the actual thread switch */ last = cpu_switch_to(prev, next); @@ -659,6 +676,7 @@ void arch_setup_new_exec(void) current->mm->context.flags = mmflags; ptrauth_thread_init_user(); mte_thread_init_user(); + el0_fgt_init_user(); erratum_1418040_new_exec(); if (task_spec_ssb_noexec(current)) { diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 6ddee2b5cf50..65c95907bee3 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -2259,7 +2259,7 @@ void syscall_trace_exit(struct pt_regs *regs) */ #define SPSR_EL1_AARCH64_RES0_BITS \ (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \ - GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5)) + GENMASK_ULL(20, 15) | BIT_ULL(13) | GENMASK_ULL(5, 5)) #define SPSR_EL1_AARCH32_RES0_BITS \ (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20)) @@ -2308,7 +2308,7 @@ static int valid_native_regs(struct user_pt_regs *regs) } /* Force PSR to a valid 64-bit EL0t */ - regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; + regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT | PSR_TINDEX_BIT; return 0; } diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index e83c9d9d56f8..3ee2474a8c08 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -909,6 +909,9 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, sme_smstop(); } + if (system_uses_el0_fgt()) + regs->pstate &= ~PSR_TINDEX_BIT; + if (ka->sa.sa_flags & SA_RESTORER) sigtramp = ka->sa.sa_restorer; else diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index ce0c60d48dff..4b211cba61d4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -24,6 +24,7 @@ HAS_DIT HAS_E0PD HAS_ECV HAS_ECV_CNTPOFF +HAS_EL0_FGT HAS_EPAN HAS_EVT HAS_FGT diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 370ed14b1ae0..5975f34999ac 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -306,4 +306,7 @@ struct prctl_mm_map { # define PR_RISCV_V_VSTATE_CTRL_NEXT_MASK 0xc # define PR_RISCV_V_VSTATE_CTRL_MASK 0x1f +#define PR_SET_SANDBOX_MODE 71 /* Enable/disable EL0_FGT */ +#define PR_GET_SANDBOX_MODE 72 /* Get current EL0_FGT status */ + #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 04d934eabbc7..3d6ddbb25f81 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -117,6 +117,7 @@ #ifdef CONFIG_FAST_SYSCALL #include <asm/xcall.h> #endif +#include <asm/el0_fgt.h> #include <trace/events/sched.h> diff --git a/kernel/sys.c b/kernel/sys.c index 47cb10a16b00..69ad8e39e710 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -74,6 +74,7 @@ #include <linux/uaccess.h> #include <asm/io.h> #include <asm/unistd.h> +#include <asm/el0_fgt.h> #include "uid16.h" @@ -2767,6 +2768,14 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, case PR_RISCV_V_GET_CONTROL: error = RISCV_V_GET_CONTROL(); break; +#ifdef CONFIG_ARM64_EL0_FGT + case PR_SET_SANDBOX_MODE: + error = arch_set_sandbox_mode(arg2, arg3); + break; + case PR_GET_SANDBOX_MODE: + error = arch_get_sandbox_mode(); + break; +#endif default: error = -EINVAL; break; -- 2.34.1

hulk inclusion category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/21064 -------------------------------- mpam_msc_hw_probe() derives two limits from an MSC's RISes: partid_max, the request PARTID limit, and intpartid_max, the number of distinct configurations. mpam_ris_hw_probe() runs once per RIS and folds the RIS's own limit into both. Folding into intpartid_max is an assignment, not a clamp. A RIS with PARTID narrowing sets intpartid_max to the MSC's INTPARTID_MAX; a RIS without narrowing sets it to msc->partid_max, which at that point is the minimum over every RIS seen so far. The second form overwrites whatever an earlier RIS established. PARTID narrowing is global to an MSC, so all RISes of one MSC agree on it, but the limits they report are not necessarily the same: MPAMF_IDR.PARTID_MAX is permitted to vary between the instances of MPAMF_IDR, each reporting the space of its own resource instance. With RIS0 reporting a smaller PARTID space than RIS1, RIS1 writes back its larger value over the minimum RIS0 established, and the minimum is lost. mpam_msc_hw_probe() takes the system-wide minimum after the loop, but msc->intpartid_max already holds the overwritten value by then, and the system-wide minimum inherits it. resctrl_arch_get_num_closid() reports intpartid_max + 1, so the driver advertises more control groups than the RIS with the smaller space can hold. Configuring one of those groups programs that RIS with an intPARTID above its maximum, which the architecture describes as CONSTRAINED UNPREDICTABLE behaviour for the request that carries it. Start intpartid_max at the MSC-wide limit before the loop and take min() of it in both branches. The non-narrowing branch now reads the RIS's own PARTID limit from ris->idr instead of msc->partid_max: by the time mpam_ris_hw_probe() runs, msc->partid_max already includes this RIS and states the common limit of the RISes processed so far, which understates the RIS's own space and depends on probe order. Every RIS reports a PARTID_MAX value, and the field is per instance, so an MSC whose RISes report different values is affected regardless of whether it implements PARTID narrowing. QEMU gives every RIS of an MSC the same IDR, so the disagreement cannot be built in emulation. Fixes: 26b0eaa72060 ("arm_mpam: Add intPARTID and reqPARTID support for narrow PARTID feature") Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- drivers/platform/mpam/mpam_devices.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/platform/mpam/mpam_devices.c b/drivers/platform/mpam/mpam_devices.c index e78352e2dcb2..e934fdb91414 100644 --- a/drivers/platform/mpam/mpam_devices.c +++ b/drivers/platform/mpam/mpam_devices.c @@ -785,9 +785,17 @@ static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) u16 partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr); mpam_set_feature(mpam_feat_partid_nrw, props); - msc->intpartid_max = min(msc->partid_max, partid_max); + msc->intpartid_max = min(msc->intpartid_max, partid_max); } else { - msc->intpartid_max = msc->partid_max; + /* + * Without narrowing every request PARTID needs its own + * configuration, so the intPARTID space is the RIS's own + * PARTID space. Take the minimum over all RIS instead of + * overwriting: a previous RIS may have had a smaller limit. + */ + msc->intpartid_max = min(msc->intpartid_max, + FIELD_GET(MPAMF_IDR_PARTID_MAX, + ris->idr)); } } @@ -824,6 +832,7 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc) /* Use these values so partid/pmg always starts with a valid value */ msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); + msc->intpartid_max = msc->partid_max; msc->pmg_max = mpam_pmg_max_workaround(idr); for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) { -- 2.34.1

hulk inclusion category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/21064 -------------------------------- Monitoring groups beyond the intPARTID count are given request PARTIDs of their own, and reqpartid_map[] records which internal PARTID each request PARTID uses. The map is indexed from req2intpartid(), which runs from mpam_thread_switch() on every context switch of a labelled task, so it runs once per switch on every CPU and takes no resctrl, MSC or MPAM SRCU lock. mpam_resctrl_exit() frees the map through reqpartid_exit(), and the caller disables the static key that guards mpam_thread_switch() only afterwards. In the interval a task holding a request PARTID above the intPARTID count switches on another CPU, indexes the freed array, and reads whatever memory occupies it, which is a use-after-free. Publish the array through RCU: build it, install it with rcu_assign_pointer(), and on teardown clear the pointer, wait for a grace period with synchronize_rcu() and then free. req2intpartid() reads the pointer under rcu_read_lock() and returns the request PARTID unchanged when the pointer is already NULL, which keeps a task that is still scheduled in from following a stale mapping. Disabling the static key alone does not retire readers that are already inside mpam_thread_switch(). The writers, resctrl_arch_rmid_expand() and resctrl_arch_rmid_reclaim(), run under rdtgroup_mutex and update the array in place, so only the pointer needs RCU treatment; they reach it through reqpartid_map_writable(). mpam_sync_config() returns with the SRCU read lock held when mpam_apply_config() fails. mpam_apply_config() discards that failure today, so the path is dead; it becomes reachable as soon as the failure is reported. Release the lock before returning. Only systems that build a request PARTID map, meaning systems with the Narrow-PARTID feature and more monitoring groups than intPARTIDs, reach the freed array, and only through the error teardown in mpam_disable_thread(). Fixes: d181cc9412cc ("arm_mpam: Implement dynamic reqPARTID allocation for monitoring groups") Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- drivers/platform/mpam/mpam_resctrl.c | 74 ++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/drivers/platform/mpam/mpam_resctrl.c b/drivers/platform/mpam/mpam_resctrl.c index 1c3df3ad8dba..ad2848e322c6 100644 --- a/drivers/platform/mpam/mpam_resctrl.c +++ b/drivers/platform/mpam/mpam_resctrl.c @@ -310,10 +310,24 @@ static u32 req_pmg2rmid(u32 reqpartid, u8 pmg) return (reqpartid << pmg_shift) | (pmg & pmg_mask); } -static u32 *reqpartid_map; +/* + * Maps a request PARTID to its internal PARTID. + * + * Most users of req2intpartid() run under rdtgroup_mutex, but two do not: + * mpam_resctrl_exit() drops the map from the error-disable path without + * taking that lock, and resctrl_arch_rmid_idx_decode() is called from + * __rmid_entry() during RMID initialisation, also without it. Those can + * therefore race with the teardown, so the array is only replaced and + * freed after an RCU grace period. The pointer is NULL before the map is + * built and after mpam_resctrl_exit(). + */ +static u32 __rcu *reqpartid_map; u32 req2intpartid(u32 reqpartid) { + u32 intpartid; + u32 __rcu *map; + /* * Directly return intPartid in case that mpam_reset_ris() access * NULL pointer. @@ -321,7 +335,17 @@ u32 req2intpartid(u32 reqpartid) if (reqpartid < resctrl_arch_get_num_closid(NULL)) return reqpartid; - return reqpartid_map[reqpartid]; + rcu_read_lock(); + map = rcu_dereference(reqpartid_map); + /* + * The map may already have been dropped by the error-disable path + * while this reader is still in flight. Fall back to the identity + * mapping. + */ + intpartid = map ? map[reqpartid] : reqpartid; + rcu_read_unlock(); + + return intpartid; } static u32 partid2closid(u32 partid) @@ -1393,28 +1417,46 @@ static int mpam_resctrl_resource_init(struct mpam_resctrl_res *res) static int reqpartid_init(void) { + u32 *map; int req_num, idx; req_num = get_num_reqpartid(); - reqpartid_map = kcalloc(req_num, sizeof(u32), GFP_KERNEL); - if (!reqpartid_map) + map = kcalloc(req_num, sizeof(u32), GFP_KERNEL); + if (!map) return -ENOMEM; for (idx = 0; idx < req_num; idx++) - reqpartid_map[idx] = idx; + map[idx] = idx; + + RCU_INIT_POINTER(reqpartid_map, map); return 0; } void reqpartid_exit(void) { - kfree(reqpartid_map); + u32 *old = rcu_dereference_protected(reqpartid_map, 1); + + RCU_INIT_POINTER(reqpartid_map, NULL); + synchronize_rcu(); + kfree(old); +} + +/* + * The mapping itself is only ever updated under rdtgroup_mutex, and the + * pointer is only replaced during setup and teardown. Callers of this + * helper are those write paths; the scheduler goes through + * req2intpartid() instead. + */ +static u32 *reqpartid_map_writable(void) +{ + return rcu_dereference_protected(reqpartid_map, 1); } void update_rmid_entries_for_reqpartid(u32 reqpartid) { int pmg; - u32 intpartid = reqpartid_map[reqpartid]; + u32 intpartid = reqpartid_map_writable()[reqpartid]; u32 closid = partid2closid(intpartid); for (pmg = 0; pmg <= mpam_pmg_max; pmg++) @@ -1431,8 +1473,10 @@ static int mpam_sync_config(u32 reqpartid) list_for_each_entry_rcu(class, &mpam_classes, classes_list) { list_for_each_entry(comp, &class->components, class_list) { err = mpam_apply_config(comp, reqpartid, NULL, true); - if (err) + if (err) { + srcu_read_unlock(&mpam_srcu, idx); return err; + } } } srcu_read_unlock(&mpam_srcu, idx); @@ -1442,20 +1486,21 @@ static int mpam_sync_config(u32 reqpartid) int resctrl_arch_rmid_expand(u32 closid) { + u32 *map = reqpartid_map_writable(); int i; for (i = resctrl_arch_get_num_closid(NULL); i < get_num_reqpartid(); i++) { - if (reqpartid_map[i] >= resctrl_arch_get_num_closid(NULL)) { + if (map[i] >= resctrl_arch_get_num_closid(NULL)) { if (cdp_enabled) { - reqpartid_map[i] = resctrl_get_config_index(closid, CDP_DATA); + map[i] = resctrl_get_config_index(closid, CDP_DATA); mpam_sync_config(i); - reqpartid_map[i + 1] = resctrl_get_config_index(closid, CDP_CODE); + map[i + 1] = resctrl_get_config_index(closid, CDP_CODE); mpam_sync_config(i + 1); } else { - reqpartid_map[i] = resctrl_get_config_index(closid, CDP_NONE); + map[i] = resctrl_get_config_index(closid, CDP_NONE); mpam_sync_config(i); } @@ -1472,6 +1517,7 @@ void resctrl_arch_rmid_reclaim(u32 closid, u32 rmid) int pmg; u32 intpartid; int reqpartid = rmid2reqpartid(rmid); + u32 *map = reqpartid_map_writable(); if (reqpartid < resctrl_arch_get_num_closid(NULL)) return; @@ -1489,9 +1535,9 @@ void resctrl_arch_rmid_reclaim(u32 closid, u32 rmid) } if (pmg > mpam_pmg_max) { - reqpartid_map[reqpartid] = reqpartid; + map[reqpartid] = reqpartid; if (cdp_enabled) - reqpartid_map[reqpartid + 1] = reqpartid + 1; + map[reqpartid + 1] = reqpartid + 1; update_rmid_entries_for_reqpartid(reqpartid); } -- 2.34.1

hulk inclusion category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/21064 -------------------------------- mpam_apply_config() sends the configuration to each RIS of a component through mpam_touch_msc(), which runs the write on a CPU that can reach the MSC. mpam_touch_msc() returns -ENXIO when none of the MSC's CPUs is online, and mpam_apply_config() discards the value and returns 0. The caller then treats a configuration that never reached the hardware as applied. resctrl_arch_update_domains() reports success to the schemata write, so userspace reads the new value back from the cached copy, while the MSC keeps the previous one. resctrl_arch_rmid_expand() records the binding of a request PARTID to an internal PARTID in reqpartid_map[] and returns, leaving the software map claiming a binding the hardware does not have. mpam_reprogram_msc() reprograms intPARTIDs when the CPUs come back, and those intPARTIDs use the mapping software recorded, so the two stay out of step until the map is rebuilt. ris->in_reset_state is cleared whether or not the write landed, so a RIS that is still in its reset state stops being reprogrammed on the next CPU online. Record the first failure from mpam_touch_msc() and return it. Clear ris->in_reset_state only for RISes whose write returned 0, and restore comp->cfg[partid] when any RIS failed, so the cached copy stops claiming a configuration the hardware rejected. resctrl_arch_rmid_expand() binds one request PARTID at a time through mpam_expand_one() and reverts the entry when the write fails; when the data half of a CDP pair fails after the code half was bound, revert the code half's map entry without re-running the sync, which would index comp->cfg[] by a request PARTID. Only writes that run while no CPU of an MSC is online fail in mpam_touch_msc(). mpam_resctrl_prepare_offline() refuses to offline a CPU while the file system is mounted and the L2 resource is visible, so the state is reached with the L2 resource hidden, or with an L3 or memory MSC whose CPUs are all offline. Fixes: be74872ad2e3 ("arm_mpam: Allow configuration to be applied and restored during cpu online") Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- drivers/platform/mpam/mpam_devices.c | 35 ++++++++++++++-- drivers/platform/mpam/mpam_resctrl.c | 61 +++++++++++++++++++++++----- 2 files changed, 81 insertions(+), 15 deletions(-) diff --git a/drivers/platform/mpam/mpam_devices.c b/drivers/platform/mpam/mpam_devices.c index afccae608d03..c7ec1315d8ed 100644 --- a/drivers/platform/mpam/mpam_devices.c +++ b/drivers/platform/mpam/mpam_devices.c @@ -2803,7 +2803,8 @@ int mpam_apply_config(struct mpam_component *comp, u16 partid, { struct mpam_write_config_arg arg; struct mpam_msc_ris *ris; - int idx; + struct mpam_config old; + int err, idx; lockdep_assert_cpus_held(); @@ -2814,6 +2815,7 @@ int mpam_apply_config(struct mpam_component *comp, u16 partid, if (!memcmp(&comp->cfg[partid], cfg, sizeof(*cfg))) return 0; + old = comp->cfg[partid]; comp->cfg[partid] = *cfg; } @@ -2821,17 +2823,42 @@ int mpam_apply_config(struct mpam_component *comp, u16 partid, arg.partid = partid; arg.sync = sync; + err = 0; idx = srcu_read_lock(&mpam_srcu); list_for_each_entry_rcu(ris, &comp->ris, comp_list) { + int ret; + arg.ris = ris; mutex_lock(&ris->msc->lock); - mpam_touch_msc(ris->msc, __write_config, &arg); - ris->in_reset_state = false; + ret = mpam_touch_msc(ris->msc, __write_config, &arg); mutex_unlock(&ris->msc->lock); + + /* + * The MSC may not be reachable from any online CPU. Report + * that instead of pretending the configuration reached it, + * otherwise the caller and the software mapping disagree + * with the hardware. + */ + if (ret && !err) + err = ret; + + /* + * Only mark the RIS as configured once the write landed. + */ + if (!ret) + ris->in_reset_state = false; } srcu_read_unlock(&mpam_srcu, idx); - return 0; + /* + * The RISes are configured as a group. If any of them was not + * reachable, undo the software copy so that it does not claim a + * configuration the hardware does not have. + */ + if (err && !sync) + comp->cfg[partid] = old; + + return err; } static const struct of_device_id mpam_of_match[] = { diff --git a/drivers/platform/mpam/mpam_resctrl.c b/drivers/platform/mpam/mpam_resctrl.c index ad2848e322c6..0019305c4087 100644 --- a/drivers/platform/mpam/mpam_resctrl.c +++ b/drivers/platform/mpam/mpam_resctrl.c @@ -1484,24 +1484,63 @@ static int mpam_sync_config(u32 reqpartid) return 0; } -int resctrl_arch_rmid_expand(u32 closid) +/* + * Bind @reqpartid to @intpartid, then push the control group's existing + * configuration to it. The mapping has to be in place first: the sync + * path looks the intPARTID up through req2intpartid(). If the hardware + * write fails the mapping is rolled back, so the software mapping never + * claims a binding the hardware does not have. + */ +static int mpam_expand_one(u32 reqpartid, u32 intpartid) { u32 *map = reqpartid_map_writable(); - int i; + int err; + + map[reqpartid] = intpartid; + + err = mpam_sync_config(reqpartid); + if (err) + map[reqpartid] = reqpartid; + + return err; +} + +int resctrl_arch_rmid_expand(u32 closid) +{ + int i, err; for (i = resctrl_arch_get_num_closid(NULL); i < get_num_reqpartid(); i++) { - if (map[i] >= resctrl_arch_get_num_closid(NULL)) { + if (reqpartid_map_writable()[i] >= + resctrl_arch_get_num_closid(NULL)) { if (cdp_enabled) { - map[i] = resctrl_get_config_index(closid, CDP_DATA); - mpam_sync_config(i); - - map[i + 1] = resctrl_get_config_index(closid, CDP_CODE); - mpam_sync_config(i + 1); - + err = mpam_expand_one(i, + resctrl_get_config_index(closid, + CDP_DATA)); + if (err) + return err; + + err = mpam_expand_one(i + 1, + resctrl_get_config_index(closid, + CDP_CODE)); + if (err) { + /* + * Put the first half back. Its + * hardware write succeeded, but the + * pair is only usable together and + * re-running the sync here would + * index comp->cfg[i], which is out + * of range for a reqPARTID. + */ + reqpartid_map_writable()[i] = i; + return err; + } } else { - map[i] = resctrl_get_config_index(closid, CDP_NONE); - mpam_sync_config(i); + err = mpam_expand_one(i, + resctrl_get_config_index(closid, + CDP_NONE)); + if (err) + return err; } update_rmid_entries_for_reqpartid(i); -- 2.34.1

hulk inclusion category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/21064 -------------------------------- resctrl stores the details of a monitoring event file in the kernfs node's private pointer, packed into union mon_data_bits: 10 bits of resource id, 8 bits of event id and 14 bits of domain id. mpam_resctrl_alloc_domain() takes a domain's id from the component id, which for a cache is the cache-id from the ACPI PPTT. That field is 32 bits wide and sparse: the L3 cache of the 8-CPU QEMU topology has id 0x2030000, 33751040, which does not fit in 14 bits. The id is truncated to 0 when the file is created. rdtgroup_mondata_show() later passes the truncated id to resctrl_arch_find_domain(), which returns NULL, and reading the file fails: $ cat /sys/fs/resctrl/mon_data/mon_L3_33751040/llc_occupancy cat: read error: No such file or directory Every monitoring event file on a topology whose cache ids exceed 14 bits is unreadable, which covers every MPAM system that reports PPTT cache ids. The 14-bit field dates from the original x86 implementation. It became reachable for MPAM only once the driver started giving the L3 cache a mon_capable resctrl resource: mkdir_mondata_all() then creates mon_<res>_<id> for each domain and packs d->id into the file's private data, and the domain id at that point is already the component id. Before that, no MPAM resource was mon_capable and no such file existed. Replace the packed union with an allocated struct mon_data whose domain id is an int, the layout mainline uses after 2a6566038544 ("x86/resctrl: Expand the width of domid by replacing mon_data_bits"). mon_get_kn_priv() returns the entry for a resource, event and domain, allocating it on first use, and mon_put_kn_priv() frees the whole list in rdt_kill_sb() when the file system is unmounted, before the kernfs root is destroyed. The same mainline commit was backported for x86 as 2e9648d1d932, which touches arch/x86/kernel/cpu/resctrl only. fs/resctrl, the copy arm64 builds, kept the union and is fixed here. Fixes: 1ca491c7bbcd ("arm_mpam: resctrl: Pick the caches we will use as resctrl resources") Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- fs/resctrl/ctrlmondata.c | 10 +++---- fs/resctrl/internal.h | 23 +++++++-------- fs/resctrl/rdtgroup.c | 60 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index cf4895b41fc4..fc2dea4ebafb 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -503,7 +503,7 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) u32 resid, evtid, domid; struct rdtgroup *rdtgrp; struct rdt_resource *r; - union mon_data_bits md; + struct mon_data *md; struct rdt_domain *d; struct rmid_read rr; int ret = 0; @@ -514,10 +514,10 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) goto out; } - md.priv = of->kn->priv; - resid = md.u.rid; - domid = md.u.domid; - evtid = md.u.evtid; + md = of->kn->priv; + resid = md->rid; + domid = md->domid; + evtid = md->evtid; r = resctrl_arch_get_resource(resid); d = resctrl_arch_find_domain(r, domid); diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index a5f35e72ebc6..2d0af2469d14 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -68,21 +68,22 @@ static inline struct rdt_fs_context *rdt_fc2context(struct fs_context *fc) } /** - * union mon_data_bits - Monitoring details for each event file - * @priv: Used to store monitoring event data in @u - * as kernfs private data + * struct mon_data - Monitoring details for each event file + * @list: Member of the global @mon_data_kn_priv_list list * @rid: Resource id associated with the event file * @evtid: Event id associated with the event file * @domid: The domain to which the event file belongs - * @u: Name of the bit fields struct + * + * Stored as the kernfs kn->priv pointer. Allocated rather than packed into + * the pointer, so that @domid keeps the full width of the cache-id from the + * ACPI PPTT (MPAM platforms use sparse ids wider than the 14 bits a bitfield + * in the pointer has room for). */ -union mon_data_bits { - void *priv; - struct { - unsigned int rid : 10; - enum resctrl_event_id evtid : 8; - unsigned int domid : 14; - } u; +struct mon_data { + struct list_head list; + enum resctrl_res_level rid; + enum resctrl_event_id evtid; + int domid; }; struct rmid_read { diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index c8c78db9da6a..3c55a7b2ead1 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -43,6 +43,9 @@ LIST_HEAD(rdt_all_groups); /* list of entries for the schemata file */ LIST_HEAD(resctrl_schema_all); +/* List of allocated struct mon_data, one per monitoring event file */ +static LIST_HEAD(mon_data_kn_priv_list); + /* The filesystem can only be mounted once. */ bool resctrl_mounted; @@ -66,6 +69,7 @@ static char last_cmd_status_buf[512]; static int rdtgroup_setup_root(struct rdt_fs_context *ctx); static void rdtgroup_destroy_root(void); +static void mon_put_kn_priv(void); struct dentry *debugfs_resctrl; @@ -3043,6 +3047,7 @@ static void rdt_kill_sb(struct super_block *sb) rdtgroup_default.mode = RDT_MODE_SHAREABLE; closid_exit(); schemata_list_destroy(); + mon_put_kn_priv(); rdtgroup_destroy_root(); if (resctrl_arch_alloc_capable()) resctrl_arch_disable_alloc(); @@ -3101,12 +3106,54 @@ static void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, } } +static struct mon_data *mon_get_kn_priv(struct rdt_resource *r, + enum resctrl_event_id evtid, int domid) +{ + struct mon_data *priv; + + lockdep_assert_held(&rdtgroup_mutex); + + list_for_each_entry(priv, &mon_data_kn_priv_list, list) { + if (priv->rid == r->rid && priv->domid == domid && + priv->evtid == evtid) + return priv; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; + + priv->rid = r->rid; + priv->domid = domid; + priv->evtid = evtid; + list_add_tail(&priv->list, &mon_data_kn_priv_list); + + return priv; +} + +/* + * Free all allocated mon_data structures. Called with rdtgroup_mutex held + * when the filesystem is unmounted: the kernfs files that point at them + * are gone by then. + */ +static void mon_put_kn_priv(void) +{ + struct mon_data *priv, *tmp; + + lockdep_assert_held(&rdtgroup_mutex); + + list_for_each_entry_safe(priv, tmp, &mon_data_kn_priv_list, list) { + list_del(&priv->list); + kfree(priv); + } +} + static int mkdir_mondata_subdir(struct kernfs_node *parent_kn, struct rdt_domain *d, struct rdt_resource *r, struct rdtgroup *prgrp) { - union mon_data_bits priv; struct kernfs_node *kn; + struct mon_data *priv; struct mon_evt *mevt; struct rmid_read rr; char name[32]; @@ -3127,11 +3174,14 @@ static int mkdir_mondata_subdir(struct kernfs_node *parent_kn, goto out_destroy; } - priv.u.rid = r->rid; - priv.u.domid = d->id; list_for_each_entry(mevt, &r->evt_list, list) { - priv.u.evtid = mevt->evtid; - ret = mon_addfile(kn, mevt->name, priv.priv); + priv = mon_get_kn_priv(r, mevt->evtid, d->id); + if (WARN_ON_ONCE(!priv)) { + ret = -ENOMEM; + goto out_destroy; + } + + ret = mon_addfile(kn, mevt->name, priv); if (ret) goto out_destroy; -- 2.34.1