From: Liao Chang <liaochang1@huawei.com> hulk inclusion category: feature bugzilla: https://gitcode.com/openeuler/kernel/issues/10066 ---------------------------------------- Add the initial infrastructure to support the Armv9.3 FEAT_TLBID extension, which allows OSPM to target TLB invalidations to a specific domain. A TLBI domain can be contained in multiple parent domains. This patch provides: - Kconfig option CONFIG_ARM64_TLBID to enable the feature - TLBIDIDR_EL1 register definition and CPU feature detection - ACPI TLBI table parser to discover TLBI domains for processors and SMMU [1]. - Data structures to manage TLBI domain to CPU mapping. - Help functions to choose the best TLBI domain for a specific set of CPUs. The TLBI table lists all the PE and SMMU components and the TLBI domains that each component belongs to. This information is used to optimize TLB invalidation by targeting specific domains instead of broadcasting to all CPUs. [1] https://github.com/user-attachments/files/25049127/ACPI_TLBID_v3.docx Signed-off-by: Liao Chang <liaochang1@huawei.com> Signed-off-by: Xinyu Zheng <zhengxinyu6@huawei.com> CC: Xia Qinxin <xiaqinxin@huawei.com> CC: Zheng Tian <zhengtian10@huawei.com> CC: Yang Jinqian <yangjinqian1@huawei.com> CC: Zeng Heng <zengheng4@huawei.com> CC: Wang Kefeng <wangkefeng.wang@huawei.com> CC: Yu Jiacheng <yujiacheng3@huawei.com> --- arch/arm64/include/asm/cpu.h | 1 + arch/arm64/include/asm/cpufeature.h | 7 + arch/arm64/include/asm/tlbidomain.h | 34 +++ arch/arm64/kernel/cpufeature.c | 2 + arch/arm64/kernel/cpuinfo.c | 3 + arch/arm64/mm/Kconfig | 19 ++ arch/arm64/mm/Makefile | 1 + arch/arm64/mm/test_tlbidomain.c | 176 +++++++++++++ arch/arm64/mm/tlbidomain.c | 371 ++++++++++++++++++++++++++++ arch/arm64/tools/sysreg | 11 + drivers/acpi/arm64/Makefile | 1 + drivers/acpi/arm64/init.c | 2 + drivers/acpi/arm64/init.h | 1 + drivers/acpi/arm64/tlbid.c | 105 ++++++++ drivers/acpi/tables.c | 9 + include/acpi/actbl2.h | 29 +++ include/linux/acpi.h | 1 + 17 files changed, 773 insertions(+) create mode 100644 arch/arm64/include/asm/tlbidomain.h create mode 100644 arch/arm64/mm/test_tlbidomain.c create mode 100644 arch/arm64/mm/tlbidomain.c create mode 100644 drivers/acpi/arm64/tlbid.c diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index 56eb090ae48d..ede40c40d410 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -47,6 +47,7 @@ struct cpuinfo_arm64 { u64 reg_gmid; u64 reg_smidr; u64 reg_mpamidr; + u64 reg_tlbididr; u64 reg_id_aa64dfr0; u64 reg_id_aa64dfr1; diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 9e617d40c50b..7a0c8981120b 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -666,6 +666,13 @@ static inline bool id_aa64pfr1_mte(u64 pfr1) return val >= ID_AA64PFR1_EL1_MTE_MTE2; } +static inline bool id_aa64mmfr4_tlbid(u64 mmfr4) +{ + u32 val = cpuid_feature_extract_unsigned_field(mmfr4, ID_AA64MMFR4_EL1_TLBID_SHIFT); + + return val > 0; +} + void __init setup_cpu_features(void); void check_local_cpu_capabilities(void); diff --git a/arch/arm64/include/asm/tlbidomain.h b/arch/arm64/include/asm/tlbidomain.h new file mode 100644 index 000000000000..1dac3771ff57 --- /dev/null +++ b/arch/arm64/include/asm/tlbidomain.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 Huawei Ltd. + */ +#ifndef __ASM_TLBIDOMAIN_H +#define __ASM_TLBIDOMAIN_H + +#include <linux/types.h> + +struct tlbi_target { + u32 identifier; + s32 *domains; + s32 nr_domain; + struct list_head list; +}; + +#define TLBI_RT_DOMAIN_WIDTH (16) +#define TLBI_INV_DOMAIN (-1) + +#ifdef CONFIG_ARM64_TLBID +void add_target(int type, struct tlbi_target *target); +int pick_best_domain(const cpumask_t *active_cpus); +int __init parse_tlbid_topology(void); +int __init tlbid_data_init(void); +void __init tlbid_data_reset(void); +void __init tlbid_early_init(void); + +#else +static inline int pick_best_domain(const cpumask_t *active_cpus) { return 0; } +static inline void __init tlbid_early_init(void) { } + +#endif /* ! CONFIG_ARM64_TLBID */ +#endif + diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 82468372fdd4..1491c9273007 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -97,6 +97,7 @@ #include <asm/vectors.h> #include <asm/virt.h> #include <asm/vip_smt.h> +#include <asm/tlbidomain.h> /* Kernel representation of AT_HWCAP and AT_HWCAP2 */ static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly; @@ -4107,6 +4108,7 @@ void __init setup_cpu_features(void) sve_setup(); sme_setup(); + tlbid_early_init(); minsigstksz_setup(); mpam_extra_caps(); diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 76d2eaeb33e4..39ae0bf1360a 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -347,6 +347,9 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) __cpuinfo_store_cpu_32bit(&info->aarch32); + if (id_aa64mmfr4_tlbid(info->reg_id_aa64mmfr4)) + info->reg_tlbididr = read_cpuid(TLBIDIDR_EL1); + if (IS_ENABLED(CONFIG_ARM64_MPAM) && mpam_detect_is_enabled() && (id_aa64pfr0_mpam(info->reg_id_aa64pfr0) || diff --git a/arch/arm64/mm/Kconfig b/arch/arm64/mm/Kconfig index 315b3322feb5..eeb79a30b2e9 100644 --- a/arch/arm64/mm/Kconfig +++ b/arch/arm64/mm/Kconfig @@ -14,3 +14,22 @@ config PFN_RANGE_ALLOC the linear mapping granule of this range is never larger than PMD. If unsure, say N. + +config TLBID_KUNIT_TEST + bool "KUnit tests for TLBI" if !KUNIT_ALL_TESTS + depends on KUNIT=y + depends on ARM64_TLBID && ACPI + default KUNIT_ALL_TESTS + help + This option enables KUnit unit tests for the arm64 Translation + Lookaside Buffer Domain topology parsing and domain picking + logic. + + It verifies the correct parsing of TLBID ACPI tables and ensures + proper selection of TLB invalidation domains during kernel boot. + + Say Y here if you want to run these unit tests as part of the + kernel test suite. Say N if you are building a production kernel + or do not need boot-time KUnit testing. + + If unsure, say N. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c02aeb729717..4d92d42826bf 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd.o obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o +obj-$(CONFIG_ARM64_TLBID) += tlbidomain.o KASAN_SANITIZE_physaddr.o += n obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm64/mm/test_tlbidomain.c b/arch/arm64/mm/test_tlbidomain.c new file mode 100644 index 000000000000..652ecff2079c --- /dev/null +++ b/arch/arm64/mm/test_tlbidomain.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* This file is intended to be included into tlbidomain.c + * Copyright (C) 2026 Huawei Ltd. + */ + +#include <kunit/test.h> + +static struct tlbid_data fake_tlbid_data __read_mostly = { + .nr_domain = 1, + .nis = TLBI_RT_DOMAIN_WIDTH, + .nvis = TLBI_RT_DOMAIN_WIDTH, + .nos = TLBI_RT_DOMAIN_WIDTH, + .nvos = TLBI_RT_DOMAIN_WIDTH, +}; + +static struct acpi_madt_generic_interrupt fake_madt_gicc[NR_CPUS]; + +static void __init reset_real_tlbid(void) +{ + tlbid_data_reset(); + memcpy(&tlbid_data, &fake_tlbid_data, sizeof(tlbid_data)); +} + +static int __init setup_fake_tlbid(struct kunit *test) +{ + int cpu, max_domains = 1 << TLBI_RT_DOMAIN_WIDTH; + struct acpi_madt_generic_interrupt *gicc; + struct tlbi_target *target; + + if (num_possible_cpus() == 1) + return 1; + + memcpy(&fake_tlbid_data, &tlbid_data, sizeof(tlbid_data)); + tlbid_data.nis = 8; + tlbid_data.nvis = 0; + tlbid_data.nos = 8; + tlbid_data.nvos = 0; + for_each_possible_cpu(cpu) + memcpy(&fake_madt_gicc[cpu], acpi_cpu_get_madt_gicc(cpu), + sizeof(struct acpi_madt_generic_interrupt)); + + tlbid_data.found_domains = kcalloc(BITS_TO_LONGS(max_domains), + sizeof(long), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tlbid_data.found_domains); + + INIT_LIST_HEAD(&tlbid_data.target_list[ACPI_TLBI_TYPE_PROCESSOR]); + INIT_LIST_HEAD(&tlbid_data.target_list[ACPI_TLBI_TYPE_SMMU]); + + /* + * Fake TLBI domain topology: + * domain0: [0, num_possible_cpus()) + * domain1: [0, (num_possible_cpus() / 2) + * domain2: [num_possible_cpus() / 2, num_possible_cpus()) + */ + for_each_possible_cpu(cpu) { + target = kzalloc(sizeof(*target), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, target); + + /* The ACPI Processor UID of the corresponding GICC MADT entry */ + gicc = acpi_cpu_get_madt_gicc(cpu); + gicc->uid = cpu; + target->nr_domain = 2; + target->identifier = cpu; + target->domains = kcalloc(target->nr_domain, + sizeof(*target->domains), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, target->domains); + target->domains[0] = 0; + target->domains[1] = (cpu < (num_possible_cpus() / 2)) ? 1 : 2; + add_target(ACPI_TLBI_TYPE_PROCESSOR, target); + } + + parse_tlbid_topology(); + + for_each_possible_cpu(cpu) + memcpy(acpi_cpu_get_madt_gicc(cpu), &fake_madt_gicc[cpu], + sizeof(struct acpi_madt_generic_interrupt)); + + return 0; +} + +static void __init test_parse_tlbid_topology(struct kunit *test) +{ + struct tlbi_domain *actbl[3]; + cpumask_t tmp; + int cpu; + + if (setup_fake_tlbid(test)) + return; + + actbl[0] = &tlbid_data.cpu_domain[num_possible_cpus() - 1][0]; + actbl[1] = &tlbid_data.cpu_domain[(num_possible_cpus() / 2) - 1][0]; + actbl[2] = &tlbid_data.cpu_domain[(num_possible_cpus() / 2) - 1][1]; + + KUNIT_EXPECT_EQ(test, actbl[0]->id, 0); + KUNIT_EXPECT_TRUE(test, cpumask_equal(cpu_possible_mask, + &actbl[0]->cpumask)); + + KUNIT_EXPECT_EQ(test, actbl[1]->id, 1); + cpumask_clear(&tmp); + for_each_possible_cpu(cpu) { + if (cpu >= (num_possible_cpus() / 2)) + continue; + cpumask_set_cpu(cpu, &tmp); + } + KUNIT_EXPECT_TRUE(test, cpumask_equal(&tmp, &actbl[1]->cpumask)); + + KUNIT_EXPECT_EQ(test, actbl[2]->id, 2); + cpumask_clear(&tmp); + for_each_possible_cpu(cpu) { + if (cpu < (num_possible_cpus() / 2)) + continue; + cpumask_set_cpu(cpu, &tmp); + } + KUNIT_EXPECT_TRUE(test, cpumask_equal(&tmp, &actbl[2]->cpumask)); + + for_each_possible_cpu(cpu) { + struct tlbi_domain *domain; + int i; + + for (i = 0; i < tlbid_data.nr_domain; i++) { + domain = &tlbid_data.cpu_domain[cpu][i]; + if (domain != actbl[0] && + domain != actbl[1] && + domain != actbl[2]) { + KUNIT_EXPECT_EQ(test, domain->id, TLBI_INV_DOMAIN); + } + } + } + + reset_real_tlbid(); +} + +static void __init test_pick_best_domain(struct kunit *test) +{ + int domain, cpu; + cpumask_t tmp; + + if (setup_fake_tlbid(test)) + return; + + domain = pick_best_domain(cpu_possible_mask); + KUNIT_EXPECT_EQ(test, domain, 0); + + cpumask_clear(&tmp); + for_each_possible_cpu(cpu) { + if (cpu >= (num_possible_cpus() / 2)) + continue; + cpumask_set_cpu(cpu, &tmp); + } + domain = pick_best_domain(&tmp); + KUNIT_EXPECT_EQ(test, domain, 1); + + cpumask_clear(&tmp); + for_each_possible_cpu(cpu) { + if (cpu < (num_possible_cpus() / 2)) + continue; + cpumask_set_cpu(cpu, &tmp); + } + domain = pick_best_domain(&tmp); + KUNIT_EXPECT_EQ(test, domain, 2); + + reset_real_tlbid(); +} + +static struct kunit_case __refdata tlbidomain_test_cases[] = { + KUNIT_CASE(test_parse_tlbid_topology), + KUNIT_CASE(test_pick_best_domain), + {} +}; + +static struct kunit_suite tlbidomain_test_suite = { + .name = "tlbidomain_test_suite", + .test_cases = tlbidomain_test_cases, +}; + +kunit_test_init_section_suites(&tlbidomain_test_suite); diff --git a/arch/arm64/mm/tlbidomain.c b/arch/arm64/mm/tlbidomain.c new file mode 100644 index 000000000000..a8abbcb94952 --- /dev/null +++ b/arch/arm64/mm/tlbidomain.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file implements the kernel functions rely on FEAT_TLBID + * + * Copyright (C) 2026 Huawei Ltd. + */ + +#define pr_fmt(fmt) "tlbidomain: " fmt + +#include <linux/acpi.h> +#include <linux/cpumask.h> + +#include <asm/cpu.h> +#include <asm/sysreg.h> +#include <asm/tlbidomain.h> +#include <asm/cpufeature.h> + +struct tlbi_domain { + cpumask_t cpumask; + int id; +}; + +struct tlbid_data { + struct list_head target_list[ACPI_TLBI_TYPE_RESERVED]; + unsigned long *found_domains; + u32 nr_domain; + struct tlbi_domain *cpu_domain[NR_CPUS]; + u8 nis; /* TLBIDIDR_EL1.NIS */ + u8 nvis; /* TLBIDIDR_EL1.NVIS */ + u8 nos; /* TLBIDIDR_EL1.NOS */ + u8 nvos; /* TLBIDIDR_EL1.NVOS */ +}; + +/* + * Global TLBI domain data. + * Actual supported domain bits will be reduced based on hardware capability. + */ +static struct tlbid_data tlbid_data __read_mostly = { + .nr_domain = 1, + .nis = TLBI_RT_DOMAIN_WIDTH, + .nvis = TLBI_RT_DOMAIN_WIDTH, + .nos = TLBI_RT_DOMAIN_WIDTH, + .nvos = TLBI_RT_DOMAIN_WIDTH, +}; + +/* + * validate_ids - Validate TLBID domain bit configurations + * + * Validates that NIS/NVIS and NOS/NVOS configurations follow Arm architecture + * constraints: if outer domain bits are zero, inner domain bits must also be + * zero. Outputs error and zeroes out all domain counts if validation fails. + */ +static void validate_ids(u8 nis, u8 nvis, u8 nos, u8 nvos) +{ + bool inv = false; + + switch (nos) { + case 0: + inv = tlbid_data.nvos > 0; + break; + case 1 ... 8: + inv = tlbid_data.nvos > 6; + break; + case 9 ... 16: + inv = tlbid_data.nvos > 5; + break; + } + + if (inv) { + pr_err("Illegal value of NVOS(%u) for NOS(%u)\n", + tlbid_data.nvos, tlbid_data.nos); + goto on_error; + } + + switch (nis) { + case 0: + inv = tlbid_data.nvis > 0; + break; + case 1 ... 8: + inv = tlbid_data.nvis > 6; + break; + case 9 ... 16: + inv = tlbid_data.nvis > 5; + break; + } + + if (!inv) + return; + + pr_err("Illegal value of NVIS(%u) for NIS(%u)\n", + tlbid_data.nvis, tlbid_data.nis); +on_error: + tlbid_data.nis = 0; + tlbid_data.nvis = 0; + tlbid_data.nos = 0; + tlbid_data.nvos = 0; +} + +/* + * find_target - Find TLBI target by type and identifier + * + * Looks up the tlbi_target structure associated with the given type and cpu. + * For PROCESSOR type, converts CPU logical ID to ACPI Processor UID. + */ +static struct tlbi_target *find_target(int type, int cpu) +{ + struct tlbi_target *target = NULL; + struct list_head *head; + int uid; + + switch (type) { + case ACPI_TLBI_TYPE_PROCESSOR: + uid = get_acpi_id_for_cpu(cpu); + break; + default: + pr_err("Only support Processor as TLBI broadcast target\n"); + return NULL; + } + + head = &tlbid_data.target_list[type]; + list_for_each_entry(target, head, list) { + if (target->identifier == uid) + break; + } + + return target; +} + +void add_target(int type, struct tlbi_target *target) +{ + s32 i, id; + + for (i = 0; i < target->nr_domain; i++) { + id = target->domains[i]; + if ((id > 0) && !test_and_set_bit(id, tlbid_data.found_domains)) + tlbid_data.nr_domain += 1; + } + + INIT_LIST_HEAD(&target->list); + list_add_tail(&target->list, &tlbid_data.target_list[type]); +} + +/* + * pick_best_domain - Select optimal TLBI domain for given CPUs + * + * Iterates through cpu_domain table to find a domain whose cpumask exactly + * matches the given active CPUs. This allows targeted TLBI instead of + * broadcasting to all CPUs, improving TLB invalidation efficiency. + * Returns domain ID on success, 0 on failure. + */ +int pick_best_domain(const cpumask_t *active_cpus) +{ + struct tlbi_domain *domain; + cpumask_t tmp; + int nr_cpu; + + /* Domain 0 is the only supported for Innern Shareable TLBI */ + if (unlikely(tlbid_data.nr_domain == 1)) + goto domain0; + + nr_cpu = cpumask_weight(active_cpus); + if (unlikely(nr_cpu == 0)) + goto domain0; + + do { + domain = &tlbid_data.cpu_domain[nr_cpu - 1][0]; + while (domain->id != TLBI_INV_DOMAIN) { + if (cpumask_and(&tmp, active_cpus, &domain->cpumask) && + cpumask_equal(&tmp, active_cpus)) + return domain->id; + domain++; + } + } while (++nr_cpu <= num_possible_cpus()); + +domain0: + return 0; +} + +int __init tlbid_data_init(void) +{ + s32 max_domains = 1 << TLBI_RT_DOMAIN_WIDTH; + int type; + + if (!tlbid_data.nis || !tlbid_data.nos) + return -EINVAL; + + tlbid_data.found_domains = kcalloc(BITS_TO_LONGS(max_domains), + sizeof(long), GFP_KERNEL); + if (!tlbid_data.found_domains) + return -ENOMEM; + + type = ACPI_TLBI_TYPE_PROCESSOR; + while (type != ACPI_TLBI_TYPE_RESERVED) + INIT_LIST_HEAD(&tlbid_data.target_list[type++]); + + return 0; +} + +void __init tlbid_data_reset(void) +{ + struct tlbi_target *target, *next; + int cpu, type; + + for (type = ACPI_TLBI_TYPE_PROCESSOR; + type < ACPI_TLBI_TYPE_RESERVED; type++) { + list_for_each_entry_safe(target, next, + &tlbid_data.target_list[type], list) { + list_del(&target->list); + kfree(target->domains); + kfree(target); + } + } + + for_each_possible_cpu(cpu) { + kfree(tlbid_data.cpu_domain[cpu]); + tlbid_data.cpu_domain[cpu] = NULL; + } + + kfree(tlbid_data.found_domains); + tlbid_data.found_domains = NULL; + tlbid_data.nr_domain = 1; + tlbid_data.nis = 0; + tlbid_data.nos = 0; + tlbid_data.nvis = 0; + tlbid_data.nvos = 0; +} + +/* + * parse_tlbid_topology - Initialize TLBI domain mapping tables + * + * Allocates and populates the cpu_domain table: + * 1. Build CPU-to-domain mapping from ACPI-parsed target data + * 2. Reorganize data for efficient lookup (domain x CPU matrix) + * + * Returns 0 on success, -ENOMEM on allocation failure, -EVINAL on topology + * parse failure + */ +int __init parse_tlbid_topology(void) +{ + struct tlbi_domain *src, *dst; + struct tlbi_target *target; + int cpu, err; + s32 i; + + if (tlbid_data.nr_domain >= (1 << max(tlbid_data.nis, tlbid_data.nos))) { + pr_err("%d TLBI Domains ACPI reports are more than CPU support.\n", + tlbid_data.nr_domain); + err = -EINVAL; + goto on_error; + } + pr_info("found %d TLBI Domain(s).\n", tlbid_data.nr_domain); + + /* + * Allocate a sparse table to map the CPU count to domain list sharing + * the same CPU count. + * + * NOTICE: Each domain is assumed to contain all system-wide SMMUs, + * Any broadcast in a specific domain must wait replies from all SMMUs + * before it complete. + */ + for_each_possible_cpu(cpu) { + tlbid_data.cpu_domain[cpu] = + kcalloc(tlbid_data.nr_domain, + sizeof(struct tlbi_domain), GFP_KERNEL); + if (!tlbid_data.cpu_domain[cpu]) { + err = -ENOMEM; + goto on_error; + } + for (i = 0; i < tlbid_data.nr_domain; i++) { + cpumask_clear(&tlbid_data.cpu_domain[cpu][i].cpumask); + tlbid_data.cpu_domain[cpu][i].id = cpu ? + TLBI_INV_DOMAIN : i; + } + } + + /* Initialize the cpumask associated with each tlbi domain */ + for_each_possible_cpu(cpu) { + target = find_target(ACPI_TLBI_TYPE_PROCESSOR, cpu); + if (!target) { + err = -EINVAL; + goto on_error; + } + for (i = 0; i < target->nr_domain; i++) { + /* Record all CPUs belong to the same domain */ + src = &tlbid_data.cpu_domain[0][target->domains[i]]; + cpumask_set_cpu(cpu, &(src->cpumask)); + } + } + + /* Move each domain data onto the right position in sparse table */ + for (i = 0; i < tlbid_data.nr_domain; i++) { + src = &tlbid_data.cpu_domain[0][i]; + cpu = cpumask_weight(&(src->cpumask)); + + if (cpu == 1) + continue; + + if (cpu == 0) { + pr_err("TLBI Domain %d has no CPU\n", i); + goto on_error; + } + + dst = &tlbid_data.cpu_domain[cpu - 1][0]; + while (dst->id != TLBI_INV_DOMAIN) + dst++; + memcpy(dst, src, sizeof(*dst)); + cpumask_clear(&src->cpumask); + src->id = TLBI_INV_DOMAIN; + } + + return 0; + +on_error: + for_each_possible_cpu(cpu) + kfree(tlbid_data.cpu_domain[cpu]); + return err; +} + +/* + * tlbid_early_init - Early initialization of TLBID hardware capability after + * all CPUs activate. + * + * Called early during boot from setup_system_features() before ACPI is parsed. + * Reads TLBIDIDR_EL1 register on each CPU to discover hardware capabilities + * (NIS, NVIS, NOS, NVOS bits), and takes the minimum across all CPUs. + * Validates the combined configuration via validate_ids(). + */ +void __init tlbid_early_init(void) +{ + u16 nis, nos, nvis, nvos; + u64 tlbididr; + int cpu; + + if (!system_supports_tlbid()) { + tlbid_data.nis = 0; + tlbid_data.nos = 0; + tlbid_data.nvis = 0; + tlbid_data.nvos = 0; + return; + } + + for_each_online_cpu(cpu) { + tlbididr = per_cpu_ptr(&cpu_data, cpu)->reg_tlbididr; + nis = FIELD_GET(TLBIDIDR_EL1_NIS_MASK, tlbididr); + nos = FIELD_GET(TLBIDIDR_EL1_NOS_MASK, tlbididr); + nvis = FIELD_GET(TLBIDIDR_EL1_NVIS_MASK, tlbididr); + nvos = FIELD_GET(TLBIDIDR_EL1_NVOS_MASK, tlbididr); + + if (tlbid_data.nis > nis) + tlbid_data.nis = nis; + + if (tlbid_data.nos > nos) + tlbid_data.nos = nos; + + if (tlbid_data.nvis > nvis) + tlbid_data.nvis = nvis; + + if (tlbid_data.nvos > nvos) + tlbid_data.nvos = nvos; + } + + validate_ids(tlbid_data.nis, tlbid_data.nvis, + tlbid_data.nos, tlbid_data.nvos); + + pr_info("TLBI Domain bits: nis %d nos %d nvis %d nvos %d\n", + tlbid_data.nis, tlbid_data.nos, tlbid_data.nvis, tlbid_data.nvos); +} + +#ifdef CONFIG_TLBID_KUNIT_TEST +#include "test_tlbidomain.c" +#endif diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 1e8dee638509..2155315581b2 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3575,3 +3575,14 @@ Field 5 F Field 4 P Field 3:0 Align EndSysreg + +Sysreg TLBIDIDR_EL1 3 0 10 4 6 +Res0 63:45 +Field 44:40 NVOS +Res0 39:37 +Field 36:32 NOS +Res0 31:13 +Field 12:8 NVIS +Res0 7:5 +Field 4:0 NIS +EndSysreg diff --git a/drivers/acpi/arm64/Makefile b/drivers/acpi/arm64/Makefile index 9497a7777729..2c068491f16e 100644 --- a/drivers/acpi/arm64/Makefile +++ b/drivers/acpi/arm64/Makefile @@ -5,5 +5,6 @@ obj-$(CONFIG_ACPI_GTDT) += gtdt.o obj-$(CONFIG_ACPI_APMT) += apmt.o obj-$(CONFIG_ARM_AMBA) += amba.o obj-$(CONFIG_ACPI_MPAM) += mpam.o +obj-$(CONFIG_ARM64_TLBID) += tlbid.o obj-y += dma.o init.o diff --git a/drivers/acpi/arm64/init.c b/drivers/acpi/arm64/init.c index d0c8aed90fd1..309f29b63f5d 100644 --- a/drivers/acpi/arm64/init.c +++ b/drivers/acpi/arm64/init.c @@ -12,4 +12,6 @@ void __init acpi_arm_init(void) acpi_iort_init(); if (IS_ENABLED(CONFIG_ARM_AMBA)) acpi_amba_init(); + if (IS_ENABLED(CONFIG_ARM64_TLBID)) + acpi_tlbi_init(); } diff --git a/drivers/acpi/arm64/init.h b/drivers/acpi/arm64/init.h index dcc277977194..c036e2622a2f 100644 --- a/drivers/acpi/arm64/init.h +++ b/drivers/acpi/arm64/init.h @@ -5,3 +5,4 @@ void __init acpi_agdi_init(void); void __init acpi_apmt_init(void); void __init acpi_iort_init(void); void __init acpi_amba_init(void); +void __init acpi_tlbi_init(void); diff --git a/drivers/acpi/arm64/tlbid.c b/drivers/acpi/arm64/tlbid.c new file mode 100644 index 000000000000..d6b1ee007e5d --- /dev/null +++ b/drivers/acpi/arm64/tlbid.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file implements handling of + * Arm Translation Lookaside Buffer Invalidate Domain table (TLBI) + * + * Copyright (C) 2026, Huawei Ltd. + */ +#define pr_fmt(fmt) "ACPI: TLBID: " fmt + +#include <linux/acpi.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/slab.h> + +#include "init.h" + +#include <asm/tlbidomain.h> + +static struct acpi_table_header *acpi_tlbid_table; + +/* + * tlbid_parse_subtable - Parse ACPI TLBI subtable entry + * + * Called for each entry in the ACPI TLBI table. Allocates a tlbi_target + * structure and copies domain IDs from the ACPI table. Builds the + * found_domains bitmap for later domain count calculation. + */ +static int __init tlbid_parse_subtable(union acpi_subtable_headers *header, + unsigned long end) +{ + struct acpi_tlbi_subtable *hdr = (struct acpi_tlbi_subtable *)header; + struct tlbi_target *target; + + if (!hdr) + return -EINVAL; + + target = kzalloc(sizeof(*target), GFP_KERNEL); + if (!target) + return -ENOMEM; + + /* The ACPI Processor UID of the corresponding GICC MADT entry */ + target->nr_domain = hdr->nr_domain; + target->identifier = hdr->identifier; + target->domains = kcalloc(hdr->nr_domain, sizeof(*target->domains), + GFP_KERNEL); + if (!target->domains) { + kfree(target); + return -ENOMEM; + } + + memcpy(target->domains, (void *)acpi_tlbid_table + hdr->offset, + hdr->nr_domain * sizeof(*target->domains)); + + add_target(header->tlbi.type, target); + + return 0; +} + +/* + * acpi_tlbi_init - Initialize TLBID from ACPI table + * + * Main entry point for TLBI table parsing. Gets the ACPI TLBI table, + * parses Processor and SMMU subtable entries, calculates total domain + * count, validates against hardware capability, and calls + * tlbid_chip_data_init() to build domain mapping tables. + * Outputs appropriate error messages on failure. + */ +void __init acpi_tlbi_init(void) +{ + enum acpi_tlbi_target_type type; + acpi_status status; + int ret; + + if (tlbid_data_init() < 0) + goto on_error; + + status = acpi_get_table(ACPI_SIG_TLBI, 0, &acpi_tlbid_table); + if (ACPI_FAILURE(status)) + goto on_error; + + type = ACPI_TLBI_TYPE_PROCESSOR; + while (type != ACPI_TLBI_TYPE_RESERVED) { + ret = acpi_table_parse_entries(ACPI_SIG_TLBI, + sizeof(struct acpi_table_tlbi), + type, tlbid_parse_subtable, 0); + if (ret <= 0) { + pr_err("Failed to get TLBID subtable for %s\n", + type == ACPI_TLBI_TYPE_PROCESSOR ? "Processor":"SMMU"); + goto on_error; + } + type++; + } + + if (parse_tlbid_topology()) { + pr_warn("Failed to initialize TLBI Domain data\n"); + goto on_error; + } + + acpi_put_table(acpi_tlbid_table); + return; + +on_error: + tlbid_data_reset(); + acpi_put_table(acpi_tlbid_table); +} diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 4fca04c8dd2d..62d786110ebf 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -42,6 +42,7 @@ enum acpi_subtable_type { ACPI_SUBTABLE_HMAT, ACPI_SUBTABLE_PRMT, ACPI_SUBTABLE_CEDT, + ACPI_SUBTABLE_TLBI, }; struct acpi_subtable_entry { @@ -287,6 +288,8 @@ acpi_get_entry_type(struct acpi_subtable_entry *entry) return 0; case ACPI_SUBTABLE_CEDT: return entry->hdr->cedt.type; + case ACPI_SUBTABLE_TLBI: + return entry->hdr->tlbi.type; } return 0; } @@ -303,6 +306,8 @@ acpi_get_entry_length(struct acpi_subtable_entry *entry) return entry->hdr->prmt.length; case ACPI_SUBTABLE_CEDT: return entry->hdr->cedt.length; + case ACPI_SUBTABLE_TLBI: + return entry->hdr->tlbi.length; } return 0; } @@ -319,6 +324,8 @@ acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) return sizeof(entry->hdr->prmt); case ACPI_SUBTABLE_CEDT: return sizeof(entry->hdr->cedt); + case ACPI_SUBTABLE_TLBI: + return sizeof(entry->hdr->tlbi); } return 0; } @@ -332,6 +339,8 @@ acpi_get_subtable_type(char *id) return ACPI_SUBTABLE_PRMT; if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) return ACPI_SUBTABLE_CEDT; + if (strncmp(id, ACPI_SIG_TLBI, 4) == 0) + return ACPI_SUBTABLE_TLBI; return ACPI_SUBTABLE_COMMON; } diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 97b05bd334a5..29dda0318c0f 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -26,6 +26,7 @@ */ #define ACPI_SIG_AGDI "AGDI" /* Arm Generic Diagnostic Dump and Reset Device Interface */ #define ACPI_SIG_APMT "APMT" /* Arm Performance Monitoring Unit table */ +#define ACPI_SIG_TLBI "TLBI" /* Translation Lookaside Buffer Invalidate Domain Table */ #define ACPI_SIG_BDAT "BDAT" /* BIOS Data ACPI Table */ #define ACPI_SIG_CCEL "CCEL" /* CC Event Log Table */ #define ACPI_SIG_CDAT "CDAT" /* Coherent Device Attribute Table */ @@ -341,6 +342,34 @@ enum acpi_apmt_node_type { #define ACPI_APMT_OVFLW_IRQ_FLAGS_TYPE_WIRED (0<<1) +/******************************************************************************* + * TLBI - Arm Translation Lookaside Buffer Invalidation Domain Table (TLBI) + * + ******************************************************************************/ +struct acpi_table_tlbi { + struct acpi_table_header header; /* Common ACPI table header */ + u32 nr_subtable; +}; + +struct acpi_tlbi_header { + u16 type; + u16 reserved; + u32 length; +}; + +struct acpi_tlbi_subtable { + struct acpi_tlbi_header header; + u32 identifier; + u32 nr_domain; + u32 offset; +}; + +enum acpi_tlbi_target_type { + ACPI_TLBI_TYPE_PROCESSOR = 1, + ACPI_TLBI_TYPE_SMMU, + ACPI_TLBI_TYPE_RESERVED, +}; + /******************************************************************************* * * BDAT - BIOS Data ACPI Table diff --git a/include/linux/acpi.h b/include/linux/acpi.h index e59a74273e6e..acd1a9241829 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -127,6 +127,7 @@ union acpi_subtable_headers { struct acpi_hmat_structure hmat; struct acpi_prmt_module_header prmt; struct acpi_cedt_header cedt; + struct acpi_tlbi_header tlbi; }; typedef int (*acpi_tbl_table_handler)(struct acpi_table_header *table); -- 2.33.0