Add the -tlbidomain command-line option for the ARM virt machine, allowing users to define TLBI domain topology by mapping domain IDs to vCPU ranges. Supports multiple cpus= values per domain entry: -tlbidomain domain-id=0,cpus=0-7 -tlbidomain domain-id=1,cpus=0-3,cpus=5-7 -tlbidomain domain-id=2,cpus=4 Each cpus= value accepts a single CPU (N) or a range (N-M). Domain 0 must cover all vCPUs (broadcast domain), and domain IDs must be contiguous starting from 0. The option is parsed into per-domain CPU bitmaps (like NUMA's approach) held in VirtMachineState, simplifying both validation and later table generation. Per-vCPU domain membership is computed on demand via virt_cpu_vdomain_bitmap() and consumed by the KVM vdomain setup and the TLBI ACPI table generation. If KVM is enabled, the requested topology is validated against the number of vdomains KVM supports, queried via kvm_arm_get_max_vdomains(). Because the tlbidomain QemuOptsList is unconditionally registered at startup, qemu_find_opts_err("tlbidomain") always returns non-NULL; guard with QTAILQ_EMPTY (consistent with the accel option handling in vl.c) so VMs start normally when no -tlbidomain option is given, even on hosts without TLBI vdomain support. Signed-off-by: Tian Zheng <zhengtian10@huawei.com> Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- hw/arm/virt.c | 161 ++++++++++++++++++++++++++++++++++++++++++ include/hw/arm/virt.h | 13 ++++ qapi/machine.json | 16 +++++ qemu-options.hx | 17 +++++ system/vl.c | 16 +++++ 5 files changed, 223 insertions(+) diff --git a/hw/arm/virt.c b/hw/arm/virt.c index f061b1c069..72e31ec6a6 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -33,7 +33,9 @@ #include "qemu/datadir.h" #include "qemu/units.h" #include "qemu/option.h" +#include "qemu/config-file.h" #include "qemu/log.h" +#include "qemu/bitmap.h" #include "monitor/qdev.h" #include "hw/sysbus.h" #include "hw/arm/boot.h" @@ -91,6 +93,8 @@ #include "hw/char/pl011.h" #include "qemu/guest-random.h" #include "qapi/qmp/qdict.h" +#include "qapi/opts-visitor.h" +#include "qapi/qapi-visit-machine.h" #include "qemu/log.h" #ifdef CONFIG_UB #include "hw/ub/ub.h" @@ -304,6 +308,7 @@ static int virt_get_socket_id(const MachineState *ms, int cpu_index); static int virt_get_cluster_id(const MachineState *ms, int cpu_index); static int virt_get_core_id(const MachineState *ms, int cpu_index); static int virt_get_thread_id(const MachineState *ms, int cpu_index); +static void virt_parse_vtlbidomain(VirtMachineState *vms); static bool cpu_type_valid(const char *cpu) { @@ -2824,6 +2829,8 @@ static void machvirt_init(MachineState *machine) finalize_gic_version(vms); + virt_parse_vtlbidomain(vms); + possible_cpus = mc->possible_cpu_arch_ids(machine); /* @@ -3378,6 +3385,160 @@ static void virt_set_oem_table_id(Object *obj, const char *value, strncpy(vms->oem_table_id, value, 8); } +/* QemuOpts foreach callback: parse one -tlbidomain option entry */ +static int virt_parse_one_vtlbidomain(void *opaque, QemuOpts *opts, + Error **errp) +{ + VirtMachineState *vms = opaque; + TlbiVdomainConfig *d; + TlbiDomainOptions *tdo = NULL; + uint8_t vdomain_id; + uint16List *cpus; + int i; + Error *err = NULL; + Visitor *v = opts_visitor_new(opts); + + visit_type_TlbiDomainOptions(v, NULL, &tdo, &err); + visit_free(v); + if (err) { + error_report("tlbidomain: OptsVisitor parse failed: %s", + error_get_pretty(err)); + error_free(err); + return -1; + } + + if (vms->num_vdomains >= vms->max_vdomains) { + error_report("tlbidomain: too many vdomains, max %d", + vms->max_vdomains); + exit(1); + } + + if (!tdo->has_cpus) { + error_report("tlbidomain: missing cpus"); + exit(1); + } + if (!tdo->has_domain_id) { + error_report("tlbidomain: missing domain-id"); + exit(1); + } + vdomain_id = tdo->domain_id; + if (vdomain_id >= vms->max_vdomains) { + error_report("tlbidomain: invalid domain-id %u (max %d)", + vdomain_id, vms->max_vdomains - 1); + exit(1); + } + for (i = 0; i < vms->num_vdomains; i++) { + if (vms->vdomains[i].vdomain_id == vdomain_id) { + error_report("tlbidomain: duplicate domain-id %u", vdomain_id); + exit(1); + } + } + + d = &vms->vdomains[vms->num_vdomains]; + d->vdomain_id = vdomain_id; + d->cpu_bitmap = bitmap_new(MACHINE(vms)->smp.max_cpus); + + for (cpus = tdo->cpus; cpus; cpus = cpus->next) { + if (cpus->value >= MACHINE(vms)->smp.max_cpus) { + error_report("tlbidomain: cpu %u >= max_cpus %u", + cpus->value, MACHINE(vms)->smp.max_cpus); + exit(1); + } + bitmap_set(d->cpu_bitmap, cpus->value, 1); + } + if (bitmap_empty(d->cpu_bitmap, MACHINE(vms)->smp.max_cpus)) { + error_report("tlbidomain: vdomain %u has no cpus", vdomain_id); + exit(1); + } + + qapi_free_TlbiDomainOptions(tdo); + vms->num_vdomains++; + return 0; +} + +uint32_t virt_cpu_vdomain_bitmap(VirtMachineState *vms, int cpu_index) +{ + int i; + uint32_t bm = 0; + + for (i = 0; i < vms->num_vdomains; i++) { + if (test_bit(cpu_index, vms->vdomains[i].cpu_bitmap)) { + bm |= (1U << vms->vdomains[i].vdomain_id); + } + } + return bm; +} + +/* + * Parse all -tlbidomain options, validate constraints: + * - vdomain 0 (broadcast) must cover every vCPU + * - vdomain IDs must be contiguous starting from 0 + * - Total vdomains must not exceed KVM max_vdomains + */ +static void virt_parse_vtlbidomain(VirtMachineState *vms) +{ + QemuOptsList *olist = qemu_find_opts_err("tlbidomain", NULL); + MachineState *ms = MACHINE(vms); + unsigned int max_cpus = ms->smp.max_cpus; + uint8_t max_vdomains; + int i; + + if (!olist || QTAILQ_EMPTY(&olist->head)) { + vms->max_vdomains = 0; + vms->vdomains = NULL; + return; + } + + if (!kvm_enabled()) { + warn_report("tlbidomain: not running under KVM, " + "TLBI domain will only be reflected in ACPI table"); + max_vdomains = VDOMAIN_BITMAP_BITS; + } else { + max_vdomains = kvm_arm_get_max_vdomains(); + if (max_vdomains == 0) { + error_report("tlbidomain: KVM does not support TLBI vdomain"); + exit(1); + } + } + + vms->max_vdomains = max_vdomains; + vms->vdomains = g_new0(TlbiVdomainConfig, max_vdomains); + + if (qemu_opts_foreach(olist, virt_parse_one_vtlbidomain, + vms, NULL) < 0) { + exit(1); + } + + if (vms->num_vdomains == 0) { + return; + } + + /* Validate: vdomain 0 must be the broadcast vdomain (all vCPUs) */ + for (i = 0; i < vms->num_vdomains; i++) { + if (vms->vdomains[i].vdomain_id == 0) { + if (!bitmap_full(vms->vdomains[i].cpu_bitmap, max_cpus)) { + error_report("tlbidomain: vdomain 0 must cover all cpus [0-%u]", + max_cpus - 1); + exit(1); + } + break; + } + } + + /* Validate: vdomain IDs are contiguous [0 .. num_vdomains-1] */ + unsigned long *id_bm = bitmap_new(max_vdomains); + for (i = 0; i < vms->num_vdomains; i++) { + bitmap_set(id_bm, vms->vdomains[i].vdomain_id, 1); + } + if (bitmap_count_one(id_bm, max_vdomains) != vms->num_vdomains || + find_last_bit(id_bm, max_vdomains) != + (unsigned long)(vms->num_vdomains - 1)) { + error_report("tlbidomain: vdomain IDs must be contiguous from 0"); + exit(1); + } + g_free(id_bm); +} + static TargetImplCpu target_impl_cpus[MAX_TARGET_IMPL_CPUS]; static void virt_set_target_impl_cpus(Object *obj, const char *value, diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h index 5dd77096cd..51ad5b0531 100644 --- a/include/hw/arm/virt.h +++ b/include/hw/arm/virt.h @@ -32,6 +32,7 @@ #include "exec/hwaddr.h" #include "qemu/notify.h" +#include "qemu/bitmap.h" #include "hw/boards.h" #include "hw/arm/boot.h" #include "hw/arm/bsa.h" @@ -217,6 +218,14 @@ typedef enum VirtGICType { #define VIRT_GIC_VERSION_3_MASK BIT(VIRT_GIC_VERSION_3) #define VIRT_GIC_VERSION_4_MASK BIT(VIRT_GIC_VERSION_4) +#define VDOMAIN_BITMAP_BITS 32 /* uint32_t vdomain_bitmap width */ + +/* TLBI vdomain: maps a vdomain ID to the set of vCPUs belonging to it */ +typedef struct TlbiVdomainConfig { + uint8_t vdomain_id; + unsigned long *cpu_bitmap; +} TlbiVdomainConfig; + struct VirtMachineClass { MachineClass parent; bool disallow_affinity_adjustment; @@ -310,6 +319,9 @@ struct VirtMachineState { char *kvm_type; NotifierList cpuhp_notifiers; Object *event_log; + TlbiVdomainConfig *vdomains; + int num_vdomains; + uint8_t max_vdomains; /* max vdomains from KVM (via ioctl) */ }; #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM) @@ -319,6 +331,7 @@ OBJECT_DECLARE_TYPE(VirtMachineState, VirtMachineClass, VIRT_MACHINE) void virt_acpi_setup(VirtMachineState *vms); bool virt_is_acpi_enabled(VirtMachineState *vms); +uint32_t virt_cpu_vdomain_bitmap(VirtMachineState *vms, int cpu_index); void virt_madt_cpu_entry(int uid, const CPUArchIdList *cpu_list, GArray *entry, bool force_enabled); bool cpu_l1_cache_unified(int cpu); diff --git a/qapi/machine.json b/qapi/machine.json index edb741aac7..a5033ab329 100644 --- a/qapi/machine.json +++ b/qapi/machine.json @@ -1969,6 +1969,22 @@ '*max-size': 'size', '*slots': 'uint64' } } +## +# @TlbiDomainOptions: +# +# Configure a TLBI vdomain. (for OptsVisitor) +# +# @domain-id: TLBI vdomain ID +# +# @cpus: VCPUs belonging to this vdomain +# +# Since: 8.2 +## +{ 'struct': 'TlbiDomainOptions', + 'data': { + '*domain-id': 'uint8', + '*cpus': ['uint16'] } } + ## # @dumpdtb: # diff --git a/qemu-options.hx b/qemu-options.hx index ec31c4d30a..105ff03bda 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -581,6 +581,23 @@ SRST -numa hmat-cache,node-id=1,size=10K,level=1,associativity=direct,policy=write-back,line=8 ERST +DEF("tlbidomain", HAS_ARG, QEMU_OPTION_tlbidomain, + "-tlbidomain domain-id=id,cpus=cpurange[,cpus=cpurange]\n", + QEMU_ARCH_ALL) +SRST +``-tlbidomain domain-id=id,cpus=firstcpu[-lastcpu][,cpus=firstcpu[-lastcpu]]`` + Define a TLBI domain and assign vCPUs to it. ``domain-id`` is + the domain identifier. ``cpus`` can be a single CPU index or a + range. Multiple ``cpus`` entries accumulate. Domain 0 must + cover all vCPUs. Domain IDs must be contiguous from 0. + + Example:: + + -tlbidomain domain-id=0,cpus=0-7 + -tlbidomain domain-id=1,cpus=0-3,cpus=5-7 + -tlbidomain domain-id=2,cpus=1,cpus=3,cpus=6 +ERST + DEF("add-fd", HAS_ARG, QEMU_OPTION_add_fd, "-add-fd fd=fd,set=set[,opaque=opaque]\n" " Add 'fd' to fd 'set'\n", QEMU_ARCH_ALL) diff --git a/system/vl.c b/system/vl.c index bcf5211e4c..bf3bc55dee 100644 --- a/system/vl.c +++ b/system/vl.c @@ -738,6 +738,14 @@ static void configure_blockdev(BlockdevOptionsQueue *bdo_queue, } +/* QemuOpts definition for -tlbidomain command-line option */ +static QemuOptsList qemu_tlbidomain_opts = { + .name = "tlbidomain", + .implied_opt_name = "domain-id", + .head = QTAILQ_HEAD_INITIALIZER(qemu_tlbidomain_opts.head), + .desc = { { 0 } } /* validated with OptsVisitor */ +}; + static QemuOptsList qemu_smp_opts = { .name = "smp-opts", .implied_opt_name = "cpus", @@ -2844,6 +2852,7 @@ void qemu_init(int argc, char **argv) qemu_add_opts(&qemu_msg_opts); qemu_add_opts(&qemu_name_opts); qemu_add_opts(&qemu_numa_opts); + qemu_add_opts(&qemu_tlbidomain_opts); qemu_add_opts(&qemu_icount_opts); qemu_add_opts(&qemu_semihosting_config_opts); qemu_add_opts(&qemu_fw_cfg_opts); @@ -2961,6 +2970,13 @@ void qemu_init(int argc, char **argv) exit(1); } break; + case QEMU_OPTION_tlbidomain: + opts = qemu_opts_parse_noisily(qemu_find_opts("tlbidomain"), + optarg, true); + if (!opts) { + exit(1); + } + break; case QEMU_OPTION_display: parse_display(optarg); break; -- 2.33.0