virt inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10066 ---------------------------------------- Add the core TLBID virtualization plumbing: - Define struct tlbidomain embedded in struct kvm_arch, holding per-VM virtual-TLBID state: domain map, per-vDomain cpumasks, per-vCPU data, guest-visible TLBIDIDR value, and guard flags. - Introduce CONFIG_KVM_ARM_VTLBID with #ifdef protection so the feature can be compiled out. - Implement kvm_arm_init_tlbidomain() / free_tlbid_vdomain() to bootstrap and tear down the per-VM domain state on VM creation and destruction. - Implement kvm_arm_tlbidomain_vcpu_init() to allocate the per-vCPU TLBID data. Default the vCPU to vDomain0 so KVM can map vDomain0 to a pDomain for guests that have not enabled TLBID. - Provide the vDomain configuration helpers kvm_arm_vcpu_set_vdomain() / kvm_arm_get_vdomain_num() for the later ioctl wiring. - Define the UAPI constants (capability, ioctl numbers, vDomain structure) and document the new KVM_CAP_ARM_TLBIDOMAIN in Documentation/virt/kvm/api.rst. Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> Signed-off-by: Zhou Wang <wangzhou1@hisilicon.com> --- Documentation/virt/kvm/api.rst | 58 +++++++++ arch/arm64/include/asm/kvm_host.h | 53 ++++++++ arch/arm64/kvm/Kconfig | 12 ++ arch/arm64/kvm/arm.c | 210 ++++++++++++++++++++++++++++++ include/uapi/linux/kvm.h | 14 ++ 5 files changed, 347 insertions(+) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 77c65f16e2a7..940e0a2f9417 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6424,6 +6424,48 @@ the capability to be present. `flags` must currently be zero. +4.145 KVM_ARM_GET_VDOMAIN_NUM +------------------------------ + +:Capability: KVM_CAP_ARM_TLBIDOMAIN +:Architectures: arm64 +:Type: vm ioctl +:Parameters: struct kvm_arm_get_vdomain (out) +:Returns: 0 on success, < 0 on error + +Parameters are specified via the following structure:: + + struct kvm_arm_get_vdomain { + __u8 max_vdomains; + }; + +The ``max_vdomains`` field is the max number of vDomains supported by the +hardware. This value is determined by TLBIDIDR_EL1[NVOS/NVIS]. NVOS/NVIS +is the bit width of the vdomain. + +4.146 KVM_ARM_VCPU_SET_VDOMAIN +------------------------------ + +:Capability: KVM_CAP_ARM_TLBIDOMAIN +:Architectures: arm64 +:Type: vcpu ioctl +:Parameters: struct kvm_arm_set_vdomain (in) +:Returns: 0 on success, < 0 on error + +This ioctl allows KVM to obtain TLBI Domain of Guest. + +Parameters are specified via the following structure:: + + struct kvm_arm_set_vdomain { + __u32 vdomain_bitmap; + __u8 num_vdomains; + }; + +The ``vdomain_bitmap`` field is a bitmap, where '1' indicates that vDomain +includes this vCPU. For example, ``vCPU0: vdomain_bitmap=0b1011`` indicates +that vCPU0 included in vDomain(0, 1, 3). + +The ``num_vdomains`` field is the number of vDomains. 5. The kvm_run structure ======================== @@ -8218,6 +8260,22 @@ Used to configure and set up the memory for a Realm. The available actions are: enter the realm until it has been activated. ================================= ============================================= +7.39 KVM_CAP_ARM_TLBIDOMAIN +------------------------------------- + +:Architectures: arm64 +:Target: VM +:Parameters: None +:Returns: 0 on success, negative value on error + +This capability enables TLBID virtualization, which optimizes tlbi broadcast +range to avoid unnecessary broadcasts. + +When this capability is enabled, KVM can obtain the vDomain bitmaps of VM. When +a vCPU is loaded, KVM writes the mapping between the vDomain and the pDomain +into the VTLBID(n). If guest enabled TLBID, hardware will convert vDomain id to +pDomain id for TLBI broadcast. + 8. Other capabilities. ====================== diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 07e8144d55f7..42a9927f7a4b 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -45,6 +45,20 @@ #define KVM_VCPU_MAX_FEATURES 9 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) +#define NIS_RANGE_MIN_1_8 1 +#define NIS_RANGE_MAX_1_8 8 +#define NIS_RANGE_MIN_9_16 9 +#define NIS_RANGE_MAX_9_16 16 +#define VTLBID_TD_WIDTH_NIS_1_8 8 +#define VTLBID_TD_WIDTH_NIS_9_16 16 +#define VTLBID_EL2_WIDTH 64 +/* + * Cap the supported NVIS so per-vDomain bitmaps have a compile-time + * upper bound; the hardware field itself is 5 bits wide. + */ +#define KVM_VTLBID_MAX_NVIS 5 +#define KVM_VTLBID_MAX_VDOMAINS (1U << KVM_VTLBID_MAX_NVIS) + #define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) @@ -57,6 +71,7 @@ #define KVM_REQ_RELOAD_TLBI_DVMBM KVM_ARCH_REQ(8) #define KVM_REQ_RELOAD_WFI_TRAPS KVM_ARCH_REQ(9) #define KVM_REQ_RELOAD_TIMER_EARLY_INJECT KVM_ARCH_REQ(10) +#define KVM_REQ_RELOAD_VTLBID KVM_ARCH_REQ(11) #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -208,6 +223,40 @@ struct kvm_protected_vm { struct kvm_hyp_memcache teardown_mc; }; +struct vcpu_tlbid_data { + /* Node in per-pCPU vCPU list of tlbidomain */ + struct list_head pcpu_node; + /* Store vDomain bitmap, indexed by vCPU ID */ + u32 vdomain_bitmap; + /* Record the last pCPU where vCPU was running, indexed by vCPU ID */ + int last_pcpu; +}; + +struct tlbidomain { + /* Using xarray to dynamically allocate vCPU-related TLBID info */ + struct xarray vcpu_data_array; + /* The number of vDomains */ + int num_domains; + /* Mapping of pDomain to vDomain, indexed by vDomain ID */ + int *domain_map; + /* Record the bitmap of pCPUs mapped to vDomain, indexed by vDomain ID */ + cpumask_var_t *vdomain_cpumasks; + /* Whether virt-TLBID is enabled by KVM */ + bool kvm_tlbid_enabled; + /* Whether virt-TLBID is enabled by Guest */ + bool guest_tlbid_enabled; + /* TLBIDIDR.NVIS */ + u8 nvis; + /* TLBIDIDR.NIS */ + u8 nis; + /* Guest-visible TLBIDIDR value (NIS/NOS carry virtual domain counts) */ + u64 tlbididr_val; + /* Used to prevent concurrent modifications to the domain mapping. */ + spinlock_t tlbid_lock; + /* Per-pCPU list of vCPUs whose last_pcpu equals this pCPU */ + struct list_head *pcpu_vcpu_list; +}; + struct kvm_arch { struct kvm_s2_mmu mmu; @@ -332,6 +381,10 @@ struct kvm_arch { KABI_EXTEND(u64 midr_el1) KABI_EXTEND(u64 revidr_el1) KABI_EXTEND(u64 aidr_el1) + +#ifdef CONFIG_KVM_ARM_VTLBID + struct tlbidomain tlbidomain; +#endif }; struct kvm_vcpu_fault_info { diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 689d32b140b7..2a71109ce082 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -145,4 +145,16 @@ config VIRT_TIMER_EARLY_INJECT The hypervisor can control the early injection latency via the module parameter: timer_early_inject_ns +config KVM_ARM_VTLBID + bool "Virtual TLBID support for KVM guests" + depends on KVM && ARM64_TLBID + default n + help + Enable virtualization of FEAT_TLBID so that each KVM guest can + use TLBI Domains independently of the host. The host TLBID + hardware is partitioned per-VM through VTLBID_EL2 registers + managed on the vCPU load/put path. + + If unsure, say N. + endif # VIRTUALIZATION diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 324553fea612..565b26248d4e 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -43,6 +43,7 @@ #include <asm/kvm_rme.h> #include <asm/sections.h> #include <asm/kvm_tmi.h> +#include <asm/tlbid.h> #include <kvm/arm_hypercalls.h> #include <kvm/arm_pmu.h> #include <kvm/arm_psci.h> @@ -319,6 +320,104 @@ static int kvm_arm_default_max_vcpus(void) return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; } +#ifdef CONFIG_KVM_ARM_VTLBID +static int kvm_arm_init_tlbidomain(struct kvm *kvm) +{ + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + int i, max_vdomains; + u64 tlbididr; + + if (!system_supports_tlbid()) + return 0; + + tlbididr = read_sysreg_s(SYS_TLBIDIDR_EL1); + + vdomain->nis = FIELD_GET(TLBIDIDR_EL1_NIS_MASK, tlbididr); + vdomain->nvis = min_t(u8, FIELD_GET(TLBIDIDR_EL1_NVIS_MASK, tlbididr), + KVM_VTLBID_MAX_NVIS); + vdomain->num_domains = -1; + max_vdomains = 1 << vdomain->nvis; + + kvm_info("TLBIDIDR: 0x%llx, NIS: %d, NVIS: %d\n", tlbididr, vdomain->nis, vdomain->nvis); + + xa_init(&vdomain->vcpu_data_array); + + vdomain->domain_map = kcalloc(max_vdomains, sizeof(*vdomain->domain_map), + GFP_KERNEL); + if (!vdomain->domain_map) + goto destroy_vcpu_data_array; + + vdomain->vdomain_cpumasks = kcalloc(max_vdomains, sizeof(cpumask_var_t), + GFP_KERNEL); + if (!vdomain->vdomain_cpumasks) + goto free_domain_map; + + for (i = 0; i < max_vdomains; i++) { + if (!alloc_cpumask_var(&vdomain->vdomain_cpumasks[i], GFP_KERNEL)) + goto free_vdomain_cpumasks; + cpumask_clear(vdomain->vdomain_cpumasks[i]); + } + + for (i = 0; i < max_vdomains; i++) + vdomain->domain_map[i] = -1; + + spin_lock_init(&vdomain->tlbid_lock); + + vdomain->pcpu_vcpu_list = kcalloc(nr_cpu_ids, + sizeof(*vdomain->pcpu_vcpu_list), GFP_KERNEL); + if (!vdomain->pcpu_vcpu_list) + goto free_vdomain_cpumasks; + + for (i = 0; i < nr_cpu_ids; i++) + INIT_LIST_HEAD(&vdomain->pcpu_vcpu_list[i]); + + /* Enable vTLBID only after all resources are allocated. */ + vdomain->kvm_tlbid_enabled = true; + + return 0; + +free_vdomain_cpumasks: + while (--i >= 0) + free_cpumask_var(vdomain->vdomain_cpumasks[i]); + kfree(vdomain->vdomain_cpumasks); +free_domain_map: + kfree(vdomain->domain_map); +destroy_vcpu_data_array: + xa_destroy(&vdomain->vcpu_data_array); + + kvm_err("vTLBID init failed with -ENOMEM, feature disabled\n"); + return -ENOMEM; +} + +static void free_tlbid_vdomain(struct kvm *kvm) +{ + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + struct vcpu_tlbid_data *data; + int i, max_vdomains; + unsigned long index; + + if (!vdomain->kvm_tlbid_enabled) + return; + + max_vdomains = 1 << vdomain->nvis; + + kfree(vdomain->domain_map); + + for (i = 0; i < max_vdomains; i++) + free_cpumask_var(vdomain->vdomain_cpumasks[i]); + kfree(vdomain->vdomain_cpumasks); + + kfree(vdomain->pcpu_vcpu_list); + + xa_for_each(&vdomain->vcpu_data_array, index, data) + kfree(data); + xa_destroy(&vdomain->vcpu_data_array); +} +#else +static inline int kvm_arm_init_tlbidomain(struct kvm *kvm) { return 0; } +static inline void free_tlbid_vdomain(struct kvm *kvm) { } +#endif + /** * kvm_arch_init_vm - initializes a VM data structure * @kvm: pointer to the KVM struct @@ -1765,6 +1864,52 @@ static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, return kvm_reset_vcpu(vcpu); } +#ifdef CONFIG_KVM_ARM_VTLBID +static int kvm_arm_tlbidomain_vcpu_init(struct kvm_vcpu *vcpu) +{ + struct tlbidomain *vdomain; + struct vcpu_tlbid_data *data; + int ret; + + if (!system_supports_tlbid()) + return 0; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + /* + * When the host has enabled TLBID but the guest has not, + * KVM can map vDomain0 to pDomainN to optimize the TLBI + * instructions for the guest. Therefore, the value is set + * to 0x1, indicating that the vCPU is in vDomain0. If QEMU + * calls KVM_ARM_VCPU_SET_VDOMAIN to configure vdomains, this + * value will be overwritten to support the guest enabling + * TLBID. + */ + data->vdomain_bitmap = 0x1; + data->last_pcpu = -1; + INIT_LIST_HEAD(&data->pcpu_node); + + vdomain = &vcpu->kvm->arch.tlbidomain; + if (!vdomain->kvm_tlbid_enabled) { + kfree(data); + return 0; + } + + ret = xa_insert(&vdomain->vcpu_data_array, + vcpu->vcpu_idx, data, GFP_KERNEL); + if (ret) { + kfree(data); + return ret == -EBUSY ? 0 : ret; + } + + return 0; +} +#else +static inline int kvm_arm_tlbidomain_vcpu_init(struct kvm_vcpu *vcpu) { return 0; } +#endif + static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) { @@ -1912,6 +2057,50 @@ static int kvm_arm_vcpu_rmm_psci_complete(struct kvm_vcpu *vcpu, return realm_psci_complete(vcpu, target, arg->psci_status); } +#ifdef CONFIG_KVM_ARM_VTLBID +static int kvm_arm_vcpu_set_vdomain(struct kvm_vcpu *vcpu, + struct kvm_arm_set_vdomain *vdomain) +{ + struct tlbidomain *td = &vcpu->kvm->arch.tlbidomain; + u32 vdomain_bitmap = vdomain->vdomain_bitmap; + u8 num_vdomains = vdomain->num_vdomains; + u8 nvis, max_vdomains; + struct vcpu_tlbid_data *data; + + if (!td->kvm_tlbid_enabled) + return -EINVAL; + + nvis = td->nvis; + max_vdomains = 1 << nvis; + + if (num_vdomains > max_vdomains) + return -EINVAL; + + if (td->num_domains != -1 && td->num_domains != num_vdomains) + return -EINVAL; + + data = xa_load(&td->vcpu_data_array, vcpu->vcpu_idx); + if (!data) { + kvm_err("vCPU%d: Failed to load vcpu_tlbid_data\n", vcpu->vcpu_idx); + return -EINVAL; + } + + data->vdomain_bitmap = vdomain_bitmap; + td->num_domains = num_vdomains; + td->guest_tlbid_enabled = true; + + kvm_info("vCPU%d: vDomain bitmap: 0x%x\n", vcpu->vcpu_idx, vdomain_bitmap); + + return 0; +} +#else +static inline int kvm_arm_vcpu_set_vdomain(struct kvm_vcpu *vcpu, + struct kvm_arm_set_vdomain *vdomain) +{ + return -EINVAL; +} +#endif + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -2110,6 +2299,27 @@ static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) } } +#ifdef CONFIG_KVM_ARM_VTLBID +static int kvm_arm_get_vdomain_num(struct kvm *kvm, + struct kvm_arm_get_vdomain *vdomain) +{ + struct tlbidomain *td = &kvm->arch.tlbidomain; + + if (!td->kvm_tlbid_enabled) + return -EINVAL; + + vdomain->max_vdomains = 1 << td->nvis; + + return 0; +} +#else +static inline int kvm_arm_get_vdomain_num(struct kvm *kvm, + struct kvm_arm_get_vdomain *vdomain) +{ + return -EINVAL; +} +#endif + int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm *kvm = filp->private_data; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 18ed07a68b2d..7d1a03393523 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1315,6 +1315,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_HYGON_COCO_EXT_CSV3_SP_MGR (1 << 4) #define KVM_CAP_ARM_HW_DIRTY_STATE_TRACK 502 +#define KVM_CAP_ARM_TLBIDOMAIN 503 #define KVM_CAP_ARM_HISI_IPIV 798 #define KVM_CAP_ARM_VIRT_MSI_BYPASS 799 @@ -2358,4 +2359,17 @@ struct kvm_pre_fault_memory { __u64 padding[5]; }; +struct kvm_arm_get_vdomain { + __u8 max_vdomains; +}; + +#define KVM_ARM_GET_VDOMAIN_NUM _IOR(KVMIO, 0xd7, struct kvm_arm_get_vdomain) + +struct kvm_arm_set_vdomain { + __u32 vdomain_bitmap; + __u8 num_vdomains; +}; + +#define KVM_ARM_VCPU_SET_VDOMAIN _IOW(KVMIO, 0xd8, struct kvm_arm_set_vdomain) + #endif /* __LINUX_KVM_H */ -- 2.33.0