virt inclusion category: feature bugzilla: https://gitcode.com/openeuler/kernel/issues/10066 ---------------------------------------- If the vCPU is first online or is migrated, the domain map needs to be checked whether need to update, and new map needs to be written into the VTLBID(n). Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> Signed-off-by: Zhou Wang <wangzhou1@hisilicon.com> --- arch/arm64/include/asm/kvm_host.h | 12 ++ arch/arm64/kvm/arm.c | 217 ++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index d0a6c5b02807..18c95e605d8a 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -45,6 +45,15 @@ #define KVM_VCPU_MAX_FEATURES 9 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) +#define NIS_RANGE_MIN_1_8 1 +#define NIS_RANGE_MAX_1_8 8 +#define NIS_RANGE_MIN_9_16 9 +#define NIS_RANGE_MAX_9_16 16 +#define VTLBID_TD_WIDTH_NIS_1_8 8 +#define VTLBID_TD_WIDTH_NIS_9_16 16 +#define VTLBID_EL2_WIDTH 64 +#define VTLBID_EL2_MAX_REGS 4 + #define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) @@ -57,6 +66,7 @@ #define KVM_REQ_RELOAD_TLBI_DVMBM KVM_ARCH_REQ(8) #define KVM_REQ_RELOAD_WFI_TRAPS KVM_ARCH_REQ(9) #define KVM_REQ_RELOAD_TIMER_EARLY_INJECT KVM_ARCH_REQ(10) +#define KVM_REQ_RELOAD_VTLBID KVM_ARCH_REQ(11) #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -232,6 +242,8 @@ struct tlbidomain { u8 nvis; /* TLBIDIDR.NIS */ u8 nis; + /* Used to prevent concurrent modifications to the domain mapping. */ + spinlock_t tlbid_lock; }; struct kvm_arch { diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 1abcb2b2e7e2..7985f85ab3cf 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -43,6 +43,7 @@ #include <asm/kvm_rme.h> #include <asm/sections.h> #include <asm/kvm_tmi.h> +#include <asm/tlbidomain.h> #include <kvm/arm_hypercalls.h> #include <kvm/arm_pmu.h> #include <kvm/arm_psci.h> @@ -359,6 +360,8 @@ static int kvm_arm_init_tlbidomain(struct kvm *kvm) for (i = 0; i < max_vdomains; i++) kvm->arch.vdomain.domain_map[i] = -1; + spin_lock_init(&kvm->arch.vdomain.tlbid_lock); + return 0; free_vdomain_cpumasks: @@ -822,6 +825,213 @@ static void update_steal_time(struct kvm_vcpu *vcpu) } #endif +static bool is_vcpu_in_vdomain(u32 vdomain_bitmap, int vdomain_idx) +{ + return vdomain_bitmap & BIT(vdomain_idx); +} + +static int update_domain_map(struct kvm_vcpu *vcpu, int vdomain_idx, + int last_pcpu, bool clear_flag) +{ + struct tlbidomain *vdomain = &vcpu->kvm->arch.vdomain; + int pdomain = vdomain->domain_map[vdomain_idx]; + cpumask_var_t new_cpus; + + cpumask_copy(new_cpus, vdomain->vdomain_cpumasks[vdomain_idx]); + + if (unlikely(pdomain == -1)) { + cpumask_set_cpu(vcpu->cpu, new_cpus); + goto update_mask; + } + + if (last_pcpu != -1 && clear_flag) { + /* remove old_cpu, not need to do so when first load */ + if (cpumask_test_cpu(last_pcpu, new_cpus)) { + cpumask_clear_cpu(last_pcpu, new_cpus); + } + } + + cpumask_set_cpu(vcpu->cpu, new_cpus); + +update_mask: + cpumask_copy(vdomain->vdomain_cpumasks[vdomain_idx], new_cpus); + return pick_best_domain(vdomain->vdomain_cpumasks[vdomain_idx]); +} + +static void vtlbidn_clear_set_s(int idx, u64 clear, u64 set) +{ + switch (idx) { + case 0: + sysreg_clear_set_s(SYS_VTLBID0_EL2, clear, set); + break; + case 1: + sysreg_clear_set_s(SYS_VTLBID1_EL2, clear, set); + break; + case 2: + sysreg_clear_set_s(SYS_VTLBID2_EL2, clear, set); + break; + case 3: + sysreg_clear_set_s(SYS_VTLBID3_EL2, clear, set); + break; + default: + BUG_ON(1); + } +} + +static void kvm_arm_update_tlbid_map(struct tlbidomain *vdomain) +{ + int i, pdomain, vtlbid_idx, offset; + int pdomain_bits, nis = vdomain->nis; + int max_vdomains = 1 << vdomain->nvis; + u64 clear, set; + + if (nis >= NIS_RANGE_MIN_1_8 && nis <= NIS_RANGE_MAX_1_8) + pdomain_bits = VTLBID_TD_WIDTH_NIS_1_8; + else if (nis >= NIS_RANGE_MIN_9_16 && nis <= NIS_RANGE_MAX_9_16) + pdomain_bits = VTLBID_TD_WIDTH_NIS_9_16; + + for (i = 0; i < max_vdomains; i++) { + pdomain = vdomain->domain_map[i]; + if (pdomain == -1) { + if (vdomain->domain_map[0] != -1) + pdomain = vdomain->domain_map[0]; + else + pdomain = 0; + } + + vtlbid_idx = i * pdomain_bits / VTLBID_EL2_WIDTH; + offset = i * pdomain_bits % VTLBID_EL2_WIDTH; + + clear = GENMASK(offset + pdomain_bits - 1, offset); + set = pdomain << offset; + + vtlbidn_clear_set_s(vtlbid_idx, clear, set); + } + + sysreg_clear_set_s(SYS_HCRX_EL2, 0, HCRX_EL2_VTLBIDEn); +} + +static void kvm_vcpu_reload_tlbid(struct kvm *kvm) +{ + if (WARN_ON_ONCE(!kvm->arch.vdomain.kvm_tlbid_enabled)) + return; + + preempt_disable(); + kvm_arm_update_tlbid_map(&kvm->arch.vdomain); + preempt_enable(); +} + +static void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + struct tlbidomain *vdomain = &kvm->arch.vdomain; + int max_vdomains = 1 << vdomain->nvis; + DECLARE_BITMAP(clear_bits, max_vdomains); + struct vcpu_tlbid_data *current_data, *other_data; + int i; + + /* + * If we support tlbid, but user does not enable it, we can do + * vdomain0 -> pdomainN as well by default. + */ + if (!vdomain->kvm_tlbid_enabled) + return; + + current_data = xa_load(&vdomain->vcpu_data_array, vcpu->vcpu_idx); + if (!current_data) { + BUG(); + return; + } + + /* check if vCPU thread will move to another pCPU */ + if (likely(vcpu->cpu == current_data->last_pcpu)) + return; + + spin_lock(&vcpu->kvm->arch.vdomain.tlbid_lock); + + /* + * check if another vcpu running on the same pCPU as the one + * the vCPU last ran. If so, do not clear last pCPU from the + * vdomain_cpumask. + */ + bitmap_fill(clear_bits, max_vdomains); + for (i = 0; i < kvm->created_vcpus; i++) { + other_data = xa_load(&vdomain->vcpu_data_array, i); + if (WARN_ON_ONCE(!other_data)) + continue; + + if (i == vcpu->vcpu_idx || other_data->last_pcpu == -1) + continue; + + if (other_data->last_pcpu != current_data->last_pcpu) + continue; + + bitmap_andnot(clear_bits, clear_bits, + (unsigned long *)&other_data->vdomain_bitmap, max_vdomains); + } + + if (!vdomain->guest_tlbid_enabled) { + /* + * If QEMU is not configured with vdomains, only vDomain0 needs + * to be mapped. + */ + vdomain->domain_map[0] = update_domain_map(vcpu, + 0, + current_data->last_pcpu, + test_bit(0, clear_bits)); + } else { + /* + * If QEMU is configured with vdomains, all vdomains need to be + * mapped. For all vdomains which includes this vCPU, check if + * vdomain map should be changed as well. + */ + for (i = 0; i < vdomain->num_domains; i++) { + if (!is_vcpu_in_vdomain(current_data->vdomain_bitmap, i)) + continue; + + vdomain->domain_map[i] = update_domain_map(vcpu, + i, + current_data->last_pcpu, + test_bit(i, clear_bits)); + } + } + + kvm_flush_remote_tlbs(kvm); + + /* + * Before this vcpu load, kick other vCPUs out, so maps for other vCPUs + * will be updated during vCPU load. + * + * Add KVM_REQUEST_WAIT to make sure vCPU is out. + */ + kvm_make_all_cpus_request(kvm, KVM_REQ_RELOAD_VTLBID | KVM_REQUEST_WAIT); + + /* update tlbid hardware map register */ + kvm_arm_update_tlbid_map(vdomain); + + current_data->last_pcpu = vcpu->cpu; + + spin_unlock(&vcpu->kvm->arch.vdomain.tlbid_lock); +} + +static void kvm_tlbidomain_vcpu_put(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + int i; + + if (!kvm->arch.vdomain.kvm_tlbid_enabled) + return; + + sysreg_clear_set_s(SYS_HCRX_EL2, HCRX_EL2_VTLBIDEn, 0); + + /* + * Clear all VTLBID registers so that no stale vdomain-to-pdomain + * mapping leaks into the next vCPU (or host) that runs on this pCPU. + */ + for (i = 0; i < VTLBID_EL2_MAX_REGS; i++) + vtlbidn_clear_set_s(i, VTLBID0_EL2_TD, 0); +} + void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct kvm_s2_mmu *mmu; @@ -877,6 +1087,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_tlbi_dvmbm_vcpu_load(vcpu); + kvm_tlbidomain_vcpu_load(vcpu); + /* * When pv_preempted is changed from enabled to disabled, preempted * state will not be updated in kvm_arch_vcpu_put/load. So we must @@ -915,6 +1127,8 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_tlbi_dvmbm_vcpu_put(vcpu); + kvm_tlbidomain_vcpu_put(vcpu); + if (kvm_arm_is_pvsched_valid(&vcpu->arch) && pv_preempted_enable) kvm_update_pvsched_preempted(vcpu, 1); } @@ -1280,6 +1494,9 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_RELOAD_TLBI_DVMBM, vcpu)) kvm_hisi_reload_lsudvmbm(vcpu->kvm); + if (kvm_check_request(KVM_REQ_RELOAD_VTLBID, vcpu)) + kvm_vcpu_reload_tlbid(vcpu->kvm); + if (kvm_check_request(KVM_REQ_RELOAD_WFI_TRAPS, vcpu)) { if (single_task_running()) vcpu_clear_wfx_traps(vcpu); -- 2.33.0