virt inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10066 ---------------------------------------- Add the VTLBID0_EL2 ~ VTLBID3_EL2 sysreg definitions and the HCRX_EL2.VTLBIDEn control field to arch/arm64/tools/sysreg. Implement kvm_tlbidomain_vcpu_load() / kvm_tlbidomain_vcpu_put() to update the VTLBIDn registers on vCPU schedule-in and tear down the hardware context on schedule-out. On first load or migration, the per-vCPU domain map is recomputed and written to VTLBIDn. Enable HCRX_EL2.VTLBIDEn from __activate_traps_common() at guest entry for VMs with vTLBID enabled, rather than from the vCPU load path, so that the host never runs with vTLBID translation enabled. The guest exit path restores the host HCRX_EL2 value. Handle the KVM_REQ_RELOAD_VTLBID request so that other vCPUs reprogram the VTLBIDn registers after a domain map change. Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> Signed-off-by: Zhou Wang <wangzhou1@hisilicon.com> --- arch/arm64/kvm/arm.c | 235 ++++++++++++++++++++++++ arch/arm64/kvm/hyp/include/hyp/switch.h | 4 + arch/arm64/tools/sysreg | 19 +- 3 files changed, 257 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 565b26248d4e..e9fbdfaabb8b 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -845,6 +845,234 @@ static void update_steal_time(struct kvm_vcpu *vcpu) } #endif +#ifdef CONFIG_KVM_ARM_VTLBID +static bool is_vcpu_in_vdomain(u32 vdomain_bitmap, int vdomain_idx) +{ + return vdomain_bitmap & BIT(vdomain_idx); +} + +static int update_domain_map(struct kvm_vcpu *vcpu, int vdomain_idx, + int last_pcpu, bool clear_flag) +{ + struct tlbidomain *vdomain = &vcpu->kvm->arch.tlbidomain; + int pdomain = vdomain->domain_map[vdomain_idx]; + cpumask_var_t new_cpus; + + cpumask_copy(new_cpus, vdomain->vdomain_cpumasks[vdomain_idx]); + + if (unlikely(pdomain == -1)) { + cpumask_set_cpu(vcpu->cpu, new_cpus); + goto update_mask; + } + + if (last_pcpu != -1 && clear_flag) { + /* remove old_cpu, not need to do so when first load */ + if (cpumask_test_cpu(last_pcpu, new_cpus)) + cpumask_clear_cpu(last_pcpu, new_cpus); + } + + cpumask_set_cpu(vcpu->cpu, new_cpus); + +update_mask: + cpumask_copy(vdomain->vdomain_cpumasks[vdomain_idx], new_cpus); + return pick_best_domain(vdomain->vdomain_cpumasks[vdomain_idx]); +} + +static void vtlbidn_clear_set_s(int idx, u64 clear, u64 set) +{ + switch (idx) { + case 0: + sysreg_clear_set_s(SYS_VTLBID0_EL2, clear, set); + break; + case 1: + sysreg_clear_set_s(SYS_VTLBID1_EL2, clear, set); + break; + case 2: + sysreg_clear_set_s(SYS_VTLBID2_EL2, clear, set); + break; + case 3: + sysreg_clear_set_s(SYS_VTLBID3_EL2, clear, set); + break; + default: + WARN_ON_ONCE(1); + } +} + +static void kvm_arm_update_tlbid_map(struct tlbidomain *vdomain) +{ + int i, pdomain, vtlbid_idx, offset; + int pdomain_bits, nis = vdomain->nis; + int max_vdomains = 1 << vdomain->nvis; + u64 clear, set; + + if (nis >= NIS_RANGE_MIN_1_8 && nis <= NIS_RANGE_MAX_1_8) + pdomain_bits = VTLBID_TD_WIDTH_NIS_1_8; + else if (nis >= NIS_RANGE_MIN_9_16 && nis <= NIS_RANGE_MAX_9_16) + pdomain_bits = VTLBID_TD_WIDTH_NIS_9_16; + + for (i = 0; i < max_vdomains; i++) { + pdomain = vdomain->domain_map[i]; + if (pdomain == -1) { + if (vdomain->domain_map[0] != -1) + pdomain = vdomain->domain_map[0]; + else + pdomain = 0; + } + + vtlbid_idx = i * pdomain_bits / VTLBID_EL2_WIDTH; + offset = i * pdomain_bits % VTLBID_EL2_WIDTH; + + clear = GENMASK(offset + pdomain_bits - 1, offset); + set = pdomain << offset; + + vtlbidn_clear_set_s(vtlbid_idx, clear, set); + } +} + +static void kvm_vcpu_reload_tlbid(struct kvm *kvm) +{ + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + + if (WARN_ON_ONCE(!vdomain->kvm_tlbid_enabled)) + return; + + preempt_disable(); + kvm_arm_update_tlbid_map(vdomain); + preempt_enable(); +} + +static void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + int max_vdomains = 1 << vdomain->nvis; + DECLARE_BITMAP(clear_bits, KVM_VTLBID_MAX_VDOMAINS); + struct vcpu_tlbid_data *current_data, *other_data; + bool domain_map_changed = false; + int i, old_map; + + /* + * If we support tlbid, but user does not enable it, we can do + * vdomain0 -> pdomainN as well by default. + */ + if (!vdomain->kvm_tlbid_enabled) + return; + + current_data = xa_load(&vdomain->vcpu_data_array, vcpu->vcpu_idx); + if (WARN_ON_ONCE(!current_data)) + return; + + /* + * Check if vCPU thread will move to another pCPU. Even when + * staying on the same pCPU, another vCPU may have reprogrammed + * the per-pCPU VTLBIDn registers in between, so reprogram them + * before re-entering the guest. + */ + if (likely(vcpu->cpu == current_data->last_pcpu)) { + spin_lock(&vdomain->tlbid_lock); + kvm_arm_update_tlbid_map(vdomain); + spin_unlock(&vdomain->tlbid_lock); + return; + } + + spin_lock(&vdomain->tlbid_lock); + + bitmap_zero(clear_bits, max_vdomains); + + /* + * Remove current vCPU from its last_pcpu list and compute clear_bits + * only when there was a previous pCPU. On first load (last_pcpu == -1), + * no previous pCPU needs to be cleared from vdomain_cpumask. + */ + if (current_data->last_pcpu != -1) { + list_del_init(¤t_data->pcpu_node); + + /* + * Iterate only vCPUs that share the same last_pcpu to compute + * clear_bits. This is O(1) in the typical case since most pCPUs + * host only one vCPU at a time. + */ + bitmap_fill(clear_bits, max_vdomains); + list_for_each_entry(other_data, + &vdomain->pcpu_vcpu_list[current_data->last_pcpu], + pcpu_node) { + bitmap_andnot(clear_bits, clear_bits, + (unsigned long *)&other_data->vdomain_bitmap, max_vdomains); + } + } + + if (!vdomain->guest_tlbid_enabled) { + old_map = vdomain->domain_map[0]; + + /* + * If QEMU is not configured with vdomains, only vDomain0 needs + * to be mapped. + */ + vdomain->domain_map[0] = update_domain_map(vcpu, + 0, + current_data->last_pcpu, + test_bit(0, clear_bits)); + if (vdomain->domain_map[0] == old_map) + goto unlock; + } else { + /* + * If QEMU is configured with vdomains, all vdomains need to be + * mapped. For all vdomains which includes this vCPU, check if + * vdomain map should be changed as well. + */ + for (i = 0; i < vdomain->num_domains; i++) { + if (!is_vcpu_in_vdomain(current_data->vdomain_bitmap, i)) + continue; + + old_map = vdomain->domain_map[i]; + vdomain->domain_map[i] = update_domain_map(vcpu, + i, + current_data->last_pcpu, + test_bit(i, clear_bits)); + + if (vdomain->domain_map[i] != old_map) + domain_map_changed = true; + } + + if (!domain_map_changed) + goto unlock; + } + + kvm_flush_remote_tlbs(kvm); + + /* + * Before this vcpu load, kick other vCPUs out, so maps for other vCPUs + * will be updated during vCPU load. + * + * Add KVM_REQUEST_WAIT to make sure vCPU is out. + */ + kvm_make_all_cpus_request(kvm, KVM_REQ_RELOAD_VTLBID | KVM_REQUEST_WAIT); + +unlock: + /* update tlbid hardware map register */ + kvm_arm_update_tlbid_map(vdomain); + + current_data->last_pcpu = vcpu->cpu; + list_add_tail(¤t_data->pcpu_node, + &vdomain->pcpu_vcpu_list[vcpu->cpu]); + + spin_unlock(&vdomain->tlbid_lock); +} + +static void kvm_tlbidomain_vcpu_put(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + + if (!vdomain->kvm_tlbid_enabled) + return; +} +#else +static inline void kvm_vcpu_reload_tlbid(struct kvm *kvm) { } +static inline void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) { } +static inline void kvm_tlbidomain_vcpu_put(struct kvm_vcpu *vcpu) { } +#endif + void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct kvm_s2_mmu *mmu; @@ -900,6 +1128,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_tlbi_dvmbm_vcpu_load(vcpu); + kvm_tlbidomain_vcpu_load(vcpu); + /* * When pv_preempted is changed from enabled to disabled, preempted * state will not be updated in kvm_arch_vcpu_put/load. So we must @@ -938,6 +1168,8 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_tlbi_dvmbm_vcpu_put(vcpu); + kvm_tlbidomain_vcpu_put(vcpu); + if (kvm_arm_is_pvsched_valid(&vcpu->arch) && pv_preempted_enable) kvm_update_pvsched_preempted(vcpu, 1); } @@ -1303,6 +1535,9 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_RELOAD_TLBI_DVMBM, vcpu)) kvm_hisi_reload_lsudvmbm(vcpu->kvm); + if (kvm_check_request(KVM_REQ_RELOAD_VTLBID, vcpu)) + kvm_vcpu_reload_tlbid(vcpu->kvm); + if (kvm_check_request(KVM_REQ_RELOAD_WFI_TRAPS, vcpu)) { if (single_task_running()) vcpu_clear_wfx_traps(vcpu); diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 6e3c26416658..a4e096463666 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -249,6 +249,10 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) #ifdef CONFIG_ENABLE_KVM_FPMR if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) hcrx |= HCRX_EL2_EnFPM; +#endif +#ifdef CONFIG_KVM_ARM_VTLBID + if (kern_hyp_va(vcpu->kvm)->arch.tlbidomain.kvm_tlbid_enabled) + hcrx |= HCRX_EL2_VTLBIDEn; #endif ctxt_sys_reg(hctxt, HCRX_EL2) = read_sysreg_s(SYS_HCRX_EL2); write_sysreg_s(hcrx, SYS_HCRX_EL2); diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 2155315581b2..c10f6985f5db 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3178,7 +3178,8 @@ Fields ZCR_ELx EndSysreg Sysreg HCRX_EL2 3 4 1 2 2 -Res0 63:25 +Res0 63:26 +Field 25 VTLBIDEn Field 24 PACMEn Field 23 EnFPM Field 22 GCSEn @@ -3586,3 +3587,19 @@ Field 12:8 NVIS Res0 7:5 Field 4:0 NIS EndSysreg + +Sysreg VTLBID0_EL2 3 4 2 8 0 +Field 63:0 TD +EndSysreg + +Sysreg VTLBID1_EL2 3 4 2 8 1 +Field 63:0 TD +EndSysreg + +Sysreg VTLBID2_EL2 3 4 2 8 2 +Field 63:0 TD +EndSysreg + +Sysreg VTLBID3_EL2 3 4 2 8 3 +Field 63:0 TD +EndSysreg -- 2.33.0