virt inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10066 ---------------------------------------- Add the VTLBID0_EL2 ~ VTLBID3_EL2 sysreg definitions and the HCRX_EL2.VTLBIDEn control field to arch/arm64/tools/sysreg. Implement kvm_tlbidomain_vcpu_load() / kvm_tlbidomain_vcpu_put() to update the VTLBIDn registers on vCPU schedule-in and tear down the hardware context on schedule-out. On first load or migration, the per-vCPU domain map is recomputed and written to VTLBIDn. Handle the KVM_REQ_RELOAD_VTLBID request so that other vCPUs reprogram the VTLBIDn registers after a domain map change. Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> Signed-off-by: Zhou Wang <wangzhou1@hisilicon.com> --- arch/arm64/kvm/arm.c | 240 ++++++++++++++++++++++++++++++++++++++++ arch/arm64/tools/sysreg | 19 +++- 2 files changed, 258 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 565b26248d4e..493769b4c436 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -845,6 +845,239 @@ static void update_steal_time(struct kvm_vcpu *vcpu) } #endif +#ifdef CONFIG_KVM_ARM_VTLBID +static bool is_vcpu_in_vdomain(u32 vdomain_bitmap, int vdomain_idx) +{ + return vdomain_bitmap & BIT(vdomain_idx); +} + +static int update_domain_map(struct kvm_vcpu *vcpu, int vdomain_idx, + int last_pcpu, bool clear_flag) +{ + struct tlbidomain *vdomain = &vcpu->kvm->arch.tlbidomain; + int pdomain = vdomain->domain_map[vdomain_idx]; + cpumask_var_t new_cpus; + + cpumask_copy(new_cpus, vdomain->vdomain_cpumasks[vdomain_idx]); + + if (unlikely(pdomain == -1)) { + cpumask_set_cpu(vcpu->cpu, new_cpus); + goto update_mask; + } + + if (last_pcpu != -1 && clear_flag) { + /* remove old_cpu, not need to do so when first load */ + if (cpumask_test_cpu(last_pcpu, new_cpus)) + cpumask_clear_cpu(last_pcpu, new_cpus); + } + + cpumask_set_cpu(vcpu->cpu, new_cpus); + +update_mask: + cpumask_copy(vdomain->vdomain_cpumasks[vdomain_idx], new_cpus); + return pick_best_domain(vdomain->vdomain_cpumasks[vdomain_idx]); +} + +static void vtlbidn_clear_set_s(int idx, u64 clear, u64 set) +{ + switch (idx) { + case 0: + sysreg_clear_set_s(SYS_VTLBID0_EL2, clear, set); + break; + case 1: + sysreg_clear_set_s(SYS_VTLBID1_EL2, clear, set); + break; + case 2: + sysreg_clear_set_s(SYS_VTLBID2_EL2, clear, set); + break; + case 3: + sysreg_clear_set_s(SYS_VTLBID3_EL2, clear, set); + break; + default: + WARN_ON_ONCE(1); + } +} + +static void kvm_arm_update_tlbid_map(struct tlbidomain *vdomain) +{ + int i, pdomain, vtlbid_idx, offset; + int pdomain_bits, nis = vdomain->nis; + int max_vdomains = 1 << vdomain->nvis; + u64 clear, set; + + if (nis >= NIS_RANGE_MIN_1_8 && nis <= NIS_RANGE_MAX_1_8) + pdomain_bits = VTLBID_TD_WIDTH_NIS_1_8; + else if (nis >= NIS_RANGE_MIN_9_16 && nis <= NIS_RANGE_MAX_9_16) + pdomain_bits = VTLBID_TD_WIDTH_NIS_9_16; + + for (i = 0; i < max_vdomains; i++) { + pdomain = vdomain->domain_map[i]; + if (pdomain == -1) { + if (vdomain->domain_map[0] != -1) + pdomain = vdomain->domain_map[0]; + else + pdomain = 0; + } + + vtlbid_idx = i * pdomain_bits / VTLBID_EL2_WIDTH; + offset = i * pdomain_bits % VTLBID_EL2_WIDTH; + + clear = GENMASK(offset + pdomain_bits - 1, offset); + set = pdomain << offset; + + vtlbidn_clear_set_s(vtlbid_idx, clear, set); + } + + sysreg_clear_set_s(SYS_HCRX_EL2, 0, HCRX_EL2_VTLBIDEn); +} + +static void kvm_vcpu_reload_tlbid(struct kvm *kvm) +{ + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + + if (WARN_ON_ONCE(!vdomain->kvm_tlbid_enabled)) + return; + + preempt_disable(); + kvm_arm_update_tlbid_map(vdomain); + preempt_enable(); +} + +static void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + int max_vdomains = 1 << vdomain->nvis; + DECLARE_BITMAP(clear_bits, KVM_VTLBID_MAX_VDOMAINS); + struct vcpu_tlbid_data *current_data, *other_data; + bool domain_map_changed = false; + int i, old_map; + + /* + * If we support tlbid, but user does not enable it, we can do + * vdomain0 -> pdomainN as well by default. + */ + if (!vdomain->kvm_tlbid_enabled) + return; + + current_data = xa_load(&vdomain->vcpu_data_array, vcpu->vcpu_idx); + if (WARN_ON_ONCE(!current_data)) + return; + + /* + * Check if vCPU thread will move to another pCPU. Even when + * staying on the same pCPU, kvm_tlbidomain_vcpu_put() has + * cleared HCRX_EL2.VTLBIDEn, and another vCPU may have + * reprogrammed the per-pCPU VTLBIDn registers in between, so + * reprogram them before re-entering the guest. + */ + if (likely(vcpu->cpu == current_data->last_pcpu)) { + spin_lock(&vdomain->tlbid_lock); + kvm_arm_update_tlbid_map(vdomain); + spin_unlock(&vdomain->tlbid_lock); + return; + } + + spin_lock(&vdomain->tlbid_lock); + + bitmap_zero(clear_bits, max_vdomains); + + /* + * Remove current vCPU from its last_pcpu list and compute clear_bits + * only when there was a previous pCPU. On first load (last_pcpu == -1), + * no previous pCPU needs to be cleared from vdomain_cpumask. + */ + if (current_data->last_pcpu != -1) { + list_del_init(¤t_data->pcpu_node); + + /* + * Iterate only vCPUs that share the same last_pcpu to compute + * clear_bits. This is O(1) in the typical case since most pCPUs + * host only one vCPU at a time. + */ + bitmap_fill(clear_bits, max_vdomains); + list_for_each_entry(other_data, + &vdomain->pcpu_vcpu_list[current_data->last_pcpu], + pcpu_node) { + bitmap_andnot(clear_bits, clear_bits, + (unsigned long *)&other_data->vdomain_bitmap, max_vdomains); + } + } + + if (!vdomain->guest_tlbid_enabled) { + old_map = vdomain->domain_map[0]; + + /* + * If QEMU is not configured with vdomains, only vDomain0 needs + * to be mapped. + */ + vdomain->domain_map[0] = update_domain_map(vcpu, + 0, + current_data->last_pcpu, + test_bit(0, clear_bits)); + if (vdomain->domain_map[0] == old_map) + goto unlock; + } else { + /* + * If QEMU is configured with vdomains, all vdomains need to be + * mapped. For all vdomains which includes this vCPU, check if + * vdomain map should be changed as well. + */ + for (i = 0; i < vdomain->num_domains; i++) { + if (!is_vcpu_in_vdomain(current_data->vdomain_bitmap, i)) + continue; + + old_map = vdomain->domain_map[i]; + vdomain->domain_map[i] = update_domain_map(vcpu, + i, + current_data->last_pcpu, + test_bit(i, clear_bits)); + + if (vdomain->domain_map[i] != old_map) + domain_map_changed = true; + } + + if (!domain_map_changed) + goto unlock; + } + + kvm_flush_remote_tlbs(kvm); + + /* + * Before this vcpu load, kick other vCPUs out, so maps for other vCPUs + * will be updated during vCPU load. + * + * Add KVM_REQUEST_WAIT to make sure vCPU is out. + */ + kvm_make_all_cpus_request(kvm, KVM_REQ_RELOAD_VTLBID | KVM_REQUEST_WAIT); + +unlock: + /* update tlbid hardware map register */ + kvm_arm_update_tlbid_map(vdomain); + + current_data->last_pcpu = vcpu->cpu; + list_add_tail(¤t_data->pcpu_node, + &vdomain->pcpu_vcpu_list[vcpu->cpu]); + + spin_unlock(&vdomain->tlbid_lock); +} + +static void kvm_tlbidomain_vcpu_put(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + struct tlbidomain *vdomain = &kvm->arch.tlbidomain; + + if (!vdomain->kvm_tlbid_enabled) + return; + + sysreg_clear_set_s(SYS_HCRX_EL2, HCRX_EL2_VTLBIDEn, 0); +} +#else +static inline void kvm_vcpu_reload_tlbid(struct kvm *kvm) { } +static inline void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) { } +static inline void kvm_tlbidomain_vcpu_put(struct kvm_vcpu *vcpu) { } +#endif + void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct kvm_s2_mmu *mmu; @@ -900,6 +1133,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_tlbi_dvmbm_vcpu_load(vcpu); + kvm_tlbidomain_vcpu_load(vcpu); + /* * When pv_preempted is changed from enabled to disabled, preempted * state will not be updated in kvm_arch_vcpu_put/load. So we must @@ -938,6 +1173,8 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_tlbi_dvmbm_vcpu_put(vcpu); + kvm_tlbidomain_vcpu_put(vcpu); + if (kvm_arm_is_pvsched_valid(&vcpu->arch) && pv_preempted_enable) kvm_update_pvsched_preempted(vcpu, 1); } @@ -1303,6 +1540,9 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_RELOAD_TLBI_DVMBM, vcpu)) kvm_hisi_reload_lsudvmbm(vcpu->kvm); + if (kvm_check_request(KVM_REQ_RELOAD_VTLBID, vcpu)) + kvm_vcpu_reload_tlbid(vcpu->kvm); + if (kvm_check_request(KVM_REQ_RELOAD_WFI_TRAPS, vcpu)) { if (single_task_running()) vcpu_clear_wfx_traps(vcpu); diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 2155315581b2..c10f6985f5db 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3178,7 +3178,8 @@ Fields ZCR_ELx EndSysreg Sysreg HCRX_EL2 3 4 1 2 2 -Res0 63:25 +Res0 63:26 +Field 25 VTLBIDEn Field 24 PACMEn Field 23 EnFPM Field 22 GCSEn @@ -3586,3 +3587,19 @@ Field 12:8 NVIS Res0 7:5 Field 4:0 NIS EndSysreg + +Sysreg VTLBID0_EL2 3 4 2 8 0 +Field 63:0 TD +EndSysreg + +Sysreg VTLBID1_EL2 3 4 2 8 1 +Field 63:0 TD +EndSysreg + +Sysreg VTLBID2_EL2 3 4 2 8 2 +Field 63:0 TD +EndSysreg + +Sysreg VTLBID3_EL2 3 4 2 8 3 +Field 63:0 TD +EndSysreg -- 2.33.0