virt inclusion category: feature bugzilla: https://gitcode.com/openeuler/kernel/issues/10066 ---------------------------------------- In kvm_tlbidomain_vcpu_load(), when a vCPU migrates to a different pCPU, the code iterates over all created vCPUs to find those whose last_pcpu matches the current vCPU last_pcpu. This O(logN) loop with xa_load() becomes a significant bottleneck when the number of vCPUs is large. Introduce a per-pCPU linked list (pcpu_vcpu_list) in struct tlbidomain to maintain reverse mapping from pCPU to vCPUs. Each vcpu_tlbid_data gets a pcpu_node to link into the list of its last_pcpu. This allows the clear_bits calculation to only iterate over vCPUs that actually share the same last_pcpu, reducing the complexity from O(logN) to O(1) in the typical case. Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- arch/arm64/include/asm/kvm_host.h | 4 +++ arch/arm64/kvm/arm.c | 46 ++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 8cea52bd34d2..d96c41719732 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -219,6 +219,8 @@ struct kvm_protected_vm { }; struct vcpu_tlbid_data { + /* Node in per-pCPU vCPU list of tlbidomain */ + struct list_head pcpu_node; /* Store vDomain bitmap, indexed by vCPU ID */ u32 vdomain_bitmap; /* Record the last pCPU where vCPU was running, indexed by vCPU ID */ @@ -244,6 +246,8 @@ struct tlbidomain { u8 nis; /* Used to prevent concurrent modifications to the domain mapping. */ spinlock_t tlbid_lock; + /* Per-pCPU list of vCPUs whose last_pcpu equals this pCPU */ + struct list_head *pcpu_vcpu_list; }; struct kvm_arch { diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index bdeeefe0eea1..3feb8c028268 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -363,6 +363,14 @@ static int kvm_arm_init_tlbidomain(struct kvm *kvm) spin_lock_init(&kvm->arch.vdomain.tlbid_lock); + vdomain->pcpu_vcpu_list = kcalloc(nr_cpu_ids, + sizeof(*vdomain->pcpu_vcpu_list), GFP_KERNEL); + if (!vdomain->pcpu_vcpu_list) + goto free_vdomain_cpumasks; + + for (i = 0; i < nr_cpu_ids; i++) + INIT_LIST_HEAD(&vdomain->pcpu_vcpu_list[i]); + return 0; free_vdomain_cpumasks: @@ -392,6 +400,8 @@ static void free_tlbid_vdomain(struct kvm *kvm) free_cpumask_var(kvm->arch.vdomain.vdomain_cpumasks[i]); kfree(kvm->arch.vdomain.vdomain_cpumasks); + kfree(kvm->arch.vdomain.pcpu_vcpu_list); + xa_for_each(&kvm->arch.vdomain.vcpu_data_array, index, data) { kfree(data); } @@ -971,24 +981,25 @@ static void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) spin_lock(&vcpu->kvm->arch.vdomain.tlbid_lock); /* - * check if another vcpu running on the same pCPU as the one - * the vCPU last ran. If so, do not clear last pCPU from the - * vdomain_cpumask. + * Remove current vCPU from its last_pcpu list and compute clear_bits + * only when there was a previous pCPU. On first load (last_pcpu == -1), + * no previous pCPU needs to be cleared from vdomain_cpumask. */ - bitmap_fill(clear_bits, max_vdomains); - for (i = 0; i < kvm->created_vcpus; i++) { - other_data = xa_load(&vdomain->vcpu_data_array, i); - if (WARN_ON_ONCE(!other_data)) - continue; - - if (i == vcpu->vcpu_idx || other_data->last_pcpu == -1) - continue; + if (current_data->last_pcpu != -1) { + list_del_init(¤t_data->pcpu_node); - if (other_data->last_pcpu != current_data->last_pcpu) - continue; - - bitmap_andnot(clear_bits, clear_bits, - (unsigned long *)&other_data->vdomain_bitmap, max_vdomains); + /* + * Iterate only vCPUs that share the same last_pcpu to compute + * clear_bits. This is O(1) in the typical case since most pCPUs + * host only one vCPU at a time. + */ + bitmap_fill(clear_bits, max_vdomains); + list_for_each_entry(other_data, + &vdomain->pcpu_vcpu_list[current_data->last_pcpu], + pcpu_node) { + bitmap_andnot(clear_bits, clear_bits, + (unsigned long *)&other_data->vdomain_bitmap, max_vdomains); + } } if (!vdomain->guest_tlbid_enabled) { @@ -1031,6 +1042,8 @@ static void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) kvm_arm_update_tlbid_map(vdomain); current_data->last_pcpu = vcpu->cpu; + list_add_tail(¤t_data->pcpu_node, + &vdomain->pcpu_vcpu_list[vcpu->cpu]); spin_unlock(&vcpu->kvm->arch.vdomain.tlbid_lock); } @@ -2112,6 +2125,7 @@ static int kvm_arm_tlbidomain_vcpu_init(struct kvm_vcpu *vcpu) */ data->vdomain_bitmap = 0x1; data->last_pcpu = -1; + INIT_LIST_HEAD(&data->pcpu_node); ret = xa_insert(&vcpu->kvm->arch.vdomain.vcpu_data_array, vcpu->vcpu_idx, data, GFP_KERNEL); -- 2.33.0