From: lishusen <lishusen2@h-partners.com> Replace the per-vCPU sched_cpus/pre_sched_cpus cpumask pair with two simple int fields (now_loaded/pre_loaded) that track the pCPU a vCPU is loaded on. This eliminates per-vCPU cpumask allocation and reduces memory overhead. Signed-off-by: lishusen <lishusen2@h-partners.com> --- arch/arm64/include/asm/kvm_host.h | 5 +-- arch/arm64/kvm/arm.c | 15 +++---- arch/arm64/kvm/hisilicon/hisi_virt.c | 65 ++++++++++------------------ arch/arm64/kvm/hisilicon/hisi_virt.h | 22 +++++++--- 4 files changed, 46 insertions(+), 61 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 240d3db685e2..a8ea5dbe8493 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -745,9 +745,8 @@ struct kvm_vcpu_arch { u32 *ccsidr; #ifdef CONFIG_KVM_HISI_VIRT - /* pCPUs this vCPU can be scheduled on. Pure copy of current->cpus_ptr */ - cpumask_var_t sched_cpus; - cpumask_var_t pre_sched_cpus; + int now_loaded; /* pCPU the vCPU will be loaded on now */ + int pre_loaded; /* pCPU the vCPU was loaded on last time */ #endif KABI_REPLACE(struct virtcca_cvm_tec tec, diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 921a0a9150a1..d8b20c039d70 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -800,6 +800,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_arm_pvspin_vcpu_init(&vcpu->arch); + kvm_sched_affinity_vcpu_init(vcpu); + #ifdef CONFIG_VIRT_VTIMER_PV_STATUS if (!kvm_is_realm(vcpu->kvm)) kvm_arm_pvtimer_status_vcpu_init(&vcpu->arch); @@ -811,13 +813,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) if (err) return err; - err = kvm_sched_affinity_vcpu_init(vcpu); - if (err) - goto vgic_vcpu_destroy; - err = kvm_share_hyp(vcpu, vcpu + 1); if (err) - goto sched_affinity_vcpu_destroy; + goto vgic_vcpu_destroy; if (kvm_is_realm(vcpu->kvm)) { err = kvm_arch_rec_init(&vcpu->arch); @@ -830,9 +828,6 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) unshare_hyp: kvm_unshare_hyp(vcpu, vcpu + 1); -sched_affinity_vcpu_destroy: - kvm_sched_affinity_vcpu_destroy(vcpu); - vgic_vcpu_destroy: kvm_vgic_vcpu_destroy(vcpu); @@ -850,8 +845,6 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_pmu_vcpu_destroy(vcpu); kvm_vgic_vcpu_destroy(vcpu); kvm_arm_vcpu_destroy(vcpu); - - kvm_sched_affinity_vcpu_destroy(vcpu); } void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) @@ -1195,6 +1188,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus)) vcpu_set_on_unsupported_cpu(vcpu); + kvm_sched_affinity_vcpu_load(vcpu); kvm_tlbi_dvmbm_vcpu_load(vcpu); kvm_tlbidomain_vcpu_load(vcpu); @@ -1235,6 +1229,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) vcpu_clear_on_unsupported_cpu(vcpu); + kvm_sched_affinity_vcpu_put(vcpu); kvm_tlbi_dvmbm_vcpu_put(vcpu); kvm_tlbidomain_vcpu_put(vcpu); diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.c b/arch/arm64/kvm/hisilicon/hisi_virt.c index e92661f201a4..b87884e736c7 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.c +++ b/arch/arm64/kvm/hisilicon/hisi_virt.c @@ -264,31 +264,6 @@ bool hisi_dvmbm_supported(void) return true; } -int kvm_sched_affinity_vcpu_init(struct kvm_vcpu *vcpu) -{ - if (!kvm_dvmbm_support) - return 0; - - if (!zalloc_cpumask_var(&vcpu->arch.sched_cpus, GFP_ATOMIC)) - return -ENOMEM; - - if (!zalloc_cpumask_var(&vcpu->arch.pre_sched_cpus, GFP_ATOMIC)) { - free_cpumask_var(vcpu->arch.sched_cpus); - return -ENOMEM; - } - - return 0; -} - -void kvm_sched_affinity_vcpu_destroy(struct kvm_vcpu *vcpu) -{ - if (!kvm_dvmbm_support) - return; - - free_cpumask_var(vcpu->arch.sched_cpus); - free_cpumask_var(vcpu->arch.pre_sched_cpus); -} - static void __kvm_write_lsudvmbm(struct kvm *kvm) { write_sysreg_s(kvm->arch.tlbi_dvmbm, SYS_LSUDVMBM_EL2); @@ -549,12 +524,31 @@ static void kvm_update_vm_lsudvmbm_hip12(struct kvm *kvm) kvm->arch.tlbi_dvmbm = val; } +void kvm_get_vm_loaded_cpu_mask(cpumask_t *mask, struct kvm *kvm) +{ + struct kvm_vcpu *tmp; + unsigned long i; + + cpumask_clear(mask); + kvm_for_each_vcpu(i, tmp, kvm) { + /* + * We may get the stale now_loaded if another thread + * is concurrently changing its running pCPU. It'll + * eventually go through vcpu_load() and we rely on + * the last sched_lock holder to make things correct. + */ + int pcpu = READ_ONCE(tmp->arch.now_loaded); + + if (pcpu < 0) + continue; + cpumask_set_cpu(pcpu, mask); + } +} + void kvm_tlbi_dvmbm_vcpu_load(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; - struct kvm_vcpu *tmp; cpumask_t mask; - unsigned long i; /* Don't bother on old hardware */ if (!kvm_dvmbm_support) @@ -562,10 +556,9 @@ void kvm_tlbi_dvmbm_vcpu_load(struct kvm_vcpu *vcpu) if (kvm_dvmbm_enabled()) hardware_enable_dvmbm(NULL); - cpumask_copy(vcpu->arch.sched_cpus, current->cpus_ptr); - if (likely(cpumask_equal(vcpu->arch.sched_cpus, - vcpu->arch.pre_sched_cpus))) { + if (READ_ONCE(vcpu->arch.now_loaded) == + READ_ONCE(vcpu->arch.pre_loaded)) { kvm_write_lsudvmbm(kvm); return; } @@ -573,16 +566,7 @@ void kvm_tlbi_dvmbm_vcpu_load(struct kvm_vcpu *vcpu) /* Re-calculate sched_cpus for this VM */ spin_lock(&kvm->arch.sched_lock); - cpumask_clear(&mask); - kvm_for_each_vcpu(i, tmp, kvm) { - /* - * We may get the stale sched_cpus if another thread - * is concurrently changing its affinity. It'll - * eventually go through vcpu_load() and we rely on - * the last sched_lock holder to make things correct. - */ - cpumask_or(&mask, &mask, tmp->arch.sched_cpus); - } + kvm_get_vm_loaded_cpu_mask(&mask, kvm); if (cpumask_equal(kvm->arch.sched_cpus, &mask)) goto out_unlock; @@ -611,7 +595,6 @@ void kvm_tlbi_dvmbm_vcpu_put(struct kvm_vcpu *vcpu) if (!kvm_dvmbm_support) return; - cpumask_copy(vcpu->arch.pre_sched_cpus, vcpu->arch.sched_cpus); if (kvm_dvmbm_enabled()) hardware_disable_dvmbm(NULL); } diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.h b/arch/arm64/kvm/hisilicon/hisi_virt.h index 0dfcbd81e1fc..9405bcf364c8 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.h +++ b/arch/arm64/kvm/hisilicon/hisi_virt.h @@ -110,8 +110,18 @@ void ipiv_gicd_init(void); #endif /* CONFIG_ARM64_HISI_IPIV */ void kvm_get_pg_cfg(void); -int kvm_sched_affinity_vcpu_init(struct kvm_vcpu *vcpu); -void kvm_sched_affinity_vcpu_destroy(struct kvm_vcpu *vcpu); +static inline void kvm_sched_affinity_vcpu_init(struct kvm_vcpu *vcpu) +{ + vcpu->arch.now_loaded = vcpu->arch.pre_loaded = -1; +} +static inline void kvm_sched_affinity_vcpu_load(struct kvm_vcpu *vcpu) +{ + vcpu->arch.now_loaded = vcpu->cpu; +} +static inline void kvm_sched_affinity_vcpu_put(struct kvm_vcpu *vcpu) +{ + vcpu->arch.pre_loaded = vcpu->arch.now_loaded; +} int kvm_sched_affinity_vm_init(struct kvm *kvm); void kvm_sched_affinity_vm_destroy(struct kvm *kvm); void kvm_tlbi_dvmbm_vcpu_load(struct kvm_vcpu *vcpu); @@ -150,11 +160,9 @@ static inline void ipiv_gicd_init(void) {} #endif /* CONFIG_ARM64_HISI_IPIV */ static inline void kvm_get_pg_cfg(void) {} -static inline int kvm_sched_affinity_vcpu_init(struct kvm_vcpu *vcpu) -{ - return 0; -} -static inline void kvm_sched_affinity_vcpu_destroy(struct kvm_vcpu *vcpu) {} +static inline void kvm_sched_affinity_vcpu_init(struct kvm_vcpu *vcpu) {} +static inline void kvm_sched_affinity_vcpu_load(struct kvm_vcpu *vcpu) {} +static inline void kvm_sched_affinity_vcpu_put(struct kvm_vcpu *vcpu) {} static inline int kvm_sched_affinity_vm_init(struct kvm *kvm) { return 0; -- 2.33.0