hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10293 ---------------------------------------- Introduce CONFIG_KVM_ARM_DIRECT_TLBI. KVM_REQ_TLB_FLUSH is now consumed on arm64, with each vCPU locally invalidating its stage-2 context on guest entry; invalidation ranges are recorded per vCPU and merged, escalating to a whole-VMID flush when unmergeable. last_ran_cpu tracks the last pCPU each vCPU was loaded on so that a migration landing flushes the stage-2 context left behind by a request consumed elsewhere. Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- arch/arm64/include/asm/kvm_host.h | 11 +++ arch/arm64/include/asm/kvm_tlbi.h | 126 +++++++++++++++++++++++++++ arch/arm64/kvm/Kconfig | 20 +++++ arch/arm64/kvm/arm.c | 67 +++++++++++++- arch/arm64/kvm/hisilicon/hisi_virt.c | 6 +- 5 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/kvm_tlbi.h diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 240d3db685e2..17ee1135109e 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -278,6 +278,8 @@ struct kvm_arch { /* Protects VM-scoped configuration data */ struct mutex config_lock; + bool tlbi_directed; + /* * If we encounter a data abort without valid instruction syndrome * information, report this to user space. User space can (and @@ -770,6 +772,15 @@ struct kvm_vcpu_arch { u64 prod_el2; } hdbss) #endif + +#ifdef CONFIG_KVM_ARM_DIRECT_TLBI + int last_ran_cpu; + + spinlock_t tlbi_pending_lock; + u64 tlbi_pending_start; + u64 tlbi_pending_size; + bool tlbi_pending; +#endif }; /* diff --git a/arch/arm64/include/asm/kvm_tlbi.h b/arch/arm64/include/asm/kvm_tlbi.h new file mode 100644 index 000000000000..7d5a046b6217 --- /dev/null +++ b/arch/arm64/include/asm/kvm_tlbi.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARM64_KVM_TLBI_H__ +#define __ARM64_KVM_TLBI_H__ + +#include <linux/kvm_host.h> + +extern bool kvm_arm_direct_tlbi_enabled; + +#if defined(__KVM_NVHE_HYPERVISOR__) +static __always_inline void kvm_tlbi_request_flush_range(struct kvm *kvm, + u64 start, u64 size) +{ +} +static __always_inline void kvm_tlbi_request_flush(struct kvm *kvm) +{ + kvm_tlbi_request_flush_range(kvm, 0, 0); +} +#else + +#include <asm/virt.h> + +static __always_inline bool kvm_tlbi_direct_enabled(void) +{ + return IS_ENABLED(CONFIG_KVM_ARM_DIRECT_TLBI) && + READ_ONCE(kvm_arm_direct_tlbi_enabled) && + is_kernel_in_hyp_mode(); +} + +/* + * Merge a pending invalidation into every vCPU's per-vCPU pending + * range, consumed on guest entry. Ranges that can't be merged + * escalate to a whole-VMID flush (size 0). + */ +static __always_inline void kvm_tlbi_mark_pending(struct kvm *kvm, + u64 start, u64 size) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + kvm_for_each_vcpu(i, vcpu, kvm) { + unsigned long flags; + u64 pend_end, new_end; + + spin_lock_irqsave(&vcpu->arch.tlbi_pending_lock, flags); + if (!vcpu->arch.tlbi_pending) { + vcpu->arch.tlbi_pending = true; + vcpu->arch.tlbi_pending_start = start; + vcpu->arch.tlbi_pending_size = size; + } else if (!size) { + vcpu->arch.tlbi_pending_size = 0; + } else if (vcpu->arch.tlbi_pending_size) { + pend_end = vcpu->arch.tlbi_pending_start + + vcpu->arch.tlbi_pending_size; + new_end = start + size; + + if (start <= pend_end && + new_end >= vcpu->arch.tlbi_pending_start) { + vcpu->arch.tlbi_pending_start = + min(vcpu->arch.tlbi_pending_start, start); + vcpu->arch.tlbi_pending_size = + max(pend_end, new_end) - + vcpu->arch.tlbi_pending_start; + } else { + vcpu->arch.tlbi_pending_size = 0; + } + } + spin_unlock_irqrestore(&vcpu->arch.tlbi_pending_lock, flags); + } +} + +/* Extract and clear the pending invalidation of this vCPU. */ +static __always_inline void kvm_tlbi_take_pending(struct kvm_vcpu *vcpu, + u64 *start, u64 *size) +{ + unsigned long flags; + + spin_lock_irqsave(&vcpu->arch.tlbi_pending_lock, flags); + if (vcpu->arch.tlbi_pending) { + *start = vcpu->arch.tlbi_pending_start; + *size = vcpu->arch.tlbi_pending_size; + vcpu->arch.tlbi_pending = false; + vcpu->arch.tlbi_pending_size = 0; + } else { + *start = 0; + *size = 0; + } + spin_unlock_irqrestore(&vcpu->arch.tlbi_pending_lock, flags); +} + +static __always_inline void kvm_tlbi_request_flush_range(struct kvm *kvm, + u64 start, u64 size) +{ + if (!kvm_tlbi_direct_enabled()) + return; + + smp_store_release(&kvm->arch.tlbi_directed, true); + kvm_tlbi_mark_pending(kvm, start, size); + kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH); +} + +static __always_inline void kvm_tlbi_request_flush(struct kvm *kvm) +{ + kvm_tlbi_request_flush_range(kvm, 0, 0); +} + +void __kvm_tlb_flush_vmid_nsh(struct kvm_s2_mmu *mmu); +void __kvm_tlb_flush_vmid_range_nsh(struct kvm_s2_mmu *mmu, + phys_addr_t start, unsigned long pages); + +static __always_inline void kvm_tlbi_flush_vmid_nsh(struct kvm_s2_mmu *mmu) +{ + __kvm_tlb_flush_vmid_nsh(mmu); + isb(); +} + +static __always_inline void kvm_tlbi_flush_vmid_range_nsh(struct kvm_s2_mmu *mmu, + phys_addr_t start, + unsigned long pages) +{ + __kvm_tlb_flush_vmid_range_nsh(mmu, start, pages); + isb(); +} + +#endif /* !__KVM_NVHE_HYPERVISOR__ */ + +#endif /* __ARM64_KVM_TLBI_H__ */ diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 2a71109ce082..8534dadcb257 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -157,4 +157,24 @@ config KVM_ARM_VTLBID If unsure, say N. +config KVM_ARM_DIRECT_TLBI + bool "Direct stage-2 TLB invalidation for KVM (EXPERIMENTAL)" + depends on KVM + default n + help + Replace KVM's broadcast stage-2 TLB invalidation with a + directed one: instead of broadcasting a TLBI over the whole + inner-shareable domain, KVM_REQ_TLB_FLUSH requests are sent + to the vCPUs of the VM, each of which locally invalidates its + stage-2 context before re-entering the guest. vCPUs are also + tracked per-pCPU so that migration leaves no stale entry + behind. This is profitable for small VMs running on big + machines, e.g. a 2-vCPU VM on a 128-CPU system. + + The behaviour is controlled at runtime via the + /sys/module/kvm/parameters/direct_tlbi knob and can be + toggled without rebooting: each flush is self-contained. + + If unsure, say N. + endif # VIRTUALIZATION diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index d6156bf0db08..feeba354c377 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -40,6 +40,7 @@ #include <asm/kvm_nested.h> #include <asm/kvm_pkvm.h> #include <asm/kvm_emulate.h> +#include <asm/kvm_tlbi.h> #include <asm/kvm_rme.h> #include <asm/sections.h> #include <asm/kvm_tmi.h> @@ -73,6 +74,30 @@ bool kvm_ncsnp_support; /* Capability of DVMBM */ bool kvm_dvmbm_support; +#ifdef CONFIG_KVM_ARM_DIRECT_TLBI +bool kvm_arm_direct_tlbi_enabled; + +static int direct_tlbi_set(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_bool(val, kp); + + if (!ret && kvm_arm_direct_tlbi_enabled && !is_kernel_in_hyp_mode()) { + kvm_arm_direct_tlbi_enabled = false; + kvm_err("direct_tlbi requires VHE\n"); + return -EINVAL; + } + + return ret; +} + +static const struct kernel_param_ops direct_tlbi_ops = { + .set = direct_tlbi_set, + .get = param_get_bool, +}; + +module_param_cb(direct_tlbi, &direct_tlbi_ops, &kvm_arm_direct_tlbi_enabled, 0644); +#endif + static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized); bool is_kvm_arm_initialised(void) @@ -804,6 +829,10 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; +#ifdef CONFIG_KVM_ARM_DIRECT_TLBI + vcpu->arch.last_ran_cpu = -1; +#endif + err = kvm_vgic_vcpu_init(vcpu); if (err) return err; @@ -1092,7 +1121,16 @@ static void kvm_tlbidomain_vcpu_load(struct kvm_vcpu *vcpu) goto unlock; } - kvm_flush_remote_tlbs(kvm); + /* + * Don't go through the directed TLBI request path here: this + * runs under tlbid_lock, and synchronously waiting for vCPUs + * (KVM_REQUEST_WAIT) while holding a spinlock risks + * soft-lockups. Stick to the hardware broadcast. + */ + if (kvm_tlbi_direct_enabled()) + kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); + else + kvm_flush_remote_tlbs(kvm); /* * Before this vcpu load, kick other vCPUs out, so maps for other vCPUs @@ -1161,6 +1199,13 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) *last_ran = vcpu->vcpu_id; } +#ifdef CONFIG_KVM_ARM_DIRECT_TLBI + if (smp_load_acquire(&vcpu->kvm->arch.tlbi_directed) && + !vcpu_is_rec(vcpu) && vcpu->arch.last_ran_cpu != cpu) + kvm_tlbi_flush_vmid_nsh(mmu); + vcpu->arch.last_ran_cpu = cpu; +#endif + vcpu->cpu = cpu; kvm_vgic_load(vcpu); @@ -1217,6 +1262,10 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_timer_vcpu_put(vcpu); kvm_vgic_put(vcpu); +#ifdef CONFIG_KVM_ARM_DIRECT_TLBI + vcpu->arch.last_ran_cpu = vcpu->cpu; +#endif + vcpu->cpu = -1; if (vcpu_is_rec(vcpu)) @@ -1563,6 +1612,22 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; +#ifdef CONFIG_KVM_ARM_DIRECT_TLBI + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { + u64 start, size; + + kvm_tlbi_take_pending(vcpu, &start, &size); + + preempt_disable(); + if (size) + kvm_tlbi_flush_vmid_range_nsh(vcpu->arch.hw_mmu, + start, size); + else + kvm_tlbi_flush_vmid_nsh(vcpu->arch.hw_mmu); + preempt_enable(); + } +#endif + if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) kvm_vcpu_sleep(vcpu); diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.c b/arch/arm64/kvm/hisilicon/hisi_virt.c index 0be4ebbc540d..663e9649ffa7 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.c +++ b/arch/arm64/kvm/hisilicon/hisi_virt.c @@ -7,6 +7,7 @@ #include <linux/of.h> #include <linux/init.h> #include <linux/kvm_host.h> +#include <asm/kvm_tlbi.h> #include "hisi_virt.h" #include <linux/bitfield.h> @@ -586,7 +587,10 @@ void kvm_tlbi_dvmbm_vcpu_load(struct kvm_vcpu *vcpu) cpumask_copy(kvm->arch.sched_cpus, &mask); - kvm_flush_remote_tlbs(kvm); + if (kvm_tlbi_direct_enabled()) + kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); + else + kvm_flush_remote_tlbs(kvm); /* * Re-calculate LSUDVMBM_EL2 for this VM and kick all vcpus -- 2.33.0