hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10293 ---------------------------------------- Route the stage-2 walker flushes (break-before-make, unmap and the deferred range flush) through the directed machinery: the walkers request the exact IPA or table range and each vCPU invalidates it locally on guest entry. Realm VMs keep the broadcast, as does nVHE. The full-VMID broadcast fallback clears kvm_arch::tlbi_directed, while range and IPA-scoped broadcasts leave it set. Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- arch/arm64/kvm/hyp/pgtable.c | 50 +++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 9f4e4046c0b5..9bdbbc5ef58a 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -8,9 +8,16 @@ */ #include <linux/bitfield.h> +#include <asm/kvm_emulate.h> +#include <asm/kvm_mmu.h> #include <asm/kvm_pgtable.h> +#include <asm/kvm_tlbi.h> #include <asm/stage2_pgtable.h> +#if !defined(__KVM_NVHE_HYPERVISOR__) +#include "../trace_arm.h" +#endif + #define KVM_PTE_TYPE BIT(1) #define KVM_PTE_TYPE_BLOCK 0 @@ -672,13 +679,44 @@ static bool stage2_has_fwb(struct kvm_pgtable *pgt) return !(pgt->flags & KVM_PGTABLE_S2_NOFWB); } +/* + * Directed TLBI routes the invalidation to KVM_REQ_TLB_FLUSH; nVHE + * and every fallback below keep the hardware broadcast. + */ +static __always_inline void kvm_tlbi_flush_vmid_ipa(struct kvm_s2_mmu *mmu, + phys_addr_t ipa, int level) +{ +#if !defined(__KVM_NVHE_HYPERVISOR__) + if (kvm_tlbi_direct_enabled() && !is_protected_kvm_enabled() && + !_kvm_is_realm(kvm_s2_mmu_to_kvm(mmu)) && + in_task() && !irqs_disabled()) { + kvm_tlbi_request_flush_range(kvm_s2_mmu_to_kvm(mmu), ipa, + kvm_granule_size(level)); + return; + } +#endif + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ipa, level); +} + void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, size_t size) { unsigned long pages = size >> PAGE_SHIFT; +#if !defined(__KVM_NVHE_HYPERVISOR__) + if (kvm_tlbi_direct_enabled() && !is_protected_kvm_enabled() && + !_kvm_is_realm(kvm_s2_mmu_to_kvm(mmu)) && + in_task() && !irqs_disabled()) { + kvm_tlbi_request_flush_range(kvm_s2_mmu_to_kvm(mmu), addr, size); + return; + } +#endif + if (!system_supports_tlb_range() || pages > MAX_TLBI_RANGE_PAGES) { kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); + /* Full VMID broadcast: the directed state is gone. */ + smp_store_release(&kvm_s2_mmu_to_kvm(mmu)->arch.tlbi_directed, + false); return; } kvm_call_hyp(__kvm_tlb_flush_vmid_range, mmu, addr, pages); @@ -824,8 +862,7 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_tlb_flush_vmid_range(mmu, addr, size); } else if (kvm_pte_valid(ctx->old)) { - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, - ctx->addr, ctx->level); + kvm_tlbi_flush_vmid_ipa(mmu, ctx->addr, ctx->level); } } @@ -875,11 +912,12 @@ static void stage2_unmap_put_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_clear_pte(ctx->ptep); if (kvm_pte_table(ctx->old, ctx->level)) { - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, - 0); + u64 size = kvm_granule_size(ctx->level); + + kvm_tlb_flush_vmid_range(mmu, ALIGN_DOWN(ctx->addr, size), + size); } else if (!stage2_unmap_defer_tlb_flush(pgt)) { - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, - ctx->level); + kvm_tlbi_flush_vmid_ipa(mmu, ctx->addr, ctx->level); } } -- 2.33.0