[PATCH OLK-6.6 0/2] *** smmu support btm ***
*** BLURB HERE *** Zhang Yuwei (2): [Backport] iommu/arm-smmu-v3: Bring back SVA BTM support [Huawei] iommu/arm-smmu-v3: support disable BTM and fix BTM interaction drivers/iommu/Kconfig | 7 + .../arm/arm-smmu-v3/arm-smmu-v3-iommufd.c | 8 +- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 152 ++++++++++++++++-- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 16 +- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 112 ++++++++----- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 11 +- 6 files changed, 235 insertions(+), 71 deletions(-) -- 2.22.0
maillist inclusion category: feature bugzilla: NA Reference: https://lore.kernel.org/linux-iommu/26-v6-228e7adf25eb+4155-smmuv3_newapi_p2... ------------------ BTM support is a feature where the CPU TLB invalidation can be forwarded to the IOMMU and also invalidate the IOTLB. For this to work the CPU and IOMMU ASID must be the same. Retain the prior SVA design here of keeping the ASID allocator for the IOMMU private to SMMU and force SVA domains to set an ASID that matches the CPU ASID. Adaptation notes for OLK 6.6: - arm_smmu_make_s1_cd() has no TTBR1 variant here; add the asid_lock lockdep assertion directly. - arm_smmu_tlb_inv_context() has no E2H variant here; extend the S1 branch to SVA domains and use READ_ONCE on cd.asid. - Keep the CONFIG_HISI_CCADA_HOST realm_smmu_domain_clear() call in arm_smmu_domain_free_paging(). - Keep the devm-protected vmid_map teardown in arm_smmu_init_strtab(). Note: the stray mutex_unlock() in arm_smmu_attach_prepare() (present in the source commit of this backport, upstream has lockdep_assert_held() there) is fixed by the follow-up commit. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- .../arm/arm-smmu-v3/arm-smmu-v3-iommufd.c | 8 +- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 152 ++++++++++++++++-- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 16 +- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 101 +++++++----- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 11 +- 5 files changed, 218 insertions(+), 70 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index 06995ee563d5..5c6ce51d4885 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -107,13 +107,15 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, }; struct arm_smmu_ste ste; int ret; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; if (nested_domain->vsmmu->smmu != master->smmu) return -EINVAL; if (arm_smmu_ssids_in_use(&master->cd_table)) return -EBUSY; - mutex_lock(&arm_smmu_asid_lock); + mutex_lock(&smmu->asid_lock); /* * The VM has to control the actual ATS state at the PCI device because * we forward the invalidations directly from the VM. If the VM doesn't @@ -125,7 +127,7 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, state.disable_ats = !nested_domain->enable_ats; ret = arm_smmu_attach_prepare(&state, domain); if (ret) { - mutex_unlock(&arm_smmu_asid_lock); + mutex_unlock(&smmu->asid_lock); return ret; } @@ -133,7 +135,7 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, state.ats_enabled); arm_smmu_install_ste_for_dev(master, &ste); arm_smmu_attach_commit(&state); - mutex_unlock(&arm_smmu_asid_lock); + mutex_unlock(&smmu->asid_lock); return 0; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 4075ef00c4c9..060d01da16bd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -15,12 +15,33 @@ static DEFINE_MUTEX(sva_lock); -static void __maybe_unused -arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) +static int arm_smmu_realloc_s1_domain_asid(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain) { struct arm_smmu_master_domain *master_domain; + u32 old_asid = smmu_domain->cd.asid; struct arm_smmu_cd target_cd; unsigned long flags; + int ret; + + lockdep_assert_held(&smmu->asid_lock); + + /* + * FIXME: The unmap and invalidation path doesn't take any locks but + * this is not fully safe. Since updating the CD tables is not atomic + * there is always a hole where invalidating only one ASID of two active + * ASIDs during unmap will cause the IOTLB to become stale. + * + * This approach is to hopefully shift the racing CPUs to the new ASID + * before we start programming the HW. This increases the chance that + * racing IOPTE changes will pick up an invalidation for the new ASID + * and we achieve eventual consistency. For the brief period where the + * old ASID is still in the CD entries it will become incoherent. + */ + ret = xa_alloc(&smmu->asid_map, &smmu_domain->cd.asid, smmu_domain, + XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); + if (ret) + return ret; spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) { @@ -36,6 +57,10 @@ arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) &target_cd); } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + + /* Clean the ASID we are about to assign to a new translation */ + arm_smmu_tlb_inv_asid(smmu, old_asid); + return 0; } static u64 page_size_to_cd(void) @@ -52,7 +77,7 @@ static u64 page_size_to_cd(void) VISIBLE_IF_KUNIT void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, struct arm_smmu_master *master, struct mm_struct *mm, - u16 asid) + u16 asid, bool btm_invalidation) { u64 par; @@ -73,7 +98,7 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, (master->stall_enabled ? CTXDESC_CD_0_S : 0) | CTXDESC_CD_0_R | CTXDESC_CD_0_A | - CTXDESC_CD_0_ASET | + (btm_invalidation ? 0 : CTXDESC_CD_0_ASET) | FIELD_PREP(CTXDESC_CD_0_ASID, asid)); if (master->smmu->features & ARM_SMMU_FEAT_HD) @@ -152,11 +177,15 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, size = 0; } - if (!size) - arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); - else - arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid, - PAGE_SIZE, false, smmu_domain); + if (!smmu_domain->btm_invalidation) { + ioasid_t asid = READ_ONCE(smmu_domain->cd.asid); + + if (!size) + arm_smmu_tlb_inv_asid(smmu_domain->smmu, asid); + else + arm_smmu_tlb_inv_range_asid(start, size, asid, + PAGE_SIZE, false, smmu_domain); + } arm_smmu_atc_inv_domain(smmu_domain, start, size); } @@ -182,8 +211,10 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); if (WARN_ON(!cdptr)) continue; + /* An SVA ASID never changes, no asid_lock required */ arm_smmu_make_sva_cd(&target, master, NULL, - smmu_domain->cd.asid); + smmu_domain->cd.asid, + smmu_domain->btm_invalidation); arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, &target); } @@ -352,7 +383,8 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, * This does not need the arm_smmu_asid_lock because SVA domains never * get reassigned */ - arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid); + arm_smmu_make_sva_cd(&target, master, domain->mm, + smmu_domain->cd.asid, smmu_domain->btm_invalidation); ret = arm_smmu_set_pasid(master, smmu_domain, id, &target); mmput(domain->mm); @@ -374,7 +406,9 @@ static void arm_smmu_sva_domain_free(struct iommu_domain *domain) * reused, and if there is a race then it just suffers harmless * unnecessary invalidation. */ - xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); + arm_smmu_domain_free_id(smmu_domain); + if (smmu_domain->btm_invalidation) + arm64_mm_context_put(domain->mm); /* * Actual free is defered to the SRCU callback @@ -388,13 +422,97 @@ static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { .free = arm_smmu_sva_domain_free }; +static int arm_smmu_share_asid(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, + struct mm_struct *mm) +{ + struct arm_smmu_domain *old_s1_domain; + int ret; + + /* + * Notice that BTM is never currently enabled, this is all dead code. + * The specification cautions: + * + * Note: Arm expects that SMMU stage 2 address spaces are generally + * shared with their respective PE virtual machine stage 2 + * configuration. If broadcast invalidation is required to be avoided + * for a particular SMMU stage 2 address space, Arm recommends that a + * hypervisor configures the STE with a VMID that is not allocated for + * virtual machine use on the PEs + * + * However, in Linux, both KVM and SMMU think they own the VMID pool. + * Unfortunately the ARM design is problematic for Linux as we do not + * currently share the S2 table with KVM. This creates a situation where + * the S2 needs to have the same VMID as KVM in order to allow the guest + * to use BTM, however we must still invalidate the S2 directly since it + * is a different radix tree. What Linux would like is something like + * ASET for the STE to disable BTM only for the S2. + * + * Arguably in a system with BTM the driver should prefer to use a S1 + * table in all cases execpt when explicitly asked to create a nesting + * parent. Then it should use the VMID of KVM to enable BTM in the + * guest. We cannot optimize away the resulting double invalidation of + * the S2 :( Or we simply ignore BTM entirely as we are doing now. + */ + if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) + return xa_alloc(&smmu->asid_map, &smmu_domain->cd.asid, + smmu_domain, + XA_LIMIT(1, (1 << smmu->asid_bits) - 1), + GFP_KERNEL); + + /* At this point the caller ensures we have a mmget() */ + smmu_domain->cd.asid = arm64_mm_context_get(mm); + + mutex_lock(&smmu->asid_lock); + old_s1_domain = xa_store(&smmu->asid_map, smmu_domain->cd.asid, + smmu_domain, GFP_KERNEL); + if (xa_err(old_s1_domain)) { + ret = xa_err(old_s1_domain); + goto out_put_asid; + } + + /* + * In BTM mode the CPU ASID and the IOMMU ASID have to be the same. + * Unfortunately we run separate allocators for this and the IOMMU + * ASID can already have been assigned to a S1 domain. SVA domains + * always align to their CPU ASIDs. In this case we change + * the S1 domain's ASID, update the CD entry and flush the caches. + * + * This is a bit tricky, all the places writing to a S1 CD, reading the + * S1 ASID, or doing xa_erase must hold the asid_lock or xa_lock to + * avoid IOTLB incoherence. + */ + if (old_s1_domain) { + if (WARN_ON(old_s1_domain->domain.type == IOMMU_DOMAIN_SVA)) { + ret = -EINVAL; + goto out_restore_s1; + } + ret = arm_smmu_realloc_s1_domain_asid(smmu, old_s1_domain); + if (ret) + goto out_restore_s1; + } + + smmu_domain->btm_invalidation = true; + + ret = 0; + goto out_unlock; + +out_restore_s1: + xa_store(&smmu->asid_map, smmu_domain->cd.asid, old_s1_domain, + GFP_KERNEL); +out_put_asid: + arm64_mm_context_put(mm); +out_unlock: + mutex_unlock(&smmu->asid_lock); + return ret; +} + struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_domain *smmu_domain; - u32 asid; int ret; smmu_domain = arm_smmu_domain_alloc(); @@ -410,12 +528,10 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; smmu_domain->smmu = smmu; - ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, - XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); + ret = arm_smmu_share_asid(smmu, smmu_domain, mm); if (ret) goto err_free; - smmu_domain->cd.asid = asid; smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops; ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm); if (ret) @@ -424,7 +540,9 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, return &smmu_domain->domain; err_asid: - xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); + arm_smmu_domain_free_id(smmu_domain); + if (smmu_domain->btm_invalidation) + arm64_mm_context_put(mm); err_free: kfree(smmu_domain); return ERR_PTR(ret); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index cceb737a7001..3eac9a8ca175 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -19,6 +19,7 @@ struct arm_smmu_test_writer { }; #define NUM_ENTRY_QWORDS 8 +#define DISABLE_BTM 0 #define NUM_EXPECTED_SYNCS(x) x static struct arm_smmu_ste bypass_ste; @@ -484,23 +485,24 @@ static void arm_smmu_v3_write_cd_test_s1_change_asid(struct kunit *test) NUM_EXPECTED_SYNCS(1)); } -static void arm_smmu_test_make_sva_cd(struct arm_smmu_cd *cd, unsigned int asid) +static void arm_smmu_test_make_sva_cd(struct arm_smmu_cd *cd, unsigned int asid, + bool btm_invalidation) { struct arm_smmu_master master = { .smmu = &smmu, }; - arm_smmu_make_sva_cd(cd, &master, &sva_mm, asid); + arm_smmu_make_sva_cd(cd, &master, &sva_mm, asid, btm_invalidation); } static void arm_smmu_test_make_sva_release_cd(struct arm_smmu_cd *cd, - unsigned int asid) + unsigned int asid, bool btm_invalidation) { struct arm_smmu_master master = { .smmu = &smmu, }; - arm_smmu_make_sva_cd(cd, &master, NULL, asid); + arm_smmu_make_sva_cd(cd, &master, NULL, asid, btm_invalidation); } static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) @@ -508,7 +510,7 @@ static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) struct arm_smmu_cd cd = {}; struct arm_smmu_cd cd_2; - arm_smmu_test_make_sva_cd(&cd_2, 1997); + arm_smmu_test_make_sva_cd(&cd_2, 1997, DISABLE_BTM); arm_smmu_v3_test_cd_expect_non_hitless_transition( test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); arm_smmu_v3_test_cd_expect_non_hitless_transition( @@ -520,8 +522,8 @@ static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) struct arm_smmu_cd cd; struct arm_smmu_cd cd_2; - arm_smmu_test_make_sva_cd(&cd, 1997); - arm_smmu_test_make_sva_release_cd(&cd_2, 1997); + arm_smmu_test_make_sva_cd(&cd, 1997, DISABLE_BTM); + arm_smmu_test_make_sva_release_cd(&cd_2, 1997, DISABLE_BTM); arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 73d14157d866..dbb3ba046c55 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -143,8 +143,6 @@ struct arm_smmu_option_prop { const char *prop; }; -DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa); -DEFINE_MUTEX(arm_smmu_asid_lock); static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" }, @@ -1586,6 +1584,8 @@ void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; + lockdep_assert_held(&master->smmu->asid_lock); + memset(target, 0, sizeof(*target)); target->data[0] = cpu_to_le64( @@ -2396,8 +2396,9 @@ static void arm_smmu_tlb_inv_context(void *cookie) * insertion to guarantee those are observed before the TLBI. Do be * careful, 007. */ - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid); + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 || + smmu_domain->domain.type == IOMMU_DOMAIN_SVA) { + arm_smmu_tlb_inv_asid(smmu, READ_ONCE(smmu_domain->cd.asid)); } else { cmd.opcode = CMDQ_OP_TLBI_S12_VMALL; cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; @@ -2654,46 +2655,57 @@ static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) return &smmu_domain->domain; } +/* + * Return the domain's ASID or VMID back to the allocator. All IDs in the + * allocator do not have an IOTLB entries referencing them. + */ +void arm_smmu_domain_free_id(struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + + arm_smmu_tlb_inv_context(smmu_domain); + + if ((smmu_domain->stage == ARM_SMMU_DOMAIN_S1 || + smmu_domain->domain.type == IOMMU_DOMAIN_SVA) && + smmu_domain->cd.asid) { + /* Prevent SVA from touching the CD while we're freeing it */ + mutex_lock(&smmu->asid_lock); + xa_erase(&smmu->asid_map, smmu_domain->cd.asid); + mutex_unlock(&smmu->asid_lock); + } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2 && + smmu_domain->s2_cfg.vmid) { + ida_free(&smmu->vmid_map, smmu_domain->s2_cfg.vmid); + } +} + static void arm_smmu_domain_free_paging(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu = smmu_domain->smmu; + /* + * At this point the page table is not programmed into any STE/CD and + * there is no possible concurrent HW walker running due to prior STE/CD + * invalidations. However entries tagged with the ASID/VMID may still be + * in the IOTLB. Invalidating the IOTLB should fully serialize any + * concurrent dirty bit write back before freeing the PTE memory. + */ + arm_smmu_domain_free_id(smmu_domain); free_io_pgtable_ops(smmu_domain->pgtbl_ops); #ifdef CONFIG_HISI_CCADA_HOST realm_smmu_domain_clear(smmu_domain); #endif - /* Free the ASID or VMID */ - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - /* Prevent SVA from touching the CD while we're freeing it */ - mutex_lock(&arm_smmu_asid_lock); - xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); - mutex_unlock(&arm_smmu_asid_lock); - } else { - struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; - if (cfg->vmid) - ida_free(&smmu->vmid_map, cfg->vmid); - } - kfree(smmu_domain); } static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, struct arm_smmu_domain *smmu_domain) { - int ret; - u32 asid = 0; struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; - /* Prevent SVA from modifying the ASID until it is written to the CD */ - mutex_lock(&arm_smmu_asid_lock); - ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, - XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); - cd->asid = (u16)asid; - mutex_unlock(&arm_smmu_asid_lock); - return ret; + return xa_alloc(&smmu->asid_map, &cd->asid, smmu_domain, + XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); } static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu, @@ -3035,7 +3047,7 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, * arm_smmu_master_domain contents otherwise it could randomly write one * or the other to the CD. */ - lockdep_assert_held(&arm_smmu_asid_lock); + mutex_unlock(&master->smmu->asid_lock); if (smmu_domain || state->cd_needs_ats) { /* @@ -3112,7 +3124,7 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state) { struct arm_smmu_master *master = state->master; - lockdep_assert_held(&arm_smmu_asid_lock); + lockdep_assert_held(&master->smmu->asid_lock); if (state->ats_enabled && !master->ats_enabled) { arm_smmu_enable_ats(master); @@ -3176,11 +3188,11 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) * This allows the STE and the smmu_domain->devices list to * be inconsistent during this routine. */ - mutex_lock(&arm_smmu_asid_lock); + mutex_lock(&master->smmu->asid_lock); ret = arm_smmu_attach_prepare(&state, domain); if (ret) { - mutex_unlock(&arm_smmu_asid_lock); + mutex_unlock(&master->smmu->asid_lock); return ret; } @@ -3209,7 +3221,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } arm_smmu_attach_commit(&state); - mutex_unlock(&arm_smmu_asid_lock); + mutex_unlock(&master->smmu->asid_lock); return 0; } @@ -3298,7 +3310,7 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, if (!cdptr) return -ENOMEM; - mutex_lock(&arm_smmu_asid_lock); + mutex_lock(&master->smmu->asid_lock); ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain); if (ret) goto out_unlock; @@ -3317,7 +3329,7 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, arm_smmu_attach_commit(&state); out_unlock: - mutex_unlock(&arm_smmu_asid_lock); + mutex_unlock(&master->smmu->asid_lock); return ret; } @@ -3329,12 +3341,12 @@ static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid, smmu_domain = to_smmu_domain(domain); - mutex_lock(&arm_smmu_asid_lock); + mutex_lock(&master->smmu->asid_lock); arm_smmu_clear_cd(master, pasid); if (master->ats_enabled) arm_smmu_atc_inv_master(master, pasid); arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid); - mutex_unlock(&arm_smmu_asid_lock); + mutex_unlock(&master->smmu->asid_lock); /* * When the last user of the CD table goes away downgrade the STE back @@ -3366,7 +3378,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, * Do not allow any ASID to be changed while are working on the STE, * otherwise we could miss invalidations. */ - mutex_lock(&arm_smmu_asid_lock); + mutex_lock(&master->smmu->asid_lock); /* * If the CD table is not in use we can use the provided STE, otherwise @@ -3386,7 +3398,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, } arm_smmu_install_ste_for_dev(master, ste); arm_smmu_attach_commit(&state); - mutex_unlock(&arm_smmu_asid_lock); + mutex_unlock(&master->smmu->asid_lock); /* * This has to be done after removing the master from the @@ -4528,6 +4540,8 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) return ret; ida_init(&smmu->vmid_map); + xa_init_flags(&smmu->asid_map, XA_FLAGS_ALLOC1); + mutex_init(&smmu->asid_lock); return devm_add_action_or_reset(smmu->dev, arm_smmu_destroy_vmid_map, &smmu->vmid_map); @@ -5266,7 +5280,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) { u32 reg; bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY; - + bool vhe = cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN); /* IDR0 */ reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); @@ -5318,12 +5332,21 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (reg & IDR0_HYP) { smmu->features |= ARM_SMMU_FEAT_HYP; - if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)) + if (vhe) smmu->features |= ARM_SMMU_FEAT_E2H; } arm_smmu_get_httu(smmu, reg); + /* + * If the CPU is using VHE, but the SMMU doesn't support it, the SMMU + * will create TLB entries for NH-EL1 world and will miss the + * broadcasted TLB invalidations that target EL2-E2H world. Don't enable + * BTM in that case. + */ + if (reg & IDR0_BTM && (!vhe || reg & IDR0_HYP)) + smmu->features |= ARM_SMMU_FEAT_BTM; + /* * The coherency feature as set by FW is used in preference to the ID * register, but warn on mismatch. diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 72bbdfb0f1e1..5c8b3cb3c3a9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -37,6 +37,7 @@ #define IDR0_ASID16 (1 << 12) #define IDR0_ATS (1 << 10) #define IDR0_HYP (1 << 9) +#define IDR0_BTM (1 << 5) #define IDR0_HTTU GENMASK(7, 6) #define IDR0_HTTU_ACCESS 1 #define IDR0_HTTU_ACCESS_DIRTY 2 @@ -696,7 +697,7 @@ struct arm_smmu_priq { /* High-level stream table and context descriptor structures */ struct arm_smmu_ctx_desc { - u16 asid; + u32 asid; }; struct arm_smmu_ctx_desc_cfg { @@ -867,6 +868,8 @@ struct arm_smmu_device { #define ARM_SMMU_MAX_ASIDS (1 << 16) unsigned int asid_bits; + struct xarray asid_map; + struct mutex asid_lock; #define ARM_SMMU_MAX_VMIDS (1 << 16) unsigned int vmid_bits; @@ -955,6 +958,7 @@ struct arm_smmu_domain { bool realm; bool pcipc_ns; #endif + bool btm_invalidation; }; struct arm_smmu_nested_domain { @@ -995,7 +999,7 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, unsigned int s1dss); void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, struct arm_smmu_master *master, struct mm_struct *mm, - u16 asid); + u16 asid, bool btm_invalidation); #endif struct arm_smmu_master_domain { @@ -1016,8 +1020,6 @@ to_smmu_nested_domain(struct iommu_domain *dom) return container_of(dom, struct arm_smmu_nested_domain, domain); } -extern struct xarray arm_smmu_asid_xa; -extern struct mutex arm_smmu_asid_lock; struct arm_smmu_domain *arm_smmu_domain_alloc(void); @@ -1035,6 +1037,7 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, struct arm_smmu_domain *smmu_domain, ioasid_t pasid, struct arm_smmu_cd *cd); +void arm_smmu_domain_free_id(struct arm_smmu_domain *smmu_domain); void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, size_t granule, bool leaf, -- 2.22.0
Offering: HULK hulk inclusion category: feature ------------------ Squash of three commits from the hulk_6.6 tree: 1) support disable BTM by cmdline Offering: HULK category: feature Disabling BTM by configuring IDR0 requires a cold reset. Offer a CONFIG_SMMU_DISABLE_BTM option and a disable_btm module parameter to turn BTM off softly at probe time. 2) Don't set PTM when BTM is enabled Offering: HULK category: bugfix Fixes: 15d1fd835ee0 ("[Backport] iommu/arm-smmu-v3: Bring back SVA BTM support") CR2.PTM==1 would cause the SMMU to ignore DVM broadcast, so if the BTM feature is used, we should not set it, otherwise the broadcast TLBI that BTM relies on never reaches the SMMU and the IOTLB becomes stale. 3) asid_lock unlocked incorrectly Offering: HULK category: bugfix Fixes: 15d1fd835ee0 ("[Backport] iommu/arm-smmu-v3: Bring back SVA BTM support") In arm_smmu_attach_prepare, it should check whether the asid_lock is held, rather than unlock it. Signed-off-by: Zhang Yuwei <zhangyuwei20@huawei.com> Signed-off-by: Wang Wensheng <wangwensheng4@huawei.com> --- drivers/iommu/Kconfig | 7 +++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index f91ab5cba204..876a8f39e93b 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -568,4 +568,11 @@ config LOONGARCH_IOMMU can isolate the DMA memory of different devices and protect the system from misbehaving device drivers or hardware. +config SMMU_DISABLE_BTM + bool "disable BTM" + depends on ARM_SMMU_V3=y + help + Disabling BTM by configuring IDR0 requires a cold reset. + Provide a module parameter to disable BTM softly. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index dbb3ba046c55..17fab83a0d18 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -42,6 +42,12 @@ #include <asm/hisi_cca_da.h> #endif +static bool disable_btm; +#ifdef CONFIG_SMMU_DISABLE_BTM +module_param(disable_btm, bool, 0444); +MODULE_PARM_DESC(disable_btm, "Disable BTM"); +#endif + static bool disable_msipolling; module_param(disable_msipolling, bool, 0444); MODULE_PARM_DESC(disable_msipolling, @@ -3047,7 +3053,7 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, * arm_smmu_master_domain contents otherwise it could randomly write one * or the other to the CD. */ - mutex_unlock(&master->smmu->asid_lock); + lockdep_assert_held(&master->smmu->asid_lock); if (smmu_domain || state->cd_needs_ats) { /* @@ -4927,11 +4933,14 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool resume) writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); /* CR2 (random crap) */ - reg = CR2_PTM | CR2_RECINVSID; + reg = CR2_RECINVSID; if (smmu->features & ARM_SMMU_FEAT_E2H) reg |= CR2_E2H; + if (!(smmu->features & ARM_SMMU_FEAT_BTM)) + reg |= CR2_PTM; + writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); /* Stream table */ @@ -5344,7 +5353,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) * broadcasted TLB invalidations that target EL2-E2H world. Don't enable * BTM in that case. */ - if (reg & IDR0_BTM && (!vhe || reg & IDR0_HYP)) + if (reg & IDR0_BTM && (!vhe || reg & IDR0_HYP) && !disable_btm) smmu->features |= ARM_SMMU_FEAT_BTM; /* -- 2.22.0
反馈: 您发送到kernel@openeuler.org的补丁/补丁集,转换为PR失败! 邮件列表地址:https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/MZ3... 失败原因:同步源码仓代码到fork仓失败 建议解决方法:请稍等,机器人会在下一次任务重新执行 FeedBack: The patch(es) which you have sent to kernel@openeuler.org has been converted to PR failed! Mailing list address: https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/MZ3... Failed Reason: sync origin kernel's codes to the fork repository failed Suggest Solution: please wait, the bot will retry in the next interval
反馈: 您发送到kernel@openeuler.org的补丁/补丁集,已成功转换为PR! PR链接地址: https://atomgit.com/openeuler/kernel/merge_requests/31004 邮件列表地址:https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/MZ3... FeedBack: The patch(es) which you have sent to kernel@openeuler.org mailing list has been converted to a pull request successfully! Pull request link: https://atomgit.com/openeuler/kernel/merge_requests/31004 Mailing list address: https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/MZ3...
participants (2)
-
patchwork bot -
Zhang Yuwei