maillist inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/8788 ------------------------------------------------------------- Add support for the contiguous hint (CONT) bit in ARM LPAE page tables, following the community v5 mapping and unmapping approach. Keep CONT disabled by default. SMMU drivers opt in with IO_PGTABLE_QUIRK_ARM_CONT_HINT (BIT(12)) before allocating page tables. Mali continues to reject quirks and never advertises CONT sizes. Derive CONT sizes from the architecture-valid base page-size bitmap, guard expansion against overflow, and constrain the result by IAS/OAS. Keep sizes which exactly span the configured address space. Only tag descriptors when the caller selects an advertised CONT size. Do not promote aligned runs of ordinary page mappings internally. Bound the number of groups before converting them to descriptor counts. Check the first and last descriptors of the current unmap batch before clearing PTEs or freeing sub-tables. Reject a partial CONT group with no changes to that batch. Earlier batches in an iommu_unmap() request may already have completed; this is not a whole-request atomicity guarantee. Unlike the community baseline, OLK 6.6 supports splitting leaf blocks on partial unmap. Reject CONT-tagged blocks before entering that path while preserving splitting for ordinary blocks. Suggested-by: Robin Murphy <robin.murphy@arm.com> Co-developed-by: Vijayanand Jitta <vijayanand.jitta@oss.qualcomm.com> Signed-off-by: Vijayanand Jitta <vijayanand.jitta@oss.qualcomm.com> Signed-off-by: Prakash Gupta <prakash.gupta@oss.qualcomm.com> Signed-off-by: Zhang Yuwei <zhangyuwei20@huawei.com> --- drivers/iommu/io-pgtable-arm.c | 141 +++++++++++++++++++++++++++++---- include/linux/io-pgtable.h | 4 + 2 files changed, 130 insertions(+), 15 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index aecd5939e7b2..c88b205b49ea 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -93,6 +93,16 @@ /* Software bit for solving coherency races */ #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) +/* PTE Contiguous Bit */ +#define ARM_LPAE_PTE_CONT (((arm_lpae_iopte)1) << 52) + +/* + * Granule CONT page Block CONT block L1 block CONT L1 + * 4K 64K 2M 32M 1G 16G + * 16K 2M 32M 1G + * 64K 2M 512M 16G + */ + /* Stage-1 PTE */ #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) #define ARM_LPAE_PTE_AP_RDONLY_BIT 7 @@ -408,6 +418,24 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, return old; } +static int arm_lpae_num_cont(size_t size) +{ + switch (size) { + case SZ_4K: + case SZ_2M: + case SZ_1G: + return 16; + case SZ_64K: + case SZ_32M: + case SZ_512M: + return 32; + case SZ_16K: + return 128; + default: + return 1; + } +} + static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, size_t pgcount, arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep, @@ -417,19 +445,40 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); size_t tblsz = ARM_LPAE_GRANULE(data); struct io_pgtable_cfg *cfg = &data->iop.cfg; - int ret = 0, num_entries, max_entries, map_idx_start; + int num_cont = arm_lpae_num_cont(block_size); + size_t cont_size = 0, entries_per_map; + int num_entries, max_entries, map_idx_start; + + if (num_cont > 1 && block_size <= SIZE_MAX / num_cont) + cont_size = num_cont * block_size; /* Find our entry at the current level */ map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); ptep += map_idx_start; /* If we can install a leaf entry at this level, then do so */ - if (size == block_size) { + if (size == block_size || + ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) && + size == cont_size)) { + bool cont = size == cont_size; + int ret; + + if (cont && (!IS_ALIGNED(iova, size) || !IS_ALIGNED(paddr, size))) + return -EINVAL; + + entries_per_map = size / block_size; max_entries = arm_lpae_max_entries(map_idx_start, data); - num_entries = min_t(int, pgcount, max_entries); - ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); + num_entries = min_t(size_t, pgcount, + max_entries / entries_per_map) * entries_per_map; + if (!num_entries) + return -EINVAL; + if (cont) + prot |= ARM_LPAE_PTE_CONT; + + ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, + num_entries, ptep); if (!ret) - *mapped += num_entries * size; + *mapped += num_entries * block_size; return ret; } @@ -683,12 +732,19 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, { arm_lpae_iopte pte; struct io_pgtable *iop = &data->iop; + size_t block_size, cont_size = 0, entries_per_map; + int num_cont; int i = 0, num_entries, max_entries, unmap_idx_start; /* Something went horribly wrong and we ran out of page table */ if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) return 0; + block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + num_cont = arm_lpae_num_cont(block_size); + if (num_cont > 1 && block_size <= SIZE_MAX / num_cont) + cont_size = num_cont * block_size; + unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); ptep += unmap_idx_start; pte = READ_ONCE(*ptep); @@ -696,9 +752,24 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, return 0; /* If the size matches this level, we're in the right place */ - if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { + if (size == block_size || + ((iop->cfg.quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) && + size == cont_size)) { + entries_per_map = size / block_size; max_entries = arm_lpae_max_entries(unmap_idx_start, data); - num_entries = min_t(int, pgcount, max_entries); + num_entries = min_t(size_t, pgcount, + max_entries / entries_per_map) * entries_per_map; + if (!num_entries) + return 0; + + /* Check this batch before clearing PTEs or freeing sub-tables. */ + if (((pte & ARM_LPAE_PTE_CONT) && + !IS_ALIGNED(iova, cont_size)) || + ((READ_ONCE(ptep[num_entries - 1]) & ARM_LPAE_PTE_CONT) && + !IS_ALIGNED(iova + num_entries * block_size, cont_size))) { + WARN_ON_ONCE(1); + return 0; + } while (i < num_entries) { pte = READ_ONCE(*ptep); @@ -709,19 +780,24 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, if (!iopte_leaf(pte, lvl, iop->fmt)) { /* Also flush any partial walks */ - io_pgtable_tlb_flush_walk(iop, iova + i * size, size, - ARM_LPAE_GRANULE(data)); + io_pgtable_tlb_flush_walk(iop, + iova + i * block_size, block_size, + ARM_LPAE_GRANULE(data)); __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); } else if (!iommu_iotlb_gather_queued(gather)) { - io_pgtable_tlb_add_page(iop, gather, iova + i * size, size); + io_pgtable_tlb_add_page(iop, gather, + iova + i * block_size, block_size); } ptep++; i++; } - return i * size; + return i * block_size; } else if (iopte_leaf(pte, lvl, iop->fmt)) { + /* Never split a block that belongs to a CONT group. */ + if (WARN_ON_ONCE(pte & ARM_LPAE_PTE_CONT)) + return 0; /* * Insert a table at the next level to map the old region, * minus the part we want to unmap @@ -856,6 +932,9 @@ static size_t __arm_lpae_split_block(struct arm_lpae_io_pgtable *data, if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { if (iopte_leaf(pte, lvl, iop->fmt)) { + /* Splitting one member would invalidate the CONT group. */ + if (WARN_ON_ONCE(pte & ARM_LPAE_PTE_CONT)) + return 0; if (lvl == (ARM_LPAE_MAX_LEVELS - 1) || (pte & ARM_LPAE_PTE_AP_RDONLY)) return size; @@ -919,18 +998,33 @@ static size_t __arm_lpae_merge_page(struct arm_lpae_io_pgtable *data, arm_lpae_iopte pte, *tablep; struct io_pgtable *iop = &data->iop; struct io_pgtable_cfg *cfg = &data->iop.cfg; + int idx, num_cont, i; if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) return 0; - ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); + idx = ARM_LPAE_LVL_IDX(iova, lvl, data); + ptep += idx; pte = READ_ONCE(*ptep); if (WARN_ON(!pte)) return 0; if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { - if (iopte_leaf(pte, lvl, iop->fmt)) + if (iopte_leaf(pte, lvl, iop->fmt)) { + if (WARN_ON_ONCE(pte & ARM_LPAE_PTE_CONT)) + return 0; return size; + } + /* Do not replace a table inside a partially split CONT group. */ + num_cont = arm_lpae_num_cont(size); + if ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) && num_cont > 1) { + arm_lpae_iopte *group = ptep - (idx % num_cont); + + for (i = 0; i < num_cont; i++) { + if (WARN_ON_ONCE(READ_ONCE(group[i]) & ARM_LPAE_PTE_CONT)) + return 0; + } + } /* Race does not exist */ if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_BBML1) { @@ -1295,8 +1389,23 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) } cfg->pgsize_bitmap &= page_sizes; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) { + unsigned long sizes = cfg->pgsize_bitmap; + + while (sizes) { + unsigned long size = BIT(__ffs(sizes)); + int num_cont = arm_lpae_num_cont(size); + + if (size <= ULONG_MAX / num_cont) + cfg->pgsize_bitmap |= num_cont * size; + sizes &= ~size; + } + } + cfg->ias = min(cfg->ias, max_addr_bits); cfg->oas = min(cfg->oas, max_addr_bits); + cfg->pgsize_bitmap &= GENMASK_ULL(cfg->ias, 0); + cfg->pgsize_bitmap &= GENMASK_ULL(cfg->oas, 0); } static struct arm_lpae_io_pgtable * @@ -1358,7 +1467,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) IO_PGTABLE_QUIRK_ARM_HD | IO_PGTABLE_QUIRK_ARM_BBML1 | IO_PGTABLE_QUIRK_ARM_BBML2 | - IO_PGTABLE_QUIRK_HISI_ERRATA)) + IO_PGTABLE_QUIRK_HISI_ERRATA | + IO_PGTABLE_QUIRK_ARM_CONT_HINT)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -1464,7 +1574,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) IO_PGTABLE_QUIRK_ARM_BBML1 | IO_PGTABLE_QUIRK_ARM_BBML2 | IO_PGTABLE_QUIRK_ARM_S2FWB | - IO_PGTABLE_QUIRK_HISI_ERRATA)) + IO_PGTABLE_QUIRK_HISI_ERRATA | + IO_PGTABLE_QUIRK_ARM_CONT_HINT)) return NULL; data = arm_lpae_alloc_pgtable(cfg); diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 62a0c5b0f55f..1febcede3791 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -98,6 +98,9 @@ struct io_pgtable_cfg { * IO_PGTABLE_QUIRK_ARM_BBML2: ARM SMMU supports BBM Level 2 behavior * when changing block size. * IO_PGTABLE_QUIRK_ARM_S2FWB: Use the FWB format for the MemAttrs bits + * IO_PGTABLE_QUIRK_ARM_CONT_HINT: Enable contiguous hints for ARM LPAE. + * Disabled by default; drivers opt in before alloc_io_pgtable_ops(). + * Only mappings using an advertised CONT group size are tagged. */ #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) @@ -110,6 +113,7 @@ struct io_pgtable_cfg { #define IO_PGTABLE_QUIRK_ARM_BBML2 BIT(9) #define IO_PGTABLE_QUIRK_ARM_S2FWB BIT(10) #define IO_PGTABLE_QUIRK_HISI_ERRATA BIT(11) + #define IO_PGTABLE_QUIRK_ARM_CONT_HINT BIT(12) unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; -- 2.22.0