maillist inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/8788 ------------------------------------------------------------- Add support for the contiguous hint (CONT) bit in ARM LPAE page tables, following the community v5 mapping and unmapping approach. Keep CONT disabled by default. Drivers opt in with IO_PGTABLE_QUIRK_ARM_CONT_HINT (BIT(12)) before allocating page tables. Derive supported CONT sizes from the architecture-valid page-size bitmap, check size expansion for overflow, and constrain the result by IAS/OAS. Only set CONT when callers select an advertised CONT group size. Do not promote ordinary page mappings into CONT groups. Bound map and unmap batches to the current table and reject incomplete CONT groups before modifying that batch. Earlier batches in a request may already have completed. Reject CONT blocks before splitting them on unmap; ordinary blocks can still be split. Dirty tracking cannot update CONT permissions as a group, so CONT and ARM_HD are mutually exclusive. Reject page-table allocation when both are requested, and fail the dirty-log entry points (split_block, merge_page, sync/clear_dirty_log and read_and_clear_dirty) on CONT page tables before walking them. The table walkers themselves are unchanged. Add each complete CONT group to the IOTLB gather at the group size after clearing its last descriptor, preserving the group boundary for range invalidation. SMMUv3 integration must separately ensure compatible range invalidation support and hardware errata handling; this patch does not change SMMUv3 policy or enable CONT in a driver. The patch has not been validated on real SMMU hardware for DMA or TLB invalidation behavior. Suggested-by: Robin Murphy <robin.murphy@arm.com> Co-developed-by: Vijayanand Jitta <vijayanand.jitta@oss.qualcomm.com> Signed-off-by: Vijayanand Jitta <vijayanand.jitta@oss.qualcomm.com> Signed-off-by: Prakash Gupta <prakash.gupta@oss.qualcomm.com> Signed-off-by: Zhang Yuwei <zhangyuwei20@huawei.com> --- drivers/iommu/io-pgtable-arm.c | 154 ++++++++++++++++++++++++++++++--- include/linux/io-pgtable.h | 4 + 2 files changed, 145 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index aecd5939e7b2..f559e258377e 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -93,6 +93,16 @@ /* Software bit for solving coherency races */ #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) +/* PTE Contiguous Bit */ +#define ARM_LPAE_PTE_CONT (((arm_lpae_iopte)1) << 52) + +/* + * Granule CONT page Block CONT block L1 block CONT L1 + * 4K 64K 2M 32M 1G 16G + * 16K 2M 32M 1G + * 64K 2M 512M 16G + */ + /* Stage-1 PTE */ #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) #define ARM_LPAE_PTE_AP_RDONLY_BIT 7 @@ -408,6 +418,24 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, return old; } +static int arm_lpae_num_cont(size_t size) +{ + switch (size) { + case SZ_4K: + case SZ_2M: + case SZ_1G: + return 16; + case SZ_64K: + case SZ_32M: + case SZ_512M: + return 32; + case SZ_16K: + return 128; + default: + return 1; + } +} + static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, size_t pgcount, arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep, @@ -417,19 +445,40 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); size_t tblsz = ARM_LPAE_GRANULE(data); struct io_pgtable_cfg *cfg = &data->iop.cfg; - int ret = 0, num_entries, max_entries, map_idx_start; + int num_cont = arm_lpae_num_cont(block_size); + size_t cont_size = 0, entries_per_map; + int num_entries, max_entries, map_idx_start; + + if (num_cont > 1 && block_size <= SIZE_MAX / num_cont) + cont_size = num_cont * block_size; /* Find our entry at the current level */ map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); ptep += map_idx_start; /* If we can install a leaf entry at this level, then do so */ - if (size == block_size) { + if (size == block_size || + ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) && + size == cont_size)) { + bool cont = size == cont_size; + int ret; + + if (cont && (!IS_ALIGNED(iova, size) || !IS_ALIGNED(paddr, size))) + return -EINVAL; + + entries_per_map = size / block_size; max_entries = arm_lpae_max_entries(map_idx_start, data); - num_entries = min_t(int, pgcount, max_entries); - ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); + num_entries = min_t(size_t, pgcount, + max_entries / entries_per_map) * entries_per_map; + if (!num_entries) + return -EINVAL; + if (cont) + prot |= ARM_LPAE_PTE_CONT; + + ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, + ptep); if (!ret) - *mapped += num_entries * size; + *mapped += num_entries * block_size; return ret; } @@ -683,12 +732,19 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, { arm_lpae_iopte pte; struct io_pgtable *iop = &data->iop; + size_t block_size, cont_size = 0, entries_per_map; + int num_cont; int i = 0, num_entries, max_entries, unmap_idx_start; /* Something went horribly wrong and we ran out of page table */ if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) return 0; + block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + num_cont = arm_lpae_num_cont(block_size); + if (num_cont > 1 && block_size <= SIZE_MAX / num_cont) + cont_size = num_cont * block_size; + unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); ptep += unmap_idx_start; pte = READ_ONCE(*ptep); @@ -696,9 +752,24 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, return 0; /* If the size matches this level, we're in the right place */ - if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { + if (size == block_size || + ((iop->cfg.quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) && + size == cont_size)) { + entries_per_map = size / block_size; max_entries = arm_lpae_max_entries(unmap_idx_start, data); - num_entries = min_t(int, pgcount, max_entries); + num_entries = min_t(size_t, pgcount, + max_entries / entries_per_map) * entries_per_map; + if (!num_entries) + return 0; + + /* Check this batch before clearing PTEs or freeing sub-tables. */ + if (((pte & ARM_LPAE_PTE_CONT) && + !IS_ALIGNED(iova, cont_size)) || + ((READ_ONCE(ptep[num_entries - 1]) & ARM_LPAE_PTE_CONT) && + !IS_ALIGNED(iova + num_entries * block_size, cont_size))) { + WARN_ON_ONCE(1); + return 0; + } while (i < num_entries) { pte = READ_ONCE(*ptep); @@ -709,19 +780,33 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, if (!iopte_leaf(pte, lvl, iop->fmt)) { /* Also flush any partial walks */ - io_pgtable_tlb_flush_walk(iop, iova + i * size, size, - ARM_LPAE_GRANULE(data)); + io_pgtable_tlb_flush_walk(iop, iova + i * block_size, + block_size, ARM_LPAE_GRANULE(data)); __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); } else if (!iommu_iotlb_gather_queued(gather)) { - io_pgtable_tlb_add_page(iop, gather, iova + i * size, size); + unsigned long end = iova + (i + 1) * block_size; + + if (pte & ARM_LPAE_PTE_CONT) { + /* Add a complete group after its last PTE is cleared. */ + if (IS_ALIGNED(end, cont_size)) + io_pgtable_tlb_add_page(iop, gather, + end - cont_size, cont_size); + } else { + io_pgtable_tlb_add_page(iop, gather, + end - block_size, + block_size); + } } ptep++; i++; } - return i * size; + return i * block_size; } else if (iopte_leaf(pte, lvl, iop->fmt)) { + /* Never split a block that belongs to a CONT group. */ + if (WARN_ON_ONCE(pte & ARM_LPAE_PTE_CONT)) + return 0; /* * Insert a table at the next level to map the old region, * minus the part we want to unmap @@ -896,6 +981,10 @@ static size_t arm_lpae_split_block(struct io_pgtable_ops *ops, int lvl = data->start_level; long iaext = (s64)iova >> cfg->ias; + /* CONT page tables do not support dirty tracking. */ + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) + return 0; + if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) return 0; @@ -968,6 +1057,10 @@ static size_t arm_lpae_merge_page(struct io_pgtable_ops *ops, unsigned long iova arm_lpae_iopte prot; long iaext = (s64)iova >> cfg->ias; + /* CONT page tables do not support dirty tracking. */ + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) + return 0; + if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) return 0; @@ -1060,6 +1153,10 @@ static int arm_lpae_sync_dirty_log(struct io_pgtable_ops *ops, int lvl = data->start_level; long iaext = (s64)iova >> cfg->ias; + /* CONT page tables do not support dirty tracking. */ + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) + return -EOPNOTSUPP; + if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) return -EINVAL; @@ -1149,6 +1246,10 @@ static int arm_lpae_clear_dirty_log(struct io_pgtable_ops *ops, int lvl = data->start_level; long iaext = (s64)iova >> cfg->ias; + /* CONT page tables do not support dirty tracking. */ + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) + return -EOPNOTSUPP; + if (WARN_ON(!size || (size & cfg->pgsize_bitmap) != size)) return -EINVAL; @@ -1246,6 +1347,10 @@ static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops, arm_lpae_iopte *ptep = data->pgd; int lvl = data->start_level; + /* CONT page tables do not support dirty tracking. */ + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) + return -EOPNOTSUPP; + if (WARN_ON(!size)) return -EINVAL; if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1))) @@ -1295,8 +1400,23 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) } cfg->pgsize_bitmap &= page_sizes; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_CONT_HINT) { + unsigned long sizes = cfg->pgsize_bitmap; + + while (sizes) { + unsigned long size = BIT(__ffs(sizes)); + int num_cont = arm_lpae_num_cont(size); + + if (size <= ULONG_MAX / num_cont) + cfg->pgsize_bitmap |= num_cont * size; + sizes &= ~size; + } + } + cfg->ias = min(cfg->ias, max_addr_bits); cfg->oas = min(cfg->oas, max_addr_bits); + cfg->pgsize_bitmap &= GENMASK_ULL(cfg->ias, 0); + cfg->pgsize_bitmap &= GENMASK_ULL(cfg->oas, 0); } static struct arm_lpae_io_pgtable * @@ -1305,6 +1425,12 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) struct arm_lpae_io_pgtable *data; int levels, va_bits, pg_shift; + /* Dirty tracking and CONT permissions cannot be updated as a group yet. */ + if ((cfg->quirks & (IO_PGTABLE_QUIRK_ARM_HD | + IO_PGTABLE_QUIRK_ARM_CONT_HINT)) == + (IO_PGTABLE_QUIRK_ARM_HD | IO_PGTABLE_QUIRK_ARM_CONT_HINT)) + return NULL; + arm_lpae_restrict_pgsizes(cfg); if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) @@ -1358,7 +1484,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) IO_PGTABLE_QUIRK_ARM_HD | IO_PGTABLE_QUIRK_ARM_BBML1 | IO_PGTABLE_QUIRK_ARM_BBML2 | - IO_PGTABLE_QUIRK_HISI_ERRATA)) + IO_PGTABLE_QUIRK_HISI_ERRATA | + IO_PGTABLE_QUIRK_ARM_CONT_HINT)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -1464,7 +1591,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) IO_PGTABLE_QUIRK_ARM_BBML1 | IO_PGTABLE_QUIRK_ARM_BBML2 | IO_PGTABLE_QUIRK_ARM_S2FWB | - IO_PGTABLE_QUIRK_HISI_ERRATA)) + IO_PGTABLE_QUIRK_HISI_ERRATA | + IO_PGTABLE_QUIRK_ARM_CONT_HINT)) return NULL; data = arm_lpae_alloc_pgtable(cfg); diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 62a0c5b0f55f..1febcede3791 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -98,6 +98,9 @@ struct io_pgtable_cfg { * IO_PGTABLE_QUIRK_ARM_BBML2: ARM SMMU supports BBM Level 2 behavior * when changing block size. * IO_PGTABLE_QUIRK_ARM_S2FWB: Use the FWB format for the MemAttrs bits + * IO_PGTABLE_QUIRK_ARM_CONT_HINT: Enable contiguous hints for ARM LPAE. + * Disabled by default; drivers opt in before alloc_io_pgtable_ops(). + * Only mappings using an advertised CONT group size are tagged. */ #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) @@ -110,6 +113,7 @@ struct io_pgtable_cfg { #define IO_PGTABLE_QUIRK_ARM_BBML2 BIT(9) #define IO_PGTABLE_QUIRK_ARM_S2FWB BIT(10) #define IO_PGTABLE_QUIRK_HISI_ERRATA BIT(11) + #define IO_PGTABLE_QUIRK_ARM_CONT_HINT BIT(12) unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; -- 2.22.0