mail inclusion
category: feature
bugzilla: NA
-------------------------------------------------------------
Add support for the contiguous hint (CONT) bit in ARM LPAE page tables.
When a set of consecutive PTEs map a naturally-aligned contiguous block
of memory, the CONT bit can be set on all entries in the group to allow
the hardware to combine them into a single TLB entry, improving TLB
utilization.
The contiguous hint sizes per granule are:
Page Size | CONT PTE | Block | CONT Block | L1 Block | CONT L1
----------+----------+---------+------------+----------+---------
4K | 64K | 2M | 32M | 1G | 16G
16K | 2M | 32M | 1G | |
64K | 2M | 512M | 16G | |
Contiguous hint sizes are advertised in pgsize_bitmap so that IOMMU API
users can align allocations to these sizes and benefit from the TLB
optimization automatically.
Partial unmaps of a contiguous group are rejected, ensuring the full
group is always invalidated as a unit. The
IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT quirk allows SMMU drivers to disable
contiguous hint support at runtime for hardware with
implementation-specific errata.
Suggested-by: Robin Murphy <robin.murphy(a)arm.com>
Co-developed-by: Vijayanand Jitta <vijayanand.jitta(a)oss.qualcomm.com>
Signed-off-by: Vijayanand Jitta <vijayanand.jitta(a)oss.qualcomm.com>
Signed-off-by: Prakash Gupta <prakash.gupta(a)oss.qualcomm.com>
Signed-off-by: Zhang Yuwei <zhangyuwei20(a)huawei.com>
---
drivers/iommu/io-pgtable-arm.c | 242 +++++++++++++++++++++++++++++++--
include/linux/io-pgtable.h | 3 +
2 files changed, 235 insertions(+), 10 deletions(-)
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index aecd5939e7b2..5c2e09cd8ec5 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -93,6 +93,21 @@
/* Software bit for solving coherency races */
#define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)
+/* PTE Contiguous Bit */
+#define ARM_LPAE_PTE_CONT (((arm_lpae_iopte)1) << 52)
+
+/*
+ * Contiguous hint group sizes per granule:
+ *
+ *------------------------------------------------------------------
+ *| Page Size | CONT PTE | Block | CONT Block | L1 Block | CONT L1 |
+ *------------------------------------------------------------------
+ *| 4K | 64K | 2M | 32M | 1G | 16G |
+ *| 16K | 2M | 32M | 1G | | |
+ *| 64K | 2M | 512M | 16G | | |
+ *------------------------------------------------------------------
+*/
+
/* Stage-1 PTE */
#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
#define ARM_LPAE_PTE_AP_RDONLY_BIT 7
@@ -408,6 +423,137 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
return old;
}
+static int arm_lpae_num_cont(size_t size)
+{
+ switch (size) {
+ case SZ_4K:
+ case SZ_2M:
+ case SZ_1G:
+ return 16;
+ case SZ_64K:
+ case SZ_32M:
+ case SZ_512M:
+ return 32;
+ case SZ_16K:
+ return 128;
+ default:
+ return 1;
+ }
+}
+
+/*
+ * A group is only usable if its span fits within both the configured
+ * ias and oas. Otherwise no aligned iova/paddr pair for a full group
+ * can exist, and advertising the size would let callers pick a
+ * mapping that unconditionally fails.
+ */
+static bool arm_lpae_cont_size_fits(struct io_pgtable_cfg *cfg, unsigned long size)
+{
+ int size_bits = ilog2(size);
+
+ return size_bits <= cfg->ias && size_bits <= cfg->oas;
+}
+
+static unsigned long arm_lpae_get_cont_sizes(struct io_pgtable_cfg *cfg)
+{
+ unsigned long pg_size, blk_size, l1_blk_size, cont_sizes = 0;
+ unsigned long cont_leaf_size, cont_blk_size, cont_l1_blk_size;
+ int pg_shift, bits_per_level;
+
+ if (!cfg->pgsize_bitmap || (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT))
+ return 0;
+
+ pg_shift = __ffs(cfg->pgsize_bitmap);
+ bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));
+ pg_size = 1UL << pg_shift;
+ blk_size = pg_size << bits_per_level;
+ l1_blk_size = blk_size << bits_per_level;
+
+ cont_leaf_size = arm_lpae_num_cont(pg_size) * pg_size;
+ if ((cfg->pgsize_bitmap & pg_size) &&
+ arm_lpae_cont_size_fits(cfg, cont_leaf_size))
+ cont_sizes |= cont_leaf_size;
+
+ if (cfg->pgsize_bitmap & blk_size) {
+ cont_blk_size = arm_lpae_num_cont(blk_size) * blk_size;
+ if (arm_lpae_cont_size_fits(cfg, cont_blk_size))
+ cont_sizes |= cont_blk_size;
+ }
+
+ /*
+ * l1_blk_size is only set in pgsize_bitmap if level-1 blocks are
+ * supported for this granule (not 16K/64K, per
+ * arm_lpae_restrict_pgsizes()), so no extra gating is needed here.
+ */
+ if (cfg->pgsize_bitmap & l1_blk_size) {
+ cont_l1_blk_size = arm_lpae_num_cont(l1_blk_size) * l1_blk_size;
+ if (arm_lpae_cont_size_fits(cfg, cont_l1_blk_size))
+ cont_sizes |= cont_l1_blk_size;
+ }
+
+ return cont_sizes;
+}
+
+/*
+ * Install num_entries leaf entries starting at ptep (index map_idx_start
+ * within the current table), tagging arm_lpae_num_cont()-sized groups with
+ * the contiguous hint where both idx and paddr are aligned to the group
+ * size. Entries in a misaligned group are installed without the hint.
+ *
+ * idx and paddr both advance by block_size per entry, so their alignment
+ * relative to the group size is invariant across a run of entries within
+ * this call: once a group qualifies (or fails to), every later whole group
+ * does too, up to num_entries. This merges each such run into a single
+ * arm_lpae_init_pte() call instead of one call per group.
+ */
+static int arm_lpae_install_leaf(struct arm_lpae_io_pgtable *data,
+ unsigned long iova, phys_addr_t paddr,
+ arm_lpae_iopte prot, int lvl,
+ int map_idx_start, int num_entries, int num_cont,
+ arm_lpae_iopte *ptep, size_t *mapped)
+{
+ size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
+ size_t cont_size = num_cont * block_size;
+ int done = 0;
+
+ while (done < num_entries) {
+ int idx = map_idx_start + done;
+ int remaining = num_entries - done;
+ int off = idx % num_cont;
+ arm_lpae_iopte pte = prot;
+ int chunk, ret;
+
+ if (off) {
+ /* Misaligned prefix: advance to the next boundary */
+ chunk = min_t(int, num_cont - off, remaining);
+ } else if (remaining >= num_cont && IS_ALIGNED(paddr, cont_size)) {
+ /* Aligned: merge every full group in this run */
+ chunk = remaining - remaining % num_cont;
+ pte |= ARM_LPAE_PTE_CONT;
+ } else {
+ /*
+ * Aligned idx but paddr doesn't line up with cont_size,
+ * or too short for a full group. That holds for the
+ * rest of this call too, so install the remainder
+ * plain in one go.
+ */
+ chunk = remaining;
+ }
+
+ ret = arm_lpae_init_pte(data, iova, paddr, pte, lvl, chunk, ptep);
+ if (ret)
+ return ret;
+
+ *mapped += chunk * block_size;
+ ptep += chunk;
+ iova += chunk * block_size;
+ paddr += chunk * block_size;
+ done += chunk;
+ }
+
+ return 0;
+}
+
static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
phys_addr_t paddr, size_t size, size_t pgcount,
arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,
@@ -417,21 +563,44 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
size_t tblsz = ARM_LPAE_GRANULE(data);
struct io_pgtable_cfg *cfg = &data->iop.cfg;
- int ret = 0, num_entries, max_entries, map_idx_start;
+ bool cont_hint_enabled = !(cfg->quirks & IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT);
+ int num_entries, max_entries, map_idx_start;
+ int num_cont = cont_hint_enabled ? arm_lpae_num_cont(block_size) : 1;
+ bool use_cont = cont_hint_enabled && num_cont > 1;
/* Find our entry at the current level */
map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
ptep += map_idx_start;
+ /*
+ * Normalize an exact whole-CONT-group request down to the
+ * equivalent block_size/pgcount so it funnels through the same
+ * leaf path below. arm_lpae_install_leaf() independently decides,
+ * per sub-chunk, whether the CONT hint actually applies.
+ */
+ if (use_cont && size == block_size * num_cont) {
+ pgcount *= num_cont;
+ size = block_size;
+ }
+
/* If we can install a leaf entry at this level, then do so */
if (size == block_size) {
+ int ret;
+
max_entries = arm_lpae_max_entries(map_idx_start, data);
- num_entries = min_t(int, pgcount, max_entries);
- ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
- if (!ret)
- *mapped += num_entries * size;
+ num_entries = min_t(size_t, pgcount, max_entries);
- return ret;
+ if (!use_cont) {
+ ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl,
+ num_entries, ptep);
+ if (!ret)
+ *mapped += num_entries * size;
+ return ret;
+ }
+
+ return arm_lpae_install_leaf(data, iova, paddr, prot, lvl,
+ map_idx_start, num_entries,
+ num_cont, ptep, mapped);
}
/* We can't allocate tables at the final level */
@@ -683,6 +852,8 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
{
arm_lpae_iopte pte;
struct io_pgtable *iop = &data->iop;
+ size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
+ int num_cont = arm_lpae_num_cont(block_size);
int i = 0, num_entries, max_entries, unmap_idx_start;
/* Something went horribly wrong and we ran out of page table */
@@ -695,16 +866,62 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
if (WARN_ON(!pte))
return 0;
+ /*
+ * Normalize an exact whole-CONT-group request down to the
+ * equivalent block_size/pgcount, mirroring __arm_lpae_map().
+ */
+ if (!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT) &&
+ num_cont > 1 && size == block_size * num_cont) {
+ pgcount *= num_cont;
+ size = block_size;
+ }
+
/* If the size matches this level, we're in the right place */
- if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
+ if (size == block_size) {
+ size_t cont_size = num_cont * block_size;
+
max_entries = arm_lpae_max_entries(unmap_idx_start, data);
- num_entries = min_t(int, pgcount, max_entries);
+ num_entries = min_t(size_t, pgcount, max_entries);
while (i < num_entries) {
pte = READ_ONCE(*ptep);
if (WARN_ON(!pte))
break;
+ /*
+ * A real CONT group must always be invalidated as a
+ * unit, so reject an unmap that splits one. Check the
+ * PTE's own CONT bit rather than the caller's size,
+ * since a legitimate unmap can span multiple prior
+ * iommu_map() calls and its size alone doesn't say how
+ * the underlying PTEs were grouped. Only the first and
+ * last entries can straddle a group boundary; an
+ * interior CONT-tagged entry's group is necessarily
+ * fully covered by this unmap, since groups can't
+ * overlap without also covering everything between
+ * them.
+ */
+ if (pte & ARM_LPAE_PTE_CONT) {
+ bool ok = true;
+
+ if (i == 0)
+ ok = ok && IS_ALIGNED(iova, cont_size);
+ if (i == num_entries - 1)
+ ok = ok && IS_ALIGNED(iova + (i + 1) * block_size,
+ cont_size);
+
+ /*
+ * Stop short of this entry instead of returning
+ * 0: entries before i may already have had
+ * non-leaf sub-tables torn down above, so the
+ * caller needs the real unmapped count, and the
+ * loop exit below still clears/gathers entries
+ * [0, i) correctly.
+ */
+ if (WARN_ON_ONCE(!ok))
+ break;
+ }
+
__arm_lpae_clear_pte(ptep, &iop->cfg);
if (!iopte_leaf(pte, lvl, iop->fmt)) {
@@ -1295,6 +1512,7 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
}
cfg->pgsize_bitmap &= page_sizes;
+ cfg->pgsize_bitmap |= arm_lpae_get_cont_sizes(cfg);
cfg->ias = min(cfg->ias, max_addr_bits);
cfg->oas = min(cfg->oas, max_addr_bits);
}
@@ -1358,7 +1576,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
IO_PGTABLE_QUIRK_ARM_HD |
IO_PGTABLE_QUIRK_ARM_BBML1 |
IO_PGTABLE_QUIRK_ARM_BBML2 |
- IO_PGTABLE_QUIRK_HISI_ERRATA))
+ IO_PGTABLE_QUIRK_HISI_ERRATA |
+ IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT))
return NULL;
data = arm_lpae_alloc_pgtable(cfg);
@@ -1464,7 +1683,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
IO_PGTABLE_QUIRK_ARM_BBML1 |
IO_PGTABLE_QUIRK_ARM_BBML2 |
IO_PGTABLE_QUIRK_ARM_S2FWB |
- IO_PGTABLE_QUIRK_HISI_ERRATA))
+ IO_PGTABLE_QUIRK_HISI_ERRATA |
+ IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT))
return NULL;
data = arm_lpae_alloc_pgtable(cfg);
@@ -1591,6 +1811,8 @@ arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
return NULL;
cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
+ /* Mali LPAE has no CONT bit - never advertise CONT page sizes */
+ cfg->quirks |= IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT;
data = arm_lpae_alloc_pgtable(cfg);
if (!data)
diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h
index 62a0c5b0f55f..16dd83f38744 100644
--- a/include/linux/io-pgtable.h
+++ b/include/linux/io-pgtable.h
@@ -98,6 +98,8 @@ struct io_pgtable_cfg {
* IO_PGTABLE_QUIRK_ARM_BBML2: ARM SMMU supports BBM Level 2 behavior
* when changing block size.
* IO_PGTABLE_QUIRK_ARM_S2FWB: Use the FWB format for the MemAttrs bits
+ * IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT: Disable use of the contiguous
+ * hint for hardware affected by implementation-specific errata.
*/
#define IO_PGTABLE_QUIRK_ARM_NS BIT(0)
#define IO_PGTABLE_QUIRK_NO_PERMS BIT(1)
@@ -110,6 +112,7 @@ struct io_pgtable_cfg {
#define IO_PGTABLE_QUIRK_ARM_BBML2 BIT(9)
#define IO_PGTABLE_QUIRK_ARM_S2FWB BIT(10)
#define IO_PGTABLE_QUIRK_HISI_ERRATA BIT(11)
+ #define IO_PGTABLE_QUIRK_ARM_NO_CONT_HINT BIT(12)
unsigned long quirks;
unsigned long pgsize_bitmap;
unsigned int ias;
--
2.22.0
