From: Nicolin Chen nicolinc@nvidia.com
Add arm_smmu_cache_invalidate_user() function for user space to invalidate TLB entries and Context Descriptors, since either an stage-1 IO page table entry or a Context Descriptor Table entry in user space is still cached by the hardware.
Add struct iommu_hwpt_arm_smmuv3_invalidate defining the format of all the entries in an invalidation request array. This format is simply the native format of a 128-bit TLB/Cache invalidation command. Each command should be scanned against the supported command list, and fixed at the SID and VMID fields by replacing with the correct physical values.
Also, a guest CMDQ might not be prepared by a Linux kernel driver, so the handler must take care of situations that might be impacted by an erratum.
Co-developed-by: Eric Auger eric.auger@redhat.com Signed-off-by: Eric Auger eric.auger@redhat.com Signed-off-by: Nicolin Chen nicolinc@nvidia.com Signed-off-by: Kunkun Jiang jiangkunkun@huawei.com --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 149 ++++++++++++++++++++ include/uapi/linux/iommufd.h | 20 +++ 2 files changed, 169 insertions(+)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4ddb6d48dde7..ef0fe6d07e86 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3375,10 +3375,159 @@ arm_smmu_get_msi_mapping_domain(struct iommu_domain *domain) return &nested_domain->s2_parent->domain; }
+static int arm_smmu_fix_user_cmd(struct arm_smmu_nested_domain *nested_domain, + u64 *cmd) +{ + struct arm_smmu_device *smmu = nested_domain->s2_parent->smmu; + struct arm_smmu_stream *stream; + + cmd[0] = le64_to_cpu(cmd[0]); + cmd[1] = le64_to_cpu(cmd[1]); + + switch (*cmd & CMDQ_0_OP) { + case CMDQ_OP_TLBI_NSNH_ALL: + *cmd &= ~CMDQ_0_OP; + *cmd |= CMDQ_OP_TLBI_NH_ALL; + fallthrough; + case CMDQ_OP_TLBI_NH_VA: + case CMDQ_OP_TLBI_NH_VAA: + case CMDQ_OP_TLBI_NH_ALL: + case CMDQ_OP_TLBI_NH_ASID: + *cmd &= ~CMDQ_TLBI_0_VMID; + *cmd |= FIELD_PREP(CMDQ_TLBI_0_VMID, + nested_domain->s2_parent->vmid); + break; + case CMDQ_OP_ATC_INV: + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_CD_ALL: + xa_lock(&smmu->streams_user); + stream = xa_load(&smmu->streams_user, + FIELD_GET(CMDQ_CFGI_0_SID, *cmd)); + xa_unlock(&smmu->streams_user); + if (!stream) + return -ENODEV; + *cmd &= ~CMDQ_CFGI_0_SID; + *cmd |= FIELD_PREP(CMDQ_CFGI_0_SID, stream->id); + break; + default: + return -EINVAL; + } + pr_debug("Fixed user CMD: %016llx : %016llx\n", cmd[1], cmd[0]); + + return 0; +} + +/* + * A helper function to detect if the current cmd hits Arm erratum before being + * added to a batch that might already contain a leaf TLBI or a CFGI command. + * + * If there is a hit, it will update the corresponding boolean flag, by assuming + * that the caller must issue the batch without this cmd. So, the two flags then + * will reflect the status of the new batch that contains the current cmd only. + */ +static bool arm_smmu_cmdq_hit_errata(struct arm_smmu_device *smmu, u64 *cmd, + bool *batch_has_leaf, bool *batch_has_cfgi) +{ + bool cmd_has_leaf; + + if (!(smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC)) + return false; + + switch (*cmd & CMDQ_0_OP) { + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_CD_ALL: + *batch_has_cfgi = true; + break; + case CMDQ_OP_TLBI_NH_VA: + case CMDQ_OP_TLBI_NH_VAA: + cmd_has_leaf = FIELD_GET(CMDQ_TLBI_1_LEAF, cmd[1]); + /* Erratum 2812531 -- a non-leaf tlbi following a leaf tlbi */ + if (!cmd_has_leaf && *batch_has_leaf) { + *batch_has_leaf = false; + return true; + } + if (cmd_has_leaf) + *batch_has_leaf = true; + fallthrough; + case CMDQ_OP_TLBI_NH_ALL: + case CMDQ_OP_TLBI_NH_ASID: + /* Erratum 2812531 -- a tlbi following a cfgi */ + if (*batch_has_cfgi) { + *batch_has_cfgi = false; + return true; + } + break; + } + + return false; +} + +static int arm_smmu_cache_invalidate_user(struct iommu_domain *domain, + struct iommu_user_data_array *array) +{ + struct arm_smmu_nested_domain *nested_domain = + container_of(domain, struct arm_smmu_nested_domain, domain); + struct arm_smmu_device *smmu = nested_domain->s2_parent->smmu; + bool has_leaf = false, has_cfgi = false; + int data_idx, n = 0, ret; + u64 *cmds; + + if (!smmu) + return -EINVAL; + if (!array->entry_num) + return -EINVAL; + + cmds = kcalloc(array->entry_num, sizeof(*cmds) * 2, GFP_KERNEL); + if (!cmds) + return -ENOMEM; + + for (data_idx = 0; data_idx < array->entry_num; data_idx++) { + struct iommu_hwpt_arm_smmuv3_invalidate *inv = + (struct iommu_hwpt_arm_smmuv3_invalidate *)&cmds[n * 2]; + + ret = iommu_copy_struct_from_user_array(inv, array, + IOMMU_HWPT_DATA_ARM_SMMUV3, + data_idx, cmd); + if (ret) + goto out; + + ret = arm_smmu_fix_user_cmd(nested_domain, inv->cmd); + if (ret) + goto out; + + if (arm_smmu_cmdq_hit_errata(smmu, inv->cmd, &has_leaf, &has_cfgi)) { + /* WAR is to issue the batch prior, with a CMD_SYNC */ + ret = arm_smmu_cmdq_issue_cmdlist(smmu, cmds, n, true); + if (ret) + goto out; + /* Then move the cmd to the head for a new batch */ + cmds[1] = cmds[n * 2 + 1]; + cmds[0] = cmds[n * 2]; + n = 1; + continue; + } + + if (++n == CMDQ_BATCH_ENTRIES - 1) { + ret = arm_smmu_cmdq_issue_cmdlist(smmu, cmds, n, true); + if (ret) + goto out; + n = 0; + } + } + + if (n) + ret = arm_smmu_cmdq_issue_cmdlist(smmu, cmds, n, true); +out: + array->entry_num = data_idx; + kfree(cmds); + return ret; +} + static const struct iommu_domain_ops arm_smmu_nested_ops = { .attach_dev = arm_smmu_attach_dev_nested, .free = arm_smmu_domain_nested_free, .get_msi_mapping_domain = arm_smmu_get_msi_mapping_domain, + .cache_invalidate_user = arm_smmu_cache_invalidate_user, };
static struct iommu_domain * diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index cf13ab623b4a..aa57a6195d89 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -721,6 +721,26 @@ struct iommu_hwpt_vtd_s1_invalidate { __u32 __reserved; };
+/** + * struct iommu_hwpt_arm_smmuv3_invalidate - ARM SMMUv3 cahce invalidation + * (IOMMU_HWPT_DATA_ARM_SMMUV3) + * @cmd: 128-bit cache invalidation command that runs in SMMU CMDQ. + * Must be little-endian. + * + * Supported command list: + * CMDQ_OP_TLBI_NSNH_ALL + * CMDQ_OP_TLBI_NH_VA + * CMDQ_OP_TLBI_NH_VAA + * CMDQ_OP_TLBI_NH_ALL + * CMDQ_OP_TLBI_NH_ASID + * CMDQ_OP_ATC_INV + * CMDQ_OP_CFGI_CD + * CMDQ_OP_CFGI_CD_ALL + */ +struct iommu_hwpt_arm_smmuv3_invalidate { + __aligned_u64 cmd[2]; +}; + /** * struct iommu_hwpt_invalidate - ioctl(IOMMU_HWPT_INVALIDATE) * @size: sizeof(struct iommu_hwpt_invalidate)