From: Xiangkai Yang yangxiangkai@huawei.com
virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IADD42
--------------------------------
virtCCA feature signed-off-by: Xiangkai Yang yangxiangkai@huawei.com --- arch/arm64/include/asm/kvm_tmi.h | 88 +- arch/arm64/include/asm/kvm_tmm.h | 22 + arch/arm64/kvm/tmi.c | 152 +++- arch/arm64/kvm/virtcca_cvm.c | 10 +- drivers/iommu/arm/arm-smmu-v3/Makefile | 1 + drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c | 811 ++++++++++++++++++ drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h | 160 ++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 175 +++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 16 + include/linux/iommu.h | 4 + include/linux/iopoll.h | 38 + 11 files changed, 1471 insertions(+), 6 deletions(-) create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h
diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index fc4fe9a711..a18848f70b 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -11,8 +11,6 @@ #include <linux/virtio_ring.h> #include <asm/sysreg.h>
-#define GRANULE_SIZE 4096 - #define NO_NUMA 0 /* numa bitmap */
#define TMM_TTT_LEVEL_2 2 @@ -68,6 +66,39 @@ struct tmi_tec_params { uint64_t ram_size; };
+struct tmi_smmu_ste_params { + uint64_t ns_src; /* non-secure STE source address */ + uint64_t sid; /* stream id */ + uint64_t smmu_id; /* smmu id */ +}; + +struct tmi_smmu_cfg_params { + uint64_t smmu_id; + uint64_t ioaddr; + uint8_t strtab_base_RA_bit : 1; + uint8_t q_base_RA_WA_bit : 1; + uint8_t is_cmd_queue : 1; +}; + +#define TMI_SMMU_CMD_QUEUE 1 +#define TMI_SMMU_EVT_QUEUE 2 +struct tmi_smmu_queue_params { + uint64_t ns_src; /* non-secure queue source address */ + uint64_t smmu_base_addr; /* smmu base address */ + uint64_t size; /* queue size */ + uint64_t smmu_id; /* smmu id */ + uint64_t type; /* cmdq or evtq */ +}; + +#define MAX_DEV_PER_PORT 256 +struct tmi_dev_delegate_params { + /* BDF of PCIe root bus, F=0. BD are used to calculate APB base and port number. */ + uint16_t root_bd; + uint16_t num_dev; /* number of attachable devices */ + uint32_t _reserved; /* padding for 64-bit alignment */ + uint16_t devs[MAX_DEV_PER_PORT]; /* BDF of each attachable device */ +}; + #define TEC_ENTRY_FLAG_EMUL_MMIO (1UL << 0U) #define TEC_ENTRY_FLAG_INJECT_SEA (1UL << 1U) #define TEC_ENTRY_FLAG_TRAP_WFI (1UL << 2U) @@ -202,6 +233,21 @@ struct tmi_tec_run { #define TMI_FNUM_TTT_UNMAP_RANGE U(0x26E) #define TMI_FNUM_INF_TEST U(0x270)
+#define TMI_FNUM_SMMU_QUEUE_CREATE U(0x277) +#define TMI_FNUM_SMMU_QUEUE_WRITE U(0x278) +#define TMI_FNUM_SMMU_STE_CREATE U(0x279) +#define TMI_FNUM_MMIO_MAP U(0x27A) +#define TMI_FNUM_MMIO_UNMAP U(0x27B) +#define TMI_FNUM_MMIO_WRITE U(0x27C) +#define TMI_FNUM_MMIO_READ U(0x27D) +#define TMI_FNUM_DEV_DELEGATE U(0x27E) +#define TMI_FNUM_DEV_ATTACH U(0x27F) +#define TMI_FNUM_HANDLE_S_EVTQ U(0x280) +#define TMI_FNUM_SMMU_DEVICE_RESET U(0x281) +#define TMI_FNUM_SMMU_WRITE U(0x282) +#define TMI_FNUM_SMMU_READ U(0x283) +#define TMI_FNUM_SMMU_PCIE_CORE_CHECK U(0x284) + /* TMI SMC64 PIDs handled by the SPMD */ #define TMI_TMM_VERSION_REQ TMI_FID(SMC_64, TMI_FNUM_VERSION_REQ) #define TMI_TMM_DATA_CREATE TMI_FID(SMC_64, TMI_FNUM_DATA_CREATE) @@ -220,6 +266,21 @@ struct tmi_tec_run { #define TMI_TMM_TTT_UNMAP_RANGE TMI_FID(SMC_64, TMI_FNUM_TTT_UNMAP_RANGE) #define TMI_TMM_INF_TEST TMI_FID(SMC_64, TMI_FNUM_INF_TEST)
+#define TMI_TMM_SMMU_QUEUE_CREATE TMI_FID(SMC_64, TMI_FNUM_SMMU_QUEUE_CREATE) +#define TMI_TMM_SMMU_QUEUE_WRITE TMI_FID(SMC_64, TMI_FNUM_SMMU_QUEUE_WRITE) +#define TMI_TMM_SMMU_STE_CREATE TMI_FID(SMC_64, TMI_FNUM_SMMU_STE_CREATE) +#define TMI_TMM_MMIO_MAP TMI_FID(SMC_64, TMI_FNUM_MMIO_MAP) +#define TMI_TMM_MMIO_UNMAP TMI_FID(SMC_64, TMI_FNUM_MMIO_UNMAP) +#define TMI_TMM_MMIO_WRITE TMI_FID(SMC_64, TMI_FNUM_MMIO_WRITE) +#define TMI_TMM_MMIO_READ TMI_FID(SMC_64, TMI_FNUM_MMIO_READ) +#define TMI_TMM_DEV_DELEGATE TMI_FID(SMC_64, TMI_FNUM_DEV_DELEGATE) +#define TMI_TMM_DEV_ATTACH TMI_FID(SMC_64, TMI_FNUM_DEV_ATTACH) +#define TMI_TMM_HANDLE_S_EVTQ TMI_FID(SMC_64, TMI_FNUM_HANDLE_S_EVTQ) +#define TMI_TMM_SMMU_DEVICE_RESET TMI_FID(SMC_64, TMI_FNUM_SMMU_DEVICE_RESET) +#define TMI_TMM_SMMU_WRITE TMI_FID(SMC_64, TMI_FNUM_SMMU_WRITE) +#define TMI_TMM_SMMU_READ TMI_FID(SMC_64, TMI_FNUM_SMMU_READ) +#define TMI_TMM_SMMU_PCIE_CORE_CHECK TMI_FID(SMC_64, TMI_FNUM_SMMU_PCIE_CORE_CHECK) + #define TMI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) #define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF)
@@ -321,6 +382,22 @@ u64 tmi_ttt_map_range(u64 rd, u64 map_addr, u64 size, u64 cur_node, u64 target_n u64 tmi_ttt_unmap_range(u64 rd, u64 map_addr, u64 size, u64 node_id); u64 tmi_mem_info_show(u64 mem_info_addr);
+u64 va_to_pa(void *addr); +u64 tmi_smmu_queue_create(u64 params_ptr); +u64 tmi_smmu_queue_write(uint64_t cmd0, uint64_t cmd1, u64 smmu_id); +u64 tmi_smmu_ste_create(u64 params_ptr); +u64 tmi_mmio_map(u64 rd, u64 map_addr, u64 level, u64 ttte); +u64 tmi_mmio_unmap(u64 rd, u64 map_addr, u64 level); +u64 tmi_mmio_write(u64 addr, u64 val, u64 bits, u64 dev_num); +u64 tmi_mmio_read(u64 addr, u64 bits, u64 dev_num); +u64 tmi_dev_delegate(u64 params); +u64 tmi_dev_attach(u64 vdev, u64 rd, u64 smmu_id); +u64 tmi_handle_s_evtq(u64 smmu_id); +u64 tmi_smmu_device_reset(u64 params); +u64 tmi_smmu_pcie_core_check(u64 smmu_base); +u64 tmi_smmu_write(u64 smmu_base, u64 reg_offset, u64 val, u64 bits); +u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits); + void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu); int kvm_load_user_data(struct kvm *kvm, unsigned long arg); unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, @@ -329,5 +406,12 @@ int kvm_cvm_vcpu_set_events(struct kvm_vcpu *vcpu, bool serror_pending, bool ext_dabt_pending); int kvm_init_cvm_vm(struct kvm *kvm); int kvm_enable_virtcca_cvm(struct kvm *kvm); + +bool virtcca_is_available(void); +#else +static inline bool virtcca_is_available(void) +{ + return false; +} #endif #endif diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index 96532ca8fd..2fd6d2bdb9 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -7,6 +7,9 @@
#include <uapi/linux/kvm.h>
+#define CVM_MSI_ORIG_IOVA 0x8000000 +#define CVM_MSI_IOVA_OFFSET (-0x1000000) + enum virtcca_cvm_state { CVM_STATE_NONE = 1, CVM_STATE_NEW, @@ -59,6 +62,7 @@ struct virtcca_cvm { u64 ram_size; struct kvm_numa_info numa_info; struct tmi_cvm_params *params; + bool is_mapped; };
/* @@ -70,6 +74,13 @@ struct virtcca_cvm_tec { void *tec_run; };
+struct cvm_ttt_addr { + struct list_head list; + u64 addr; +}; + +struct iommu_group {}; + int kvm_init_tmm(void); int kvm_cvm_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap); void kvm_destroy_cvm(struct kvm *kvm); @@ -83,9 +94,20 @@ int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target);
void kvm_cvm_unmap_destroy_range(struct kvm *kvm);
+int kvm_cvm_map_range(struct kvm *kvm); +int cvm_arm_smmu_domain_set_kvm(struct iommu_group *group); +int kvm_cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, + unsigned long map_size, uint32_t is_map); +int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size); + #define CVM_TTT_BLOCK_LEVEL 2 #define CVM_TTT_MAX_LEVEL 3
+#define CVM_MAP_IPA_RAM 1 +#define CVM_MAP_IPA_SMMU 2 +#define CVM_MAP_IPA_UNPROTECTED 4 + #define CVM_PAGE_SHIFT 12 #define CVM_PAGE_SIZE BIT(CVM_PAGE_SHIFT) #define CVM_TTT_LEVEL_SHIFT(l) \ diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index c1f22139d7..366d9200ad 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -6,6 +6,30 @@ #include <asm/kvm_tmi.h> #include <asm/memory.h>
+u64 va_to_pa(void *addr) +{ + uint64_t pa, par_el1; + + asm volatile( + "AT S1E1W, %0\n" + ::"r"((uint64_t)(addr)) + ); + isb(); + asm volatile( + "mrs %0, par_el1\n" + : "=r"(par_el1) + ); + + pa = ((uint64_t)(addr) & (PAGE_SIZE - 1)) | + (par_el1 & ULL(0x000ffffffffff000)); + + if (par_el1 & UL(1 << 0)) + return (uint64_t)(addr); + else + return pa; +} +EXPORT_SYMBOL(va_to_pa); + u64 tmi_version(void) { struct arm_smccc_res res; @@ -138,4 +162,130 @@ u64 tmi_tmm_inf_test(u64 x1, u64 x2, u64 x3, u64 x4, u64 x5) arm_smccc_1_1_smc(TMI_TMM_INF_TEST, x1, vttbr_el2_pa, cvm_params_pa, tec_params_pa, x5, &res); return res.a1; } -EXPORT_SYMBOL_GPL(tmi_tmm_inf_test); \ No newline at end of file +EXPORT_SYMBOL_GPL(tmi_tmm_inf_test); + +u64 tmi_smmu_queue_create(u64 params_ptr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_QUEUE_CREATE, params_ptr, &res); + return res.a1; +} +EXPORT_SYMBOL_GPL(tmi_smmu_queue_create); + + +u64 tmi_smmu_queue_write(uint64_t cmd0, uint64_t cmd1, u64 smmu_id) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_QUEUE_WRITE, cmd0, cmd1, smmu_id, &res); + return res.a1; +} +EXPORT_SYMBOL_GPL(tmi_smmu_queue_write); + +u64 tmi_smmu_ste_create(u64 params_ptr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_STE_CREATE, params_ptr, &res); + return res.a1; +} +EXPORT_SYMBOL_GPL(tmi_smmu_ste_create); + +u64 tmi_mmio_map(u64 rd, u64 map_addr, u64 level, u64 ttte) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_MAP, rd, map_addr, level, ttte, &res); + return res.a1; +} + +u64 tmi_mmio_unmap(u64 rd, u64 map_addr, u64 level) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_UNMAP, rd, map_addr, level, &res); + return res.a1; +} + +u64 tmi_mmio_write(u64 addr, u64 val, u64 bits, u64 dev_num) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_WRITE, addr, val, bits, dev_num, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_mmio_write); + +u64 tmi_mmio_read(u64 addr, u64 bits, u64 dev_num) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_READ, addr, bits, dev_num, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_mmio_read); + +u64 tmi_dev_delegate(u64 params) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DEV_DELEGATE, params, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_dev_delegate); + +u64 tmi_dev_attach(u64 vdev, u64 rd, u64 smmu_id) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DEV_ATTACH, vdev, rd, smmu_id, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_dev_attach); + +u64 tmi_handle_s_evtq(u64 smmu_id) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_HANDLE_S_EVTQ, smmu_id, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_handle_s_evtq); + +u64 tmi_smmu_device_reset(u64 params) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_DEVICE_RESET, params, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_smmu_device_reset); + +u64 tmi_smmu_pcie_core_check(u64 smmu_base) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_PCIE_CORE_CHECK, smmu_base, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_smmu_pcie_core_check); + +u64 tmi_smmu_write(u64 smmu_base, u64 reg_offset, u64 val, u64 bits) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_WRITE, smmu_base, reg_offset, val, bits, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_smmu_write); + +u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_READ, smmu_base, reg_offset, bits, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_smmu_read); + diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 367bbf4fa3..73e61528a7 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -521,7 +521,7 @@ static int kvm_tmm_config_cvm(struct kvm *kvm, struct kvm_enable_cap *cap) return r; }
-static int kvm_cvm_map_range(struct kvm *kvm) +int kvm_cvm_map_range(struct kvm *kvm) { int ret; u64 curr_numa_set; @@ -887,3 +887,11 @@ int kvm_init_cvm_vm(struct kvm *kvm)
return 0; } + +bool virtcca_is_available(void) +{ + if (static_branch_unlikely(&virtcca_cvm_is_available)) + return true; + return false; +} +EXPORT_SYMBOL(virtcca_is_available); diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index 54feb1eccc..e4020cafcc 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-objs-y += arm-smmu-v3.o +arm_smmu_v3-objs-$(CONFIG_HISI_VIRTCCA_HOST) += arm-s-smmu-v3.o arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c new file mode 100644 index 0000000000..a877fe23c4 --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c @@ -0,0 +1,811 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#include <linux/crash_dump.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/iopoll.h> +#include <linux/pci.h> +#include <linux/hashtable.h> +#include <asm/kvm_tmi.h> + +#include "arm-smmu-v3.h" +#include "arm-s-smmu-v3.h" +#include "../../dma-iommu.h" + +struct cc_dev_config { + u32 sid; /* BDF number of the device */ + u32 vmid; + u32 root_bd; /* root bus and device number. Multiple sid can have the same root_bd. */ + bool secure; + struct hlist_node node; +}; + +static bool g_smmu_id_map_init; + +static DEFINE_HASHTABLE(g_cc_dev_htable, MAX_CC_DEV_NUM_ORDER); +static DECLARE_BITMAP(g_smmu_id_map, ARM_SMMU_MAX_IDS); + +/* Traverse pcie topology to find the root <bus,device> number + * return -1 if error + * return -1 if not pci device + */ +int get_root_bd(struct device *dev) +{ + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) + return -1; + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return -1; + while (pdev->bus->parent != NULL) + pdev = pdev->bus->self; + + return pci_dev_id(pdev) & 0xfff8; +} + +/* if dev is a bridge, get all it's children. + * if dev is a regular device, get itself. + */ +void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, int max_devs, int *ndev) +{ + struct pci_bus *bus = dev->subordinate; + + if (bus) { /* dev is a bridge */ + struct pci_dev *child; + + list_for_each_entry(child, &bus->devices, bus_list) { + get_child_devices_rec(child, devs, max_devs, ndev); + } + } else { /* dev is a regular device */ + uint16_t bdf = pci_dev_id(dev); + int i; + /* check if bdf is already in devs */ + for (i = 0; i < *ndev; i++) { + if (devs[i] == bdf) + return; + } + /* check overflow */ + if (*ndev >= max_devs) { + WARN_ON(1); + return; + } + devs[*ndev] = bdf; + *ndev = *ndev + 1; + } +} + +/* get all devices which share the same root_bd as dev + * return 0 on failure + */ +int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs) +{ + int ndev = 0; + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) + return ndev; + + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return ndev; + + while (pdev->bus->parent != NULL) + pdev = pdev->bus->self; + + get_child_devices_rec(pdev, devs, max_devs, &ndev); + return ndev; +} + +void set_g_cc_dev(u32 sid, u32 vmid, u32 root_bd, bool secure) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj->sid == sid) { + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + return; + } + } + + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) { + WARN_ON(1); + return; + } + + obj->sid = sid; + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + + hash_add(g_cc_dev_htable, &obj->node, sid); +} + +void free_g_cc_dev_htable(void) +{ + int i; + struct cc_dev_config *obj; + struct hlist_node *tmp; + + hash_for_each_safe(g_cc_dev_htable, i, tmp, obj, node) { + hash_del(&obj->node); + kfree(obj); + } +} + +/* Has the root bus device number switched to secure? */ +bool is_cc_root_bd(u32 root_bd) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (obj->root_bd == root_bd && obj->secure) + return true; + } + + return false; +} + +static bool is_cc_vmid(u32 vmid) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (vmid > 0 && obj->vmid == vmid) + return true; + } + + return false; +} + +bool is_cc_dev(u32 sid) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid) + return obj->secure; + } + + return false; +} +EXPORT_SYMBOL(is_cc_dev); + +void s_smmu_cmdq_need_forward(u64 cmd0, u64 cmd1, u64 *forward) +{ + u64 opcode = FIELD_GET(CMDQ_0_OP, cmd0); + + switch (opcode) { + case CMDQ_OP_TLBI_EL2_ALL: + case CMDQ_OP_TLBI_NSNH_ALL: + *forward = 1; + break; + case CMDQ_OP_PREFETCH_CFG: + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_STE: + case CMDQ_OP_CFGI_CD_ALL: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_CFGI_0_SID, cmd0)); + break; + + case CMDQ_OP_CFGI_ALL: + *forward = 1; + break; + case CMDQ_OP_TLBI_NH_VA: + case CMDQ_OP_TLBI_S2_IPA: + case CMDQ_OP_TLBI_NH_ASID: + case CMDQ_OP_TLBI_S12_VMALL: + *forward = (uint64_t)is_cc_vmid(FIELD_GET(CMDQ_TLBI_0_VMID, cmd0)); + break; + case CMDQ_OP_TLBI_EL2_VA: + case CMDQ_OP_TLBI_EL2_ASID: + *forward = 0; + break; + case CMDQ_OP_ATC_INV: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_ATC_0_SID, cmd0)); + break; + case CMDQ_OP_PRI_RESP: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_PRI_0_SID, cmd0)); + break; + case CMDQ_OP_RESUME: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_RESUME_0_SID, cmd0)); + break; + case CMDQ_OP_CMD_SYNC: + *forward = 0; + break; + default: + *forward = 0; + } +} + +void s_queue_write(struct arm_smmu_device *smmu, u64 *src, size_t n_dwords) +{ + u64 cmd0, cmd1; + u64 forward = 0; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + if (n_dwords == 2) { + cmd0 = cpu_to_le64(src[0]); + cmd1 = cpu_to_le64(src[1]); + s_smmu_cmdq_need_forward(cmd0, cmd1, &forward); + + /* need forward queue command to TMM */ + if (forward) { + if (tmi_smmu_queue_write(cmd0, cmd1, smmu->id)) + pr_err("tmi_smmu_queue_write err!\n"); + } + } + } +} + +void arm_s_smmu_cmdq_write_entries(struct arm_smmu_device *smmu, u64 *cmds, int n) +{ + int i; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + for (i = 0; i < n; ++i) { + u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS]; + + s_queue_write(smmu, cmd, CMDQ_ENT_DWORDS); + } + } +} + +int arm_s_smmu_init_one_queue(struct arm_smmu_device *smmu, + struct arm_smmu_queue *q, + size_t qsz, const char *name) +{ + if (smmu->id != ARM_SMMU_INVALID_ID) { + struct tmi_smmu_queue_params *params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + + if (!params_ptr) + return -ENOMEM; + + if (!strcmp(name, "cmdq")) { + params_ptr->ns_src = q->base_dma; + params_ptr->smmu_base_addr = smmu->ioaddr; + params_ptr->size = qsz; + params_ptr->smmu_id = smmu->id; + params_ptr->type = TMI_SMMU_CMD_QUEUE; + tmi_smmu_queue_create(__pa(params_ptr)); + } + + if (!strcmp(name, "evtq")) { + params_ptr->ns_src = q->base_dma; + params_ptr->smmu_base_addr = smmu->ioaddr; + params_ptr->size = qsz; + params_ptr->smmu_id = smmu->id; + params_ptr->type = TMI_SMMU_EVT_QUEUE; + tmi_smmu_queue_create(__pa(params_ptr)); + } + + kfree(params_ptr); + } + + return 0; +} + +int arm_smmu_write_s_reg_sync(struct arm_smmu_device *smmu, u32 val, u32 cmp_val, + unsigned int reg_off, unsigned int ack_off) +{ + u32 reg; + + if (tmi_smmu_write(smmu->ioaddr, reg_off, val, 32)) + return -ENXIO; + + return kvm_cvm_read_poll_timeout_atomic(tmi_smmu_read, reg, reg == cmp_val, + 1, ARM_SMMU_POLL_TIMEOUT_US, false, + smmu->ioaddr, ack_off, 32); +} + +int arm_smmu_update_s_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr) +{ + int ret; + u32 reg; + + ret = kvm_cvm_read_poll_timeout_atomic(tmi_smmu_read, reg, !(reg & S_GBPA_UPDATE), + 1, ARM_SMMU_POLL_TIMEOUT_US, false, + smmu->ioaddr, ARM_SMMU_S_GBPA, 32); + if (ret) + return ret; + + reg &= ~clr; + reg |= set; + + ret = tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_GBPA, reg | S_GBPA_UPDATE, 32); + if (ret) + return ret; + + ret = kvm_cvm_read_poll_timeout_atomic(tmi_smmu_read, reg, !(reg & S_GBPA_UPDATE), + 1, ARM_SMMU_POLL_TIMEOUT_US, false, + smmu->ioaddr, ARM_SMMU_S_GBPA, 32); + if (ret) + dev_err(smmu->dev, "S_GBPA not responding to update\n"); + return ret; +} + +irqreturn_t arm_smmu_s_evtq_thread(int irq, void *dev) +{ + struct arm_smmu_device *smmu = dev; + + if (smmu->id != ARM_SMMU_INVALID_ID) + tmi_handle_s_evtq(smmu->id); + + return IRQ_HANDLED; +} + +irqreturn_t arm_smmu_s_gerror_handler(int irq, void *dev) +{ + u32 gerror, gerrorn, active; + u64 ret; + struct arm_smmu_device *smmu = dev; + + ret = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_GERROR, 32); + if (ret >> 32) { + dev_err(smmu->dev, "Get ARM_SMMU_S_GERROR register failed\n"); + return IRQ_NONE; + } + gerror = (u32)ret; + + ret = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_GERRORN, 32); + if (ret >> 32) { + dev_err(smmu->dev, "Get ARM_SMMU_S_GERRORN register failed\n"); + return IRQ_NONE; + } + gerrorn = (u32)ret; + + active = gerror ^ gerrorn; + if (!(active & GERROR_ERR_MASK)) + return IRQ_NONE; /* No errors pending */ + + dev_warn(smmu->dev, + "unexpected secure global error reported, this could be serious, active %x\n", + active); + + if (active & GERROR_SFM_ERR) { + dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); + arm_s_smmu_device_disable(smmu); + } + + if (active & GERROR_MSI_GERROR_ABT_ERR) + dev_warn(smmu->dev, "GERROR MSI write aborted\n"); + + if (active & GERROR_MSI_PRIQ_ABT_ERR) + dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); + + if (active & GERROR_MSI_EVTQ_ABT_ERR) + dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); + + if (active & GERROR_MSI_CMDQ_ABT_ERR) + dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); + + if (active & GERROR_PRIQ_ABT_ERR) + dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); + + if (active & GERROR_EVTQ_ABT_ERR) + dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); + + if (active & GERROR_CMDQ_ERR) + dev_warn(smmu->dev, "CMDQ ERR\n"); + + if (tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_GERRORN, gerror, 32)) { + dev_err(smmu->dev, "SMMU write ARM_SMMU_S_GERRORN failed\n"); + return IRQ_NONE; + } + + return IRQ_HANDLED; +} + +int arm_smmu_disable_s_irq(struct arm_smmu_device *smmu) +{ + int ret; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + ret = arm_smmu_write_s_reg_sync(smmu, 0, 0, + ARM_SMMU_S_IRQ_CTRL, ARM_SMMU_S_IRQ_CTRLACK); + if (ret) { + dev_err(smmu->dev, "failed to disable secure irqs\n"); + return ret; + } + } + + return 0; +} + +int arm_smmu_enable_s_irq(struct arm_smmu_device *smmu, u32 irqen_flags) +{ + int ret; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + ret = arm_smmu_write_s_reg_sync(smmu, irqen_flags, + irqen_flags, ARM_SMMU_S_IRQ_CTRL, ARM_SMMU_S_IRQ_CTRLACK); + if (ret) { + dev_err(smmu->dev, "failed to enable irq for secure evtq\n"); + return ret; + } + } + + return 0; +} + +void arm_s_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) +{ + int irq, ret; + + irq = smmu->s_evtq_irq; + if (irq && smmu->id != ARM_SMMU_INVALID_ID) { + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + arm_smmu_s_evtq_thread, + IRQF_ONESHOT, + "arm-smmu-v3-s_evtq", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable s_evtq irq\n"); + } else { + dev_warn(smmu->dev, "no s_evtq irq - events will not be reported!\n"); + } + + irq = smmu->s_gerr_irq; + if (irq && smmu->id != ARM_SMMU_INVALID_ID) { + ret = devm_request_irq(smmu->dev, irq, arm_smmu_s_gerror_handler, + 0, "arm-smmu-v3-s_gerror", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable s_gerror irq\n"); + } else { + dev_warn(smmu->dev, "no s_gerr irq - errors will not be reported!\n"); + } +} + +void arm_smmu_write_s_msi_msg(struct arm_smmu_device *smmu, phys_addr_t *cfg, + struct msi_msg *msg, phys_addr_t doorbell) +{ + tmi_smmu_write((u64)smmu->ioaddr, cfg[0], doorbell, 64); + tmi_smmu_write((u64)smmu->ioaddr, cfg[1], msg->data, 32); + tmi_smmu_write((u64)smmu->ioaddr, cfg[2], ARM_SMMU_MEMATTR_DEVICE_nGnRE, 32); +} + +void platform_get_s_irq_byname_optional(struct platform_device *pdev, struct arm_smmu_device *smmu) +{ + int irq; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + irq = platform_get_irq_byname_optional(pdev, "s_eventq"); + if (irq > 0) + smmu->s_evtq_irq = irq; + + irq = platform_get_irq_byname_optional(pdev, "s_gerror"); + if (irq > 0) + smmu->s_gerr_irq = irq; + } +} + +int arm_smmu_enable_secure(struct iommu_domain *domain) +{ + int ret = 0; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + mutex_lock(&smmu_domain->init_mutex); + if (smmu_domain->smmu) + ret = -EPERM; + else + smmu_domain->secure = true; + mutex_unlock(&smmu_domain->init_mutex); + + return ret; +} + +u32 arm_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, + struct kvm *kvm) +{ + int ret = -1; + u64 cmd[CMDQ_ENT_DWORDS] = {0}; + unsigned long flags; + int i, j; + struct arm_smmu_master *master; + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + + spin_lock_irqsave(&arm_smmu_domain->devices_lock, flags); + list_for_each_entry(master, &arm_smmu_domain->devices, domain_head) { + if (master && master->num_streams >= 0) { + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + ret = tmi_dev_attach(sid, virtcca_cvm->rd, + arm_smmu_domain->smmu->id); + if (ret) { + kvm_err("dev protected failed!\n"); + ret = -ENXIO; + goto out; + } + cmd[0] |= FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CFGI_STE); + cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); + cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, true); + tmi_smmu_queue_write(cmd[0], cmd[1], arm_smmu_domain->smmu->id); + } + } + } + +out: + spin_unlock_irqrestore(&arm_smmu_domain->devices_lock, flags); + return ret; +} + +int arm_smmu_secure_dev_ste_create(struct arm_smmu_device *smmu, + struct arm_smmu_master *master, u32 sid) +{ + struct tmi_smmu_ste_params *params_ptr; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; + + params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!params_ptr) + return -ENOMEM; + + /* Sync Level 2 STE to TMM */ + params_ptr->ns_src = desc->l2ptr_dma + ((sid & ((1 << STRTAB_SPLIT) - 1)) * STE_ENTRY_SIZE); + params_ptr->sid = sid; + params_ptr->smmu_id = smmu->id; + + if (tmi_smmu_ste_create(__pa(params_ptr)) != 0) + dev_err(smmu->dev, "failed to create STE level 2"); + + kfree(params_ptr); + + return 0; +} + +int arm_smmu_secure_set_dev(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master *master, + struct device *dev) +{ + int i, j; + u64 ret = 0; + uint16_t root_bd = get_root_bd(dev); + + WARN_ON_ONCE(root_bd < 0); + if (!is_cc_root_bd(root_bd)) { + struct tmi_dev_delegate_params *params = kzalloc( + sizeof(struct tmi_dev_delegate_params), GFP_KERNEL); + + params->root_bd = root_bd; + params->num_dev = get_sibling_devices(dev, params->devs, MAX_DEV_PER_PORT); + WARN_ON_ONCE(params->num_dev == 0); + pr_info("Delegate %d devices as %02x:%02x to secure\n", + params->num_dev, root_bd >> 8, (root_bd & 0xff) >> 3); + ret = tmi_dev_delegate(__pa(params)); + if (!ret) { + for (i = 0; i < params->num_dev; i++) + set_g_cc_dev(params->devs[i], 0, root_bd, true); + } + kfree(params); + } + + if (ret) + return ret; + + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + WARN_ON_ONCE(!is_cc_dev(sid)); + set_g_cc_dev(sid, smmu_domain->s2_cfg.vmid, root_bd, true); + } + + return ret; +} + +int arm_smmu_id_alloc(void) +{ + int idx; + + do { + idx = find_first_zero_bit(g_smmu_id_map, ARM_SMMU_MAX_IDS); + if (idx == ARM_SMMU_MAX_IDS) + return -ENOSPC; + } while (test_and_set_bit(idx, g_smmu_id_map)); + + return idx; +} + +void arm_smmu_id_free(int idx) +{ + if (idx != ARM_SMMU_INVALID_ID) + clear_bit(idx, g_smmu_id_map); +} + +void arm_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr) +{ + if (!g_smmu_id_map_init) { + set_bit(0, g_smmu_id_map); + g_smmu_id_map_init = true; + } + smmu->ioaddr = ioaddr; + + if (virtcca_is_available() && tmi_smmu_pcie_core_check(ioaddr)) + smmu->id = arm_smmu_id_alloc(); + else + smmu->id = ARM_SMMU_INVALID_ID; + + hash_init(g_cc_dev_htable); +} + +int arm_s_smmu_device_disable(struct arm_smmu_device *smmu) +{ + int ret = 0; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + ret = arm_smmu_write_s_reg_sync(smmu, 0, 0, ARM_SMMU_S_CR0, ARM_SMMU_S_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to clear s_cr0\n"); + return ret; + } + } + + return ret; +} + +int arm_s_smmu_device_reset(struct arm_smmu_device *smmu) +{ + int ret; + u64 rv; + u32 reg, enables; + struct tmi_smmu_cfg_params *params_ptr; + + if (smmu->id == ARM_SMMU_INVALID_ID) + return 0; + + rv = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_CR0, 32); + if (rv >> 32) + return -ENXIO; + + ret = (int)rv; + if (ret & S_CR0_SMMUEN) { + dev_warn(smmu->dev, "Secure SMMU currently enabled! Resetting...\n"); + arm_smmu_update_s_gbpa(smmu, S_GBPA_ABORT, 0); + } + + ret = arm_s_smmu_device_disable(smmu); + if (ret) + return ret; + + /* CR1 (table and queue memory attributes) */ + reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) | + FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) | + FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) | + FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) | + FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) | + FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB); + + ret = tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_CR1, reg, 32); + if (ret) + return ret; + + /* CR2 (random crap) */ + reg = CR2_PTM | CR2_RECINVSID; + + if (smmu->features & ARM_SMMU_FEAT_E2H) + reg |= CR2_E2H; + + ret = tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_CR2, reg, 32); + if (ret) + return ret; + + params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!params_ptr) + return -ENOMEM; + + params_ptr->is_cmd_queue = 1; + params_ptr->smmu_id = smmu->id; + params_ptr->ioaddr = smmu->ioaddr; + params_ptr->strtab_base_RA_bit = + (smmu->strtab_cfg.strtab_base >> S_STRTAB_BASE_RA_SHIFT) & 0x1; + params_ptr->q_base_RA_WA_bit = + (smmu->cmdq.q.q_base >> S_CMDQ_BASE_RA_SHIFT) & 0x1; + if (tmi_smmu_device_reset(__pa(params_ptr)) != 0) { + dev_err(smmu->dev, "failed to set s cmd queue regs\n"); + return -ENXIO; + } + + enables = CR0_CMDQEN; + ret = arm_smmu_write_s_reg_sync(smmu, enables, enables, ARM_SMMU_S_CR0, + ARM_SMMU_S_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable secure command queue\n"); + return ret; + } + + enables |= CR0_EVTQEN; + + /* Secure event queue */ + memset(params_ptr, 0, sizeof(struct tmi_smmu_ste_params)); + params_ptr->is_cmd_queue = 0; + params_ptr->ioaddr = smmu->ioaddr; + params_ptr->smmu_id = smmu->id; + params_ptr->q_base_RA_WA_bit = + (smmu->evtq.q.q_base >> S_EVTQ_BASE_WA_SHIFT) & 0x1; + params_ptr->strtab_base_RA_bit = + (smmu->strtab_cfg.strtab_base >> S_STRTAB_BASE_RA_SHIFT) & 0x1; + if (tmi_smmu_device_reset(__pa(params_ptr)) != 0) { + dev_err(smmu->dev, "failed to set s event queue regs"); + return -ENXIO; + } + + /* Enable secure eventq */ + ret = arm_smmu_write_s_reg_sync(smmu, enables, enables, ARM_SMMU_S_CR0, + ARM_SMMU_S_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to disable secure event queue\n"); + return ret; + } + + ret = arm_smmu_write_s_reg_sync(smmu, SMMU_S_INIT_INV_ALL, 0, + ARM_SMMU_S_INIT, ARM_SMMU_S_INIT); + if (ret) { + dev_err(smmu->dev, "failed to write S_INIT\n"); + return ret; + } + + return 0; +} + +int arm_s_smmu_device_enable(struct arm_smmu_device *smmu, + u32 enables, bool bypass, bool disable_bypass) +{ + int ret = 0; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + /* Enable the SMMU interface, or ensure bypass */ + if (!bypass || disable_bypass) { + enables |= CR0_SMMUEN; + } else { + ret = arm_smmu_update_s_gbpa(smmu, 0, S_GBPA_ABORT); + if (ret) + return ret; + } + /* Mask BIT1 and BIT4 which are RES0 in SMMU_S_CRO */ + ret = arm_smmu_write_s_reg_sync(smmu, enables & ~SMMU_S_CR0_RESERVED, + enables & ~SMMU_S_CR0_RESERVED, ARM_SMMU_S_CR0, ARM_SMMU_S_CR0ACK); + dev_info(smmu->dev, "SMMUv3: Secure smmu id:%lld init end!\n", smmu->id); + } + return ret; +} + +int arm_smmu_s_idr1_support_secure(struct arm_smmu_device *smmu) +{ + u32 ret; + u64 rv; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + rv = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_IDR1, 32); + if (rv >> 32) { + dev_err(smmu->dev, "Get ARM_SMMU_S_IDR1 register failed!\n"); + return -ENXIO; + } + ret = (u32)rv; + if (!(ret & S_IDR1_SECURE_IMPL)) { + dev_err(smmu->dev, "SMMUv3 does not implement secure state!\n"); + return -ENXIO; + } + + if (!(ret & S_IDR1_SEL2)) { + dev_err(smmu->dev, "SMMUv3: Secure stage2 translation not supported!\n"); + return -ENXIO; + } + dev_info(smmu->dev, "SMMUv3: Secure smmu id:%lld start init!\n", smmu->id); + } + + return 0; +} diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h new file mode 100644 index 0000000000..fa2061d94a --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#ifndef _ARM_S_SMMU_V3_H +#define _ARM_S_SMMU_V3_H + +#include <linux/platform_device.h> + +#define MAX_CC_DEV_NUM_ORDER 8 +#define STE_ENTRY_SIZE 0x40 + +#define ARM_SMMU_MAX_IDS (1 << 5) +#define ARM_SMMU_INVALID_ID ARM_SMMU_MAX_IDS + +/* Secure MMIO registers */ +#define ARM_SMMU_S_IDR0 0x8000 +#define S_IDR0_STALL_MODEL GENMASK(25, 24) +#define S_IDR0_ECMDQ (1 << 31) +#define S_IDR0_MSI (1 << 13) + +#define ARM_SMMU_S_IDR1 0x8004 +#define S_IDR1_SECURE_IMPL (1 << 31) +#define S_IDR1_SEL2 (1 << 29) +#define S_IDR1_SIDSIZE GENMASK(5, 0) + +#define ARM_SMMU_S_IDR3 0x800c +#define S_IDR3_SAMS (1 << 6) + +#define ARM_SMMU_S_CR0 0x8020 +#define S_CR0_SIF (1 << 9) +#define S_CR0_NSSTALLD (1 << 5) +#define S_CR0_CMDQEN (1 << 3) +#define S_CR0_EVTQEN (1 << 2) +#define S_CR0_SMMUEN (1 << 0) + +#define ARM_SMMU_S_CR0ACK 0x8024 + +#define ARM_SMMU_S_CR1 0x8028 +#define S_CR1_TABLE_SH GENMASK(11, 10) +#define S_CR1_TABLE_OC GENMASK(9, 8) +#define S_CR1_TABLE_IC GENMASK(7, 6) +#define S_CR1_QUEUE_SH GENMASK(5, 4) +#define S_CR1_QUEUE_OC GENMASK(3, 2) +#define S_CR1_QUEUE_IC GENMASK(1, 0) + +/* S_CR1 cacheability fields don't quite follow the usual TCR-style encoding */ +#define S_CR1_CACHE_NC 0 +#define S_CR1_CACHE_WB 1 +#define S_CR1_CACHE_WT 2 + +#define ARM_SMMU_S_CR2 0x802c +#define S_CR2_PTM (1 << 2) +#define S_CR2_RECINVSID (1 << 1) +#define S_CR2_E2H (1 << 0) + +#define ARM_SMMU_S_INIT U(0x803c) +/* SMMU_S_INIT register fields */ +#define SMMU_S_INIT_INV_ALL (1UL << 0) + +#define ARM_SMMU_S_GBPA 0x8044 +#define S_GBPA_UPDATE (1 << 31) +#define S_GBPA_ABORT (1 << 20) + +#define ARM_SMMU_S_IRQ_CTRL 0x8050 +#define S_IRQ_CTRL_EVTQ_IRQEN (1 << 2) +#define S_IRQ_CTRL_GERROR_IRQEN (1 << 0) + +#define ARM_SMMU_S_IRQ_CTRLACK 0x8054 + +#define ARM_SMMU_S_GERROR 0x8060 +#define S_GERROR_SFM_ERR (1 << 8) +#define S_GERROR_MSI_GERROR_ABT_ERR (1 << 7) +#define S_GERROR_MSI_EVTQ_ABT_ERR (1 << 5) +#define S_GERROR_MSI_CMDQ_ABT_ERR (1 << 4) +#define S_GERROR_EVTQ_ABT_ERR (1 << 2) +#define S_GERROR_CMDQ_ERR (1 << 0) + +#define ARM_SMMU_S_GERRORN 0x8064 + +#define ARM_SMMU_S_GERROR_IRQ_CFG0 0x8068 +#define ARM_SMMU_S_GERROR_IRQ_CFG1 0x8070 +#define ARM_SMMU_S_GERROR_IRQ_CFG2 0x8074 + +#define ARM_SMMU_S_STRTAB_BASE 0x8080 +#define S_STRTAB_BASE_RA_SHIFT 62 +#define S_STRTAB_BASE_RA (1UL << S_STRTAB_BASE_RA_SHIFT) +#define S_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) + +#define ARM_SMMU_S_STRTAB_BASE_CFG 0x8088 +#define S_STRTAB_BASE_CFG_FMT GENMASK(17, 16) +#define S_STRTAB_BASE_CFG_SPLIT GENMASK(10, 6) +#define S_STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0) + +#define ARM_SMMU_S_CMDQ_BASE 0x8090 +#define ARM_SMMU_S_CMDQ_PROD 0x8098 +#define ARM_SMMU_S_CMDQ_CONS 0x809c +#define S_CMDQ_BASE_ADDR_MASK GENMASK_ULL(51, 5) +#define S_CMDQ_BASE_RA_SHIFT 62 + +#define ARM_SMMU_S_EVTQ_BASE 0x80a0 +#define ARM_SMMU_S_EVTQ_PROD 0x80a8 +#define ARM_SMMU_S_EVTQ_CONS 0x80ac +#define ARM_SMMU_S_EVTQ_IRQ_CFG0 0x80b0 +#define ARM_SMMU_S_EVTQ_IRQ_CFG1 0x80b8 +#define ARM_SMMU_S_EVTQ_IRQ_CFG2 0x80bc +#define S_EVTQ_BASE_ADDR_MASK GENMASK_ULL(51, 5) +#define S_EVTQ_BASE_WA_SHIFT 62 + +/* + * BIT1 is PRIQEN, BIT4 is ATSCHK in SMMU_CRO + * BIT1 and BIT4 are RES0 in SMMU_S_CRO + */ +#define SMMU_S_CR0_RESERVED 0xFFFFFC12 + +int get_root_bd(struct device *dev); +void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, int max_devs, int *ndev); +int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs); +void set_g_cc_dev(u32 sid, u32 vmid, u32 root_bd, bool secure); +void free_g_cc_dev_htable(void); +/* Has the root bus device number switched to secure? */ +bool is_cc_root_bd(u32 root_bd); +bool is_cc_dev(u32 sid); + +void s_smmu_cmdq_need_forward(u64 cmd0, u64 cmd1, u64 *forward); +void s_queue_write(struct arm_smmu_device *smmu, u64 *src, size_t n_dwords); +void arm_s_smmu_cmdq_write_entries(struct arm_smmu_device *smmu, u64 *cmds, int n); +int arm_s_smmu_init_one_queue(struct arm_smmu_device *smmu, + struct arm_smmu_queue *q, + size_t qsz, const char *name); +int arm_smmu_write_s_reg_sync(struct arm_smmu_device *smmu, u32 val, u32 cmp_val, + unsigned int reg_off, unsigned int ack_off); +int arm_smmu_update_s_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr); + +irqreturn_t arm_smmu_s_evtq_thread(int irq, void *dev); +irqreturn_t arm_smmu_s_gerror_handler(int irq, void *dev); +int arm_smmu_disable_s_irq(struct arm_smmu_device *smmu); +int arm_smmu_enable_s_irq(struct arm_smmu_device *smmu, u32 irqen_flags); +void arm_s_smmu_setup_unique_irqs(struct arm_smmu_device *smmu); +void arm_smmu_write_s_msi_msg(struct arm_smmu_device *smmu, phys_addr_t *cfg, + struct msi_msg *msg, phys_addr_t doorbell); +void platform_get_s_irq_byname_optional(struct platform_device *pdev, struct arm_smmu_device *smmu); +int arm_smmu_enable_secure(struct iommu_domain *domain); +u32 arm_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, + struct kvm *kvm); +int arm_smmu_secure_dev_ste_create(struct arm_smmu_device *smmu, + struct arm_smmu_master *master, u32 sid); +int arm_smmu_secure_set_dev(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master *master, + struct device *dev); + +int arm_smmu_id_alloc(void); +void arm_smmu_id_free(int idx); +void arm_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr); +int arm_s_smmu_device_disable(struct arm_smmu_device *smmu); +int arm_s_smmu_device_reset(struct arm_smmu_device *smmu); +int arm_s_smmu_device_enable(struct arm_smmu_device *smmu, + u32 enables, bool bypass, bool disable_bypass); +int arm_smmu_s_idr1_support_secure(struct arm_smmu_device *smmu); + +#endif /* _ARM_S_SMMU_V3_H */ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 7148c336b3..cbe04b1b83 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -30,6 +30,11 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h"
+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#include "arm-s-smmu-v3.h" +#endif + static bool disable_bypass = true; module_param(disable_bypass, bool, 0444); MODULE_PARM_DESC(disable_bypass, @@ -102,6 +107,10 @@ static int arm_smmu_bypass_dev_domain_type(struct device *dev) enum arm_smmu_msi_index { EVTQ_MSI_INDEX, GERROR_MSI_INDEX, +#ifdef CONFIG_HISI_VIRTCCA_HOST + S_EVTQ_MSI_INDEX, + S_GERROR_MSI_INDEX, +#endif PRIQ_MSI_INDEX, ARM_SMMU_MAX_MSIS, }; @@ -117,6 +126,18 @@ static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { ARM_SMMU_GERROR_IRQ_CFG1, ARM_SMMU_GERROR_IRQ_CFG2, }, +#ifdef CONFIG_HISI_VIRTCCA_HOST + [S_EVTQ_MSI_INDEX] = { + ARM_SMMU_S_EVTQ_IRQ_CFG0, + ARM_SMMU_S_EVTQ_IRQ_CFG1, + ARM_SMMU_S_EVTQ_IRQ_CFG2, + }, + [S_GERROR_MSI_INDEX] = { + ARM_SMMU_S_GERROR_IRQ_CFG0, + ARM_SMMU_S_GERROR_IRQ_CFG1, + ARM_SMMU_S_GERROR_IRQ_CFG2, + }, +#endif [PRIQ_MSI_INDEX] = { ARM_SMMU_PRIQ_IRQ_CFG0, ARM_SMMU_PRIQ_IRQ_CFG1, @@ -995,11 +1016,16 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, * Dependency ordering from the cmpxchg() loop above. */ arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); +#ifdef CONFIG_HISI_VIRTCCA_HOST + arm_s_smmu_cmdq_write_entries(smmu, cmds, n); +#endif if (sync) { prod = queue_inc_prod_n(&llq, n); arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod); queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); - + #ifdef CONFIG_HISI_VIRTCCA_HOST + s_queue_write(smmu, cmd_sync, CMDQ_ENT_DWORDS); + #endif /* * In order to determine completion of our CMD_SYNC, we must * ensure that the queue can't wrap twice without us noticing. @@ -2376,6 +2402,12 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu_domain->secure) { + smmu_domain->stage = ARM_SMMU_DOMAIN_S2; + } +#endif + switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; @@ -2596,6 +2628,52 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); }
+#ifdef CONFIG_HISI_VIRTCCA_HOST +static void arm_smmu_secure_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) +{ + struct arm_smmu_cmdq_ent prefetch_cmd = { + .opcode = CMDQ_OP_PREFETCH_CFG, + .prefetch = { + .sid = sid, + }, + }; + arm_smmu_sync_ste_for_sid(smmu, sid); + + /* It's likely that we'll want to use the new STE soon */ + if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) + arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); +} + +static int arm_smmu_secure_dev_operator(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu, struct arm_smmu_master *master, struct device *dev) +{ + int i, j; + int ret; + + ret = arm_smmu_secure_set_dev(smmu_domain, master, dev); + if (ret) + return ret; + + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; + /* Bridged PCI devices may end up with duplicated IDs */ + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + if (arm_smmu_secure_dev_ste_create(smmu, master, sid)) + return -ENOMEM; + + arm_smmu_secure_sync_ste_for_sid(smmu, sid); + } + + dev_info(smmu->dev, "attach confidential dev: %s", dev_name(dev)); + + return ret; +} +#endif + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; @@ -2677,6 +2755,11 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
arm_smmu_install_ste_for_dev(master);
+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu_domain->secure) + ret = arm_smmu_secure_dev_operator(smmu_domain, smmu, master, dev); +#endif + arm_smmu_enable_ats(master); return 0;
@@ -3311,6 +3394,9 @@ static struct iommu_ops arm_smmu_ops = { .def_domain_type = arm_smmu_def_domain_type, .pgsize_bitmap = -1UL, /* Restricted during device attach */ .owner = THIS_MODULE, + #ifdef CONFIG_HISI_VIRTCCA_HOST + .iommu_enable_secure = arm_smmu_enable_secure, + #endif .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = arm_smmu_attach_dev, .map_pages = arm_smmu_map_pages, @@ -3373,6 +3459,13 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
q->llq.prod = q->llq.cons = 0; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (arm_s_smmu_init_one_queue(smmu, q, qsz, name)) { + return -ENOMEM; + }; +#endif + return 0; }
@@ -3676,6 +3769,15 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) desc->msg.data = msg->data; #endif
+#ifdef CONFIG_HISI_VIRTCCA_HOST + if ((desc->msi_index == S_EVTQ_MSI_INDEX || + desc->msi_index == S_GERROR_MSI_INDEX) && + smmu->id != ARM_SMMU_INVALID_ID) { + arm_smmu_write_s_msi_msg(smmu, cfg, msg, doorbell); + return; + } +#endif + writeq_relaxed(doorbell, smmu->base + cfg[0]); writel_relaxed(msg->data, smmu->base + cfg[1]); writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); @@ -3690,6 +3792,15 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu->id != ARM_SMMU_INVALID_ID) { + if (tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_GERROR_IRQ_CFG0, 0, 64)) + return; + if (tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_EVTQ_IRQ_CFG0, 0, 64)) + return; + } +#endif + if (smmu->features & ARM_SMMU_FEAT_PRI) writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); else @@ -3714,6 +3825,13 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX); smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu->id != ARM_SMMU_INVALID_ID) { + smmu->s_evtq_irq = msi_get_virq(dev, S_EVTQ_MSI_INDEX); + smmu->s_gerr_irq = msi_get_virq(dev, S_GERROR_MSI_INDEX); + } +#endif + /* Add callback to free MSIs on teardown */ devm_add_action(dev, arm_smmu_free_msis, dev); } @@ -3802,6 +3920,11 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu, bool resume dev_warn(smmu->dev, "no priq irq - PRI will be broken\n"); } } + +#ifdef CONFIG_HISI_VIRTCCA_HOST + arm_s_smmu_setup_unique_irqs(smmu); +#endif + }
static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) @@ -3817,6 +3940,12 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) return ret; }
+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_smmu_disable_s_irq(smmu); + if (ret) + return ret; +#endif + irq = smmu->combined_irq; if (irq) { /* @@ -3842,6 +3971,12 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) if (ret) dev_warn(smmu->dev, "failed to enable irqs\n");
+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_smmu_enable_s_irq(smmu, IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN); + if (ret) + return ret; +#endif + return 0; }
@@ -3853,6 +3988,10 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu) if (ret) dev_err(smmu->dev, "failed to clear cr0\n");
+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_s_smmu_device_disable(smmu); +#endif + return ret; }
@@ -3946,6 +4085,12 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool resume) FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB); writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_s_smmu_device_reset(smmu); + if (ret) + return ret; +#endif + /* CR2 (random crap) */ reg = CR2_PTM | CR2_RECINVSID;
@@ -4038,6 +4183,14 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool resume) if (is_kdump_kernel()) enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_s_smmu_device_enable(smmu, enables, smmu->bypass, disable_bypass); + if (ret) { + dev_err(smmu->dev, "failed to enable secure SMMU interface\n"); + return ret; + } +#endif + /* Enable the SMMU interface, or ensure bypass */ if (!smmu->bypass || disable_bypass) { enables |= CR0_SMMUEN; @@ -4404,7 +4557,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) }
#ifdef CONFIG_ARM_SMMU_V3_ECMDQ - if (reg & IDR1_ECMDQ) + if ((reg & IDR1_ECMDQ) && !virtcca_is_available()) smmu->features |= ARM_SMMU_FEAT_ECMDQ; #endif
@@ -4440,6 +4593,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (smmu->sid_bits <= STRTAB_SPLIT) smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (arm_smmu_s_idr1_support_secure(smmu)) + return -ENXIO; +#endif + /* IDR3 */ reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3); switch (FIELD_GET(IDR3_BBML, reg)) { @@ -4800,6 +4958,10 @@ static int arm_smmu_device_probe(struct platform_device *pdev) } ioaddr = res->start;
+#ifdef CONFIG_HISI_VIRTCCA_HOST + arm_smmu_map_init(smmu, ioaddr); +#endif + /* * Don't map the IMPLEMENTATION DEFINED regions, since they may contain * the PMCG registers which are reserved by the PMU driver. @@ -4834,6 +4996,11 @@ static int arm_smmu_device_probe(struct platform_device *pdev) irq = platform_get_irq_byname_optional(pdev, "gerror"); if (irq > 0) smmu->gerr_irq = irq; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + platform_get_s_irq_byname_optional(pdev, smmu); +#endif + } /* Probe the h/w */ ret = arm_smmu_device_hw_probe(smmu); @@ -4881,6 +5048,10 @@ static void arm_smmu_device_remove(struct platform_device *pdev) arm_smmu_device_disable(smmu); iopf_queue_free(smmu->evtq.iopf); ida_destroy(&smmu->vmid_map); +#ifdef CONFIG_HISI_VIRTCCA_HOST + free_g_cc_dev_htable(); + arm_smmu_id_free(smmu->id); +#endif }
static void arm_smmu_device_shutdown(struct platform_device *pdev) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index bc34d5a6aa..be0f8ed353 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -13,6 +13,9 @@ #include <linux/kernel.h> #include <linux/mmzone.h> #include <linux/sizes.h> +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <linux/kvm_host.h> +#endif
/* MMIO registers */ #define ARM_SMMU_IDR0 0x0 @@ -753,6 +756,13 @@ struct arm_smmu_device { struct mutex streams_mutex;
bool bypass; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + int s_evtq_irq; + int s_gerr_irq; + resource_size_t ioaddr; + uint64_t id; +#endif };
struct arm_smmu_stream { @@ -805,6 +815,12 @@ struct arm_smmu_domain { spinlock_t devices_lock;
struct list_head mmu_notifiers; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + bool secure; + struct list_head node; + struct kvm *kvm; +#endif };
static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 293219ee6f..16791cf835 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -635,7 +635,11 @@ struct iommu_ops { struct iommu_domain *blocked_domain; struct iommu_domain *default_domain;
+#ifdef CONFIG_HISI_VIRTCCA_HOST + KABI_USE(1, int (*iommu_enable_secure)(struct iommu_domain *domain)) +#else KABI_RESERVE(1) +#endif KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h index 19a7b00baf..865938aebf 100644 --- a/include/linux/iopoll.h +++ b/include/linux/iopoll.h @@ -113,6 +113,44 @@ (cond) ? 0 : -ETIMEDOUT; \ })
+#ifdef CONFIG_HISI_VIRTCCA_HOST +#define kvm_cvm_read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, \ + delay_before_read, args...) \ +({ \ + u64 __timeout_us = (timeout_us); \ + u64 rv = 0; \ + int result = 0; \ + unsigned long __delay_us = (delay_us); \ + ktime_t __timeout = ktime_add_us(ktime_get(), __timeout_us); \ + if (delay_before_read && __delay_us) \ + udelay(__delay_us); \ + for (;;) { \ + rv = op(args); \ + if (rv >> 32) { \ + result = -ENXIO; \ + break; \ + } \ + (val) = (u32)rv; \ + if (cond) \ + break; \ + if (__timeout_us && \ + ktime_compare(ktime_get(), __timeout) > 0) { \ + rv = op(args); \ + if (rv >> 32) { \ + result = -ENXIO; \ + break; \ + } \ + (val) = (u32)rv; \ + break; \ + } \ + if (__delay_us) \ + udelay(__delay_us); \ + cpu_relax(); \ + } \ + result ? result : ((cond) ? 0 : -ETIMEDOUT); \ +}) +#endif + /** * readx_poll_timeout - Periodically poll an address until a condition is met or a timeout occurs * @op: accessor function (takes @addr as its only argument)