From: JunBin Li lijunbin4@huawei.com

virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IADD42

--------------------------------

virtCCA feature

Signed-off-by: JunBin Li lijunbin4@huawei.com

--- arch/arm64/Kconfig | 8 + arch/arm64/configs/openeuler_defconfig | 5 +- arch/arm64/include/asm/kvm_emulate.h | 18 + arch/arm64/include/asm/kvm_host.h | 15 +- arch/arm64/include/asm/kvm_tmi.h | 410 +++++++ arch/arm64/include/asm/kvm_tmm.h | 52 +- arch/arm64/include/asm/set_memory.h | 1 + arch/arm64/include/asm/virtcca_cvm_guest.h | 39 + arch/arm64/include/uapi/asm/kvm.h | 4 + arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/virtcca_cvm_guest.c | 93 ++ arch/arm64/kvm/Kconfig | 6 +- arch/arm64/kvm/Makefile | 3 + arch/arm64/kvm/arch_timer.c | 92 ++ arch/arm64/kvm/arm.c | 136 ++- arch/arm64/kvm/guest.c | 8 + arch/arm64/kvm/mmio.c | 17 +- arch/arm64/kvm/mmu.c | 30 + arch/arm64/kvm/pmu-emul.c | 9 + arch/arm64/kvm/psci.c | 12 +- arch/arm64/kvm/reset.c | 13 + arch/arm64/kvm/tmi.c | 279 +++++ arch/arm64/kvm/vgic/vgic-v3.c | 17 +- arch/arm64/kvm/vgic/vgic.c | 55 +- arch/arm64/kvm/virtcca_cvm.c | 1065 +++++++++++++++++ arch/arm64/kvm/virtcca_cvm_exit.c | 221 ++++ arch/arm64/mm/init.c | 3 + arch/arm64/mm/mmu.c | 5 +- arch/arm64/mm/pageattr.c | 3 + drivers/iommu/arm/arm-smmu-v3/Makefile | 1 + drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c | 809 +++++++++++++ drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h | 160 +++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 178 ++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 16 + drivers/iommu/io-pgtable-arm.c | 60 + drivers/iommu/iommu.c | 11 + drivers/irqchip/irq-gic-v3-its.c | 231 +++- drivers/pci/access.c | 33 + drivers/pci/msi/msi.c | 90 +- drivers/perf/arm_pmu.c | 17 + drivers/vfio/pci/vfio_pci_core.c | 5 + drivers/vfio/pci/vfio_pci_rdwr.c | 61 + drivers/vfio/vfio_iommu_type1.c | 17 + drivers/vfio/vfio_main.c | 18 + include/kvm/arm_arch_timer.h | 4 + include/linux/iommu.h | 9 + include/linux/iopoll.h | 38 + include/linux/kvm_host.h | 22 + include/linux/pci.h | 4 + include/linux/perf/arm_pmu.h | 4 + include/linux/vfio.h | 5 + include/uapi/linux/kvm.h | 13 + include/uapi/linux/vfio.h | 3 + kernel/dma/swiotlb.c | 16 + virt/kvm/kvm_main.c | 4 + virt/kvm/vfio.c | 121 +- virt/kvm/vfio.h | 10 + 57 files changed, 4511 insertions(+), 69 deletions(-) create mode 100644 arch/arm64/include/asm/kvm_tmi.h create mode 100644 arch/arm64/include/asm/virtcca_cvm_guest.h create mode 100644 arch/arm64/kernel/virtcca_cvm_guest.c create mode 100644 arch/arm64/kvm/tmi.c create mode 100644 arch/arm64/kvm/virtcca_cvm.c create mode 100644 arch/arm64/kvm/virtcca_cvm_exit.c create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a162b83bd7..501ec560a9 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2576,6 +2576,14 @@ config COMPAT def_bool y depends on AARCH32_EL0 || ARM64_ILP32

+config HISI_VIRTCCA_GUEST + bool "Enable cvm guest run" + depends on DMA_RESTRICTED_POOL + help + Support VIRTCCA CVM guest based on S-EL2 + + If unsure, say N. + menu "Power management options"

source "kernel/power/Kconfig" diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 17338d53a8..072d26fcc2 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -595,6 +595,7 @@ CONFIG_DMI=y # end of Boot options

CONFIG_COMPAT=y +CONFIG_HISI_VIRTCCA_GUEST=y

# # Power management options @@ -758,7 +759,7 @@ CONFIG_KVM_GENERIC_HARDWARE_ENABLING=y CONFIG_KVM_HISI_VIRT=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=y -CONFIG_CVM_HOST=y +CONFIG_HISI_VIRTCCA_HOST=y # CONFIG_NVHE_EL2_DEBUG is not set CONFIG_KVM_ARM_MULTI_LPI_TRANSLATE_CACHE=y CONFIG_ARCH_VCPU_STAT=y @@ -7821,7 +7822,7 @@ CONFIG_ARCH_HAS_DMA_PREP_COHERENT=y CONFIG_SWIOTLB=y # CONFIG_SWIOTLB_DYNAMIC is not set CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC=y -# CONFIG_DMA_RESTRICTED_POOL is not set +CONFIG_DMA_RESTRICTED_POOL=y CONFIG_DMA_NONCOHERENT_MMAP=y CONFIG_DMA_COHERENT_POOL=y CONFIG_DMA_DIRECT_REMAP=y diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 589ee66e64..2293caab9f 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -639,4 +639,22 @@ static __always_inline void kvm_reset_cptr_el2(struct kvm_vcpu *vcpu)

kvm_write_cptr_el2(val); } + +#ifdef CONFIG_HISI_VIRTCCA_HOST +static inline bool kvm_is_virtcca_cvm(struct kvm *kvm) +{ + if (static_branch_unlikely(&virtcca_cvm_is_available)) + return kvm->arch.is_virtcca_cvm; + return false; +} + +static inline enum virtcca_cvm_state virtcca_cvm_state(struct kvm *kvm) +{ + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + if (!virtcca_cvm) + return 0; + return READ_ONCE(virtcca_cvm->state); +} +#endif #endif /* __ARM64_KVM_EMULATE_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 66c0bb96f0..ac8115098e 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -27,7 +27,7 @@ #include <asm/fpsimd.h> #include <asm/kvm.h> #include <asm/kvm_asm.h> -#ifdef CONFIG_CVM_HOST +#ifdef CONFIG_HISI_VIRTCCA_HOST #include <asm/kvm_tmm.h> #endif

@@ -292,9 +292,12 @@ struct kvm_arch { u64 tlbi_dvmbm; #endif

-#ifdef CONFIG_CVM_HOST - struct cvm cvm; - bool is_cvm; +#ifdef CONFIG_HISI_VIRTCCA_HOST + union { + struct cvm cvm; + struct virtcca_cvm *virtcca_cvm; + }; + bool is_virtcca_cvm; #endif };

@@ -622,8 +625,8 @@ struct kvm_vcpu_arch { cpumask_var_t pre_sched_cpus; #endif

-#ifdef CONFIG_CVM_HOST - struct cvm_tec tec; +#ifdef CONFIG_HISI_VIRTCCA_HOST + struct virtcca_cvm_tec tec; #endif };

diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h new file mode 100644 index 0000000000..69bf908f9b --- /dev/null +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -0,0 +1,410 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#ifndef __TMM_TMI_H +#define __TMM_TMI_H +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <linux/kvm_host.h> +#include <asm/kvm_asm.h> +#include <asm/kvm_pgtable.h> +#include <linux/virtio_ring.h> +#include <asm/sysreg.h> + +#define NO_NUMA 0 /* numa bitmap */ + +#define TMM_TTT_LEVEL_2 2 +#define TMM_TTT_LEVEL_3 3 + +/* TMI error codes. */ +#define TMI_SUCCESS 0 +#define TMI_ERROR_INPUT 1 +#define TMI_ERROR_MEMORY 2 +#define TMI_ERROR_ALIAS 3 +#define TMI_ERROR_IN_USE 4 +#define TMI_ERROR_CVM_STATE 5 +#define TMI_ERROR_OWNER 6 +#define TMI_ERROR_TEC 7 +#define TMI_ERROR_TTT_WALK 8 +#define TMI_ERROR_TTT_ENTRY 9 +#define TMI_ERROR_NOT_SUPPORTED 10 +#define TMI_ERROR_INTERNAL 11 +#define TMI_ERROR_CVM_POWEROFF 12 +#define TMI_ERROR_TTT_CREATED 13 + +#define TMI_RETURN_STATUS(ret) ((ret) & 0xFF) +#define TMI_RETURN_INDEX(ret) (((ret) >> 8) & 0xFF) + +#define TMI_FEATURE_REGISTER_0_S2SZ GENMASK(7, 0) +#define TMI_FEATURE_REGISTER_0_LPA2 BIT(8) +#define TMI_FEATURE_REGISTER_0_SVE_EN BIT(9) +#define TMI_FEATURE_REGISTER_0_SVE_VL GENMASK(13, 10) +#define TMI_FEATURE_REGISTER_0_NUM_BPS GENMASK(17, 14) +#define TMI_FEATURE_REGISTER_0_NUM_WPS GENMASK(21, 18) +#define TMI_FEATURE_REGISTER_0_PMU_EN BIT(22) +#define TMI_FEATURE_REGISTER_0_PMU_NUM_CTRS GENMASK(27, 23) +#define TMI_FEATURE_REGISTER_0_HASH_SHA_256 BIT(28) +#define TMI_FEATURE_REGISTER_0_HASH_SHA_512 BIT(29) + +#define TMI_CVM_PARAM_FLAG_LPA2 BIT(0) +#define TMI_CVM_PARAM_FLAG_SVE BIT(1) +#define TMI_CVM_PARAM_FLAG_PMU BIT(2) + +#define TMI_NOT_RUNNABLE 0 +#define TMI_RUNNABLE 1 + +/* + * The number of GPRs (starting from X0) that are + * configured by the host when a TEC is created. + */ +#define TEC_CREATE_NR_GPRS (8U) + +struct tmi_tec_params { + uint64_t gprs[TEC_CREATE_NR_GPRS]; + uint64_t pc; + uint64_t flags; + uint64_t ram_size; +}; + +struct tmi_smmu_ste_params { + uint64_t ns_src; /* non-secure STE source address */ + uint64_t sid; /* stream id */ + uint64_t smmu_id; /* smmu id */ +}; + +struct tmi_smmu_cfg_params { + uint64_t smmu_id; + uint64_t ioaddr; + uint8_t strtab_base_RA_bit : 1; + uint8_t q_base_RA_WA_bit : 1; + uint8_t is_cmd_queue : 1; +}; + +#define TMI_SMMU_CMD_QUEUE 1 +#define TMI_SMMU_EVT_QUEUE 2 +struct tmi_smmu_queue_params { + uint64_t ns_src; /* non-secure queue source address */ + uint64_t smmu_base_addr; /* smmu base address */ + uint64_t size; /* queue size */ + uint64_t smmu_id; /* smmu id */ + uint64_t type; /* cmdq or evtq */ +}; + +#define MAX_DEV_PER_PORT 256 +struct tmi_dev_delegate_params { + /* BDF of PCIe root bus, F=0. BD are used to calculate APB base and port number. */ + uint16_t root_bd; + uint16_t num_dev; /* number of attachable devices */ + uint32_t _reserved; /* padding for 64-bit alignment */ + uint16_t devs[MAX_DEV_PER_PORT]; /* BDF of each attachable device */ +}; + +#define TEC_ENTRY_FLAG_EMUL_MMIO (1UL << 0U) +#define TEC_ENTRY_FLAG_INJECT_SEA (1UL << 1U) +#define TEC_ENTRY_FLAG_TRAP_WFI (1UL << 2U) +#define TEC_ENTRY_FLAG_TRAP_WFE (1UL << 3U) + +#define TMI_EXIT_SYNC 0 +#define TMI_EXIT_IRQ 1 +#define TMI_EXIT_FIQ 2 +#define TMI_EXIT_PSCI 3 +#define TMI_EXIT_HOST_CALL 5 +#define TMI_EXIT_SERROR 6 + +/* + * The number of GPRs (starting from X0) per voluntary exit context. + * Per SMCCC. + */ + #define TEC_EXIT_NR_GPRS (31U) + +/* Maximum number of Interrupt Controller List Registers. */ +#define TEC_GIC_NUM_LRS (16U) + +struct tmi_tec_entry { + uint64_t flags; + uint64_t gprs[TEC_EXIT_NR_GPRS]; + uint64_t gicv3_lrs[TEC_GIC_NUM_LRS]; + uint64_t gicv3_hcr; +}; + +struct tmi_tec_exit { + uint64_t exit_reason; + uint64_t esr; + uint64_t far; + uint64_t hpfar; + uint64_t gprs[TEC_EXIT_NR_GPRS]; + uint64_t gicv3_hcr; + uint64_t gicv3_lrs[TEC_GIC_NUM_LRS]; + uint64_t gicv3_misr; + uint64_t gicv3_vmcr; + uint64_t cntv_ctl; + uint64_t cntv_cval; + uint64_t cntp_ctl; + uint64_t cntp_cval; + uint64_t imm; + uint64_t pmu_ovf_status; +}; + +struct tmi_tec_run { + struct tmi_tec_entry tec_entry; + struct tmi_tec_exit tec_exit; +}; + +#define TMI_FNUM_MIN_VALUE U(0x150) +#define TMI_FNUM_MAX_VALUE U(0x18F) + +/****************************************************************************** + * Bit definitions inside the function id as per the SMC calling convention + ******************************************************************************/ +#define FUNCID_TYPE_SHIFT 31 +#define FUNCID_CC_SHIFT 30 +#define FUNCID_OEN_SHIFT 24 +#define FUNCID_NUM_SHIFT 0 + +#define FUNCID_TYPE_MASK 0x1 +#define FUNCID_CC_MASK 0x1 +#define FUNCID_OEN_MASK 0x3f +#define FUNCID_NUM_MASK 0xffff + +#define FUNCID_TYPE_WIDTH 1 +#define FUNCID_CC_WIDTH 1 +#define FUNCID_OEN_WIDTH 6 +#define FUNCID_NUM_WIDTH 16 + +#define SMC_64 1 +#define SMC_32 0 +#define SMC_TYPE_FAST 1 +#define SMC_TYPE_STD 0 + +/***************************************************************************** + * Owning entity number definitions inside the function id as per the SMC + * calling convention + *****************************************************************************/ +#define OEN_ARM_START 0 +#define OEN_ARM_END 0 +#define OEN_CPU_START 1 +#define OEN_CPU_END 1 +#define OEN_SIP_START 2 +#define OEN_SIP_END 2 +#define OEN_OEM_START 3 +#define OEN_OEM_END 3 +#define OEN_STD_START 4 /* Standard Calls */ +#define OEN_STD_END 4 +#define OEN_TAP_START 48 /* Trusted Applications */ +#define OEN_TAP_END 49 +#define OEN_TOS_START 50 /* Trusted OS */ +#define OEN_TOS_END 63 +#define OEN_LIMIT 64 + +/* Get TMI fastcall std FID from function number */ +#define TMI_FID(smc_cc, func_num) \ + ((SMC_TYPE_FAST << FUNCID_TYPE_SHIFT) | \ + ((smc_cc) << FUNCID_CC_SHIFT) | \ + (OEN_STD_START << FUNCID_OEN_SHIFT) | \ + ((func_num) << FUNCID_NUM_SHIFT)) + +#define U(_x) (_x##U) + +#define TMI_NO_MEASURE_CONTENT U(0) +#define TMI_MEASURE_CONTENT U(1) + +#define CVM_IPA_MAX_VAL (1UL << 48) + +/* + * SMC_TMM_INIT_COMPLETE is the only function in the TMI that originates from + * the CVM world and is handled by the SPMD. The remaining functions are + * always invoked by the Normal world, forward by SPMD and handled by the + * TMM. + */ +#define TMI_FNUM_VERSION_REQ U(0x260) +#define TMI_FNUM_MEM_INFO_SHOW U(0x261) +#define TMI_FNUM_DATA_CREATE U(0x262) +#define TMI_FNUM_DATA_DESTROY U(0x263) +#define TMI_FNUM_CVM_ACTIVATE U(0x264) +#define TMI_FNUM_CVM_CREATE U(0x265) +#define TMI_FNUM_CVM_DESTROY U(0x266) +#define TMI_FNUM_TEC_CREATE U(0x267) +#define TMI_FNUM_TEC_DESTROY U(0x268) +#define TMI_FNUM_TEC_ENTER U(0x269) +#define TMI_FNUM_TTT_CREATE U(0x26A) +#define TMI_FNUM_PSCI_COMPLETE U(0x26B) +#define TMI_FNUM_FEATURES U(0x26C) +#define TMI_FNUM_TTT_MAP_RANGE U(0x26D) +#define TMI_FNUM_TTT_UNMAP_RANGE U(0x26E) +#define TMI_FNUM_INF_TEST U(0x270) + +#define TMI_FNUM_SMMU_QUEUE_CREATE U(0x277) +#define TMI_FNUM_SMMU_QUEUE_WRITE U(0x278) +#define TMI_FNUM_SMMU_STE_CREATE U(0x279) +#define TMI_FNUM_MMIO_MAP U(0x27A) +#define TMI_FNUM_MMIO_UNMAP U(0x27B) +#define TMI_FNUM_MMIO_WRITE U(0x27C) +#define TMI_FNUM_MMIO_READ U(0x27D) +#define TMI_FNUM_DEV_DELEGATE U(0x27E) +#define TMI_FNUM_DEV_ATTACH U(0x27F) +#define TMI_FNUM_HANDLE_S_EVTQ U(0x280) +#define TMI_FNUM_SMMU_DEVICE_RESET U(0x281) +#define TMI_FNUM_SMMU_WRITE U(0x282) +#define TMI_FNUM_SMMU_READ U(0x283) +#define TMI_FNUM_SMMU_PCIE_CORE_CHECK U(0x284) + +/* TMI SMC64 PIDs handled by the SPMD */ +#define TMI_TMM_VERSION_REQ TMI_FID(SMC_64, TMI_FNUM_VERSION_REQ) +#define TMI_TMM_DATA_CREATE TMI_FID(SMC_64, TMI_FNUM_DATA_CREATE) +#define TMI_TMM_DATA_DESTROY TMI_FID(SMC_64, TMI_FNUM_DATA_DESTROY) +#define TMI_TMM_CVM_ACTIVATE TMI_FID(SMC_64, TMI_FNUM_CVM_ACTIVATE) +#define TMI_TMM_CVM_CREATE TMI_FID(SMC_64, TMI_FNUM_CVM_CREATE) +#define TMI_TMM_CVM_DESTROY TMI_FID(SMC_64, TMI_FNUM_CVM_DESTROY) +#define TMI_TMM_TEC_CREATE TMI_FID(SMC_64, TMI_FNUM_TEC_CREATE) +#define TMI_TMM_TEC_DESTROY TMI_FID(SMC_64, TMI_FNUM_TEC_DESTROY) +#define TMI_TMM_TEC_ENTER TMI_FID(SMC_64, TMI_FNUM_TEC_ENTER) +#define TMI_TMM_TTT_CREATE TMI_FID(SMC_64, TMI_FNUM_TTT_CREATE) +#define TMI_TMM_PSCI_COMPLETE TMI_FID(SMC_64, TMI_FNUM_PSCI_COMPLETE) +#define TMI_TMM_FEATURES TMI_FID(SMC_64, TMI_FNUM_FEATURES) +#define TMI_TMM_MEM_INFO_SHOW TMI_FID(SMC_64, TMI_FNUM_MEM_INFO_SHOW) +#define TMI_TMM_TTT_MAP_RANGE TMI_FID(SMC_64, TMI_FNUM_TTT_MAP_RANGE) +#define TMI_TMM_TTT_UNMAP_RANGE TMI_FID(SMC_64, TMI_FNUM_TTT_UNMAP_RANGE) +#define TMI_TMM_INF_TEST TMI_FID(SMC_64, TMI_FNUM_INF_TEST) + +#define TMI_TMM_SMMU_QUEUE_CREATE TMI_FID(SMC_64, TMI_FNUM_SMMU_QUEUE_CREATE) +#define TMI_TMM_SMMU_QUEUE_WRITE TMI_FID(SMC_64, TMI_FNUM_SMMU_QUEUE_WRITE) +#define TMI_TMM_SMMU_STE_CREATE TMI_FID(SMC_64, TMI_FNUM_SMMU_STE_CREATE) +#define TMI_TMM_MMIO_MAP TMI_FID(SMC_64, TMI_FNUM_MMIO_MAP) +#define TMI_TMM_MMIO_UNMAP TMI_FID(SMC_64, TMI_FNUM_MMIO_UNMAP) +#define TMI_TMM_MMIO_WRITE TMI_FID(SMC_64, TMI_FNUM_MMIO_WRITE) +#define TMI_TMM_MMIO_READ TMI_FID(SMC_64, TMI_FNUM_MMIO_READ) +#define TMI_TMM_DEV_DELEGATE TMI_FID(SMC_64, TMI_FNUM_DEV_DELEGATE) +#define TMI_TMM_DEV_ATTACH TMI_FID(SMC_64, TMI_FNUM_DEV_ATTACH) +#define TMI_TMM_HANDLE_S_EVTQ TMI_FID(SMC_64, TMI_FNUM_HANDLE_S_EVTQ) +#define TMI_TMM_SMMU_DEVICE_RESET TMI_FID(SMC_64, TMI_FNUM_SMMU_DEVICE_RESET) +#define TMI_TMM_SMMU_WRITE TMI_FID(SMC_64, TMI_FNUM_SMMU_WRITE) +#define TMI_TMM_SMMU_READ TMI_FID(SMC_64, TMI_FNUM_SMMU_READ) +#define TMI_TMM_SMMU_PCIE_CORE_CHECK TMI_FID(SMC_64, TMI_FNUM_SMMU_PCIE_CORE_CHECK) + +#define TMI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) +#define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) + +#define TMI_ABI_VERSION_MAJOR U(0x1) + +/* KVM_CAP_ARM_TMM on VM fd */ +#define KVM_CAP_ARM_TMM_CONFIG_CVM_HOST 0 +#define KVM_CAP_ARM_TMM_CREATE_RD 1 +#define KVM_CAP_ARM_TMM_POPULATE_CVM 2 +#define KVM_CAP_ARM_TMM_ACTIVATE_CVM 3 + +#define KVM_CAP_ARM_TMM_MEASUREMENT_ALGO_SHA256 0 +#define KVM_CAP_ARM_TMM_MEASUREMENT_ALGO_SHA512 1 + +#define KVM_CAP_ARM_TMM_RPV_SIZE 64 + +/* List of configuration items accepted for KVM_CAP_ARM_TMM_CONFIG_CVM_HOST */ +#define KVM_CAP_ARM_TMM_CFG_RPV 0 +#define KVM_CAP_ARM_TMM_CFG_HASH_ALGO 1 +#define KVM_CAP_ARM_TMM_CFG_SVE 2 +#define KVM_CAP_ARM_TMM_CFG_DBG 3 +#define KVM_CAP_ARM_TMM_CFG_PMU 4 + +DECLARE_STATIC_KEY_FALSE(virtcca_cvm_is_available); +DECLARE_STATIC_KEY_FALSE(virtcca_cvm_is_enable); + +struct kvm_cap_arm_tmm_config_item { + __u32 cfg; + union { + /* cfg == KVM_CAP_ARM_TMM_CFG_RPV */ + struct { + __u8 rpv[KVM_CAP_ARM_TMM_RPV_SIZE]; + }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_HASH_ALGO */ + struct { + __u32 hash_algo; + }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_SVE */ + struct { + __u32 sve_vq; + }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_DBG */ + struct { + __u32 num_brps; + __u32 num_wrps; + }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_PMU */ + struct { + __u32 num_pmu_cntrs; + }; + /* Fix the size of the union */ + __u8 reserved[256]; + }; +}; + +#define KVM_ARM_TMM_POPULATE_FLAGS_MEASURE (1U << 0) +struct kvm_cap_arm_tmm_populate_region_args { + __u64 populate_ipa_base1; + __u64 populate_ipa_size1; + __u64 populate_ipa_base2; + __u64 populate_ipa_size2; + __u32 flags; + __u32 reserved[3]; +}; + +static inline bool tmm_is_addr_ttt_level_aligned(uint64_t addr, int level) +{ + uint64_t mask = (1 << (12 + 9 * (3 - level))) - 1; + + return (addr & mask) == 0; +} + +#define ID_AA64PFR0_SEL2_MASK ULL(0xf) +#define ID_AA64PFR0_SEL2_SHIFT 36 + +static inline bool is_armv8_4_sel2_present(void) +{ + return ((read_sysreg(id_aa64pfr0_el1) >> ID_AA64PFR0_SEL2_SHIFT) & + ID_AA64PFR0_SEL2_MASK) == 1UL; +} + +u64 tmi_version(void); +u64 tmi_data_create(u64 data, u64 rd, u64 map_addr, u64 src, u64 level); +u64 tmi_data_destroy(u64 rd, u64 map_addr, u64 level); +u64 tmi_cvm_activate(u64 rd); +u64 tmi_cvm_create(u64 params_ptr, u64 numa_set); +u64 tmi_cvm_destroy(u64 rd); +u64 tmi_tec_create(u64 numa_set, u64 rd, u64 mpidr, u64 params_ptr); +u64 tmi_tec_destroy(u64 tec); +u64 tmi_tec_enter(u64 tec, u64 run_ptr); +u64 tmi_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level); +u64 tmi_psci_complete(u64 calling_tec, u64 target_tec); +u64 tmi_features(u64 index); +u64 tmi_ttt_map_range(u64 rd, u64 map_addr, u64 size, u64 cur_node, u64 target_node); +u64 tmi_ttt_unmap_range(u64 rd, u64 map_addr, u64 size, u64 node_id); +u64 tmi_mem_info_show(u64 mem_info_addr); + +u64 va_to_pa(void *addr); +u64 tmi_smmu_queue_create(u64 params_ptr); +u64 tmi_smmu_queue_write(uint64_t cmd0, uint64_t cmd1, u64 smmu_id); +u64 tmi_smmu_ste_create(u64 params_ptr); +u64 tmi_mmio_map(u64 rd, u64 map_addr, u64 level, u64 ttte); +u64 tmi_mmio_unmap(u64 rd, u64 map_addr, u64 level); +u64 tmi_mmio_write(u64 addr, u64 val, u64 bits, u64 dev_num); +u64 tmi_mmio_read(u64 addr, u64 bits, u64 dev_num); +u64 tmi_dev_delegate(u64 params); +u64 tmi_dev_attach(u64 vdev, u64 rd, u64 smmu_id); +u64 tmi_handle_s_evtq(u64 smmu_id); +u64 tmi_smmu_device_reset(u64 params); +u64 tmi_smmu_pcie_core_check(u64 smmu_base); +u64 tmi_smmu_write(u64 smmu_base, u64 reg_offset, u64 val, u64 bits); +u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits); + +void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu); +int kvm_load_user_data(struct kvm *kvm, unsigned long arg); +unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, + unsigned long target_affinity, unsigned long lowest_affinity_level); +int kvm_cvm_vcpu_set_events(struct kvm_vcpu *vcpu, + bool serror_pending, bool ext_dabt_pending); +int kvm_init_cvm_vm(struct kvm *kvm); +int kvm_enable_virtcca_cvm(struct kvm *kvm); +#endif +#endif diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index f70d73be03..2fd6d2bdb9 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -7,8 +7,11 @@

#include <uapi/linux/kvm.h>

-enum cvm_state { - CVM_STATE_NONE, +#define CVM_MSI_ORIG_IOVA 0x8000000 +#define CVM_MSI_IOVA_OFFSET (-0x1000000) + +enum virtcca_cvm_state { + CVM_STATE_NONE = 1, CVM_STATE_NEW, CVM_STATE_ACTIVE, CVM_STATE_DYING @@ -36,7 +39,7 @@ struct tmi_cvm_params { };

struct cvm { - enum cvm_state state; + enum virtcca_cvm_state state; u32 cvm_vmid; u64 rd; u64 loader_start; @@ -48,33 +51,62 @@ struct cvm { bool is_cvm; };

+struct virtcca_cvm { + enum virtcca_cvm_state state; + u32 cvm_vmid; + u64 rd; + u64 loader_start; + u64 image_end; + u64 initrd_start; + u64 dtb_end; + u64 ram_size; + struct kvm_numa_info numa_info; + struct tmi_cvm_params *params; + bool is_mapped; +}; + /* * struct cvm_tec - Additional per VCPU data for a CVM */ -struct cvm_tec { +struct virtcca_cvm_tec { u64 tec; bool tec_created; void *tec_run; };

+struct cvm_ttt_addr { + struct list_head list; + u64 addr; +}; + +struct iommu_group {}; + int kvm_init_tmm(void); int kvm_cvm_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap); -int kvm_init_cvm_vm(struct kvm *kvm); void kvm_destroy_cvm(struct kvm *kvm); -int kvm_create_tec(struct kvm_vcpu *vcpu); +int kvm_finalize_vcpu_tec(struct kvm_vcpu *vcpu); void kvm_destroy_tec(struct kvm_vcpu *vcpu); int kvm_tec_enter(struct kvm_vcpu *vcpu); int handle_cvm_exit(struct kvm_vcpu *vcpu, int rec_run_status); int kvm_arm_create_cvm(struct kvm *kvm); void kvm_free_rd(struct kvm *kvm); -int cvm_create_rd(struct kvm *kvm); int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target); -int kvm_arch_tec_init(struct kvm_vcpu *vcpu);

void kvm_cvm_unmap_destroy_range(struct kvm *kvm);

-#define CVM_TTT_BLOCK_LEVEL 2 -#define CVM_TTT_MAX_LEVEL 3 +int kvm_cvm_map_range(struct kvm *kvm); +int cvm_arm_smmu_domain_set_kvm(struct iommu_group *group); +int kvm_cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, + unsigned long map_size, uint32_t is_map); +int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size); + +#define CVM_TTT_BLOCK_LEVEL 2 +#define CVM_TTT_MAX_LEVEL 3 + +#define CVM_MAP_IPA_RAM 1 +#define CVM_MAP_IPA_SMMU 2 +#define CVM_MAP_IPA_UNPROTECTED 4

#define CVM_PAGE_SHIFT 12 #define CVM_PAGE_SIZE BIT(CVM_PAGE_SHIFT) diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h index 0f740b7811..2031b31c05 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -4,6 +4,7 @@ #define _ASM_ARM64_SET_MEMORY_H

#include <asm-generic/set_memory.h> +#include <asm/virtcca_cvm_guest.h>

bool can_set_direct_map(void); #define can_set_direct_map can_set_direct_map diff --git a/arch/arm64/include/asm/virtcca_cvm_guest.h b/arch/arm64/include/asm/virtcca_cvm_guest.h new file mode 100644 index 0000000000..2a68626a4f --- /dev/null +++ b/arch/arm64/include/asm/virtcca_cvm_guest.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#ifndef __VIRTCCA_CVM_GUEST_H +#define __VIRTCCA_CVM_GUEST_H + +#ifdef CONFIG_HISI_VIRTCCA_GUEST +struct device; + +extern int set_cvm_memory_encrypted(unsigned long addr, int numpages); + +extern int set_cvm_memory_decrypted(unsigned long addr, int numpages); + +extern bool is_virtcca_cvm_world(void); + +extern void __init swiotlb_cvm_update_mem_attributes(void); + +#else + +static inline int set_cvm_memory_encrypted(unsigned long addr, int numpages) +{ + return 0; +} + +static inline int set_cvm_memory_decrypted(unsigned long addr, int numpages) +{ + return 0; +} + +static inline bool is_virtcca_cvm_world(void) +{ + return false; +} + +static inline void __init swiotlb_cvm_update_mem_attributes(void) {} + +#endif /* CONFIG_HISI_VIRTCCA_GUEST */ +#endif /* __VIRTCCA_CVM_GUEST_H */ diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index f7ddd73a8c..97941e582d 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -110,6 +110,7 @@ struct kvm_regs { #define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */ #define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */ #define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */ +#define KVM_ARM_VCPU_TEC 8 /* VCPU TEC state as part of cvm */

struct kvm_vcpu_init { __u32 target; @@ -415,6 +416,9 @@ enum { #define KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES 3 #define KVM_DEV_ARM_ITS_CTRL_RESET 4

+#define KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA256 0 +#define KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA512 1 + /* Device Control API on vcpu fd */ #define KVM_ARM_VCPU_PMU_V3_CTRL 0 #define KVM_ARM_VCPU_PMU_V3_IRQ 0 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 21ef9c21a4..cccf8332ed 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -81,6 +81,7 @@ obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o obj-$(CONFIG_ARM64_ILP32) += vdso-ilp32/ obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o +obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o CFLAGS_patch-scs.o += -mbranch-protection=none

# Force dependency (vdso*-wrap.S includes vdso.so through incbin) diff --git a/arch/arm64/kernel/virtcca_cvm_guest.c b/arch/arm64/kernel/virtcca_cvm_guest.c new file mode 100644 index 0000000000..6c49875467 --- /dev/null +++ b/arch/arm64/kernel/virtcca_cvm_guest.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/vmalloc.h> + +#include <asm/cacheflush.h> +#include <asm/set_memory.h> +#include <asm/tlbflush.h> + +#define CVM_PTE_NS_BIT 5 +#define CVM_PTE_NS_MASK (1 << CVM_PTE_NS_BIT) + +static bool cvm_guest_enable __read_mostly; + +/* please use 'virtcca_cvm_guest=1' to enable cvm guest feature */ +static int __init setup_cvm_guest(char *str) +{ + int ret; + unsigned int val; + + if (!str) + return 0; + + ret = kstrtouint(str, 10, &val); + if (ret) { + pr_warn("Unable to parse cvm_guest.\n"); + } else { + if (val) + cvm_guest_enable = true; + } + return ret; +} +early_param("virtcca_cvm_guest", setup_cvm_guest); + +bool is_virtcca_cvm_world(void) +{ + return cvm_guest_enable; +} + +static int change_page_range_cvm(pte_t *ptep, unsigned long addr, void *data) +{ + bool encrypt = (bool)data; + pte_t pte = READ_ONCE(*ptep); + + if (encrypt) { + if (!(pte.pte & CVM_PTE_NS_MASK)) + return 0; + pte.pte = pte.pte & (~CVM_PTE_NS_MASK); + } else { + if (pte.pte & CVM_PTE_NS_MASK) + return 0; + /* Set NS BIT */ + pte.pte = pte.pte | CVM_PTE_NS_MASK; + } + set_pte(ptep, pte); + + return 0; +} + +static int __change_memory_common_cvm(unsigned long start, unsigned long size, bool encrypt) +{ + int ret; + + ret = apply_to_page_range(&init_mm, start, size, change_page_range_cvm, (void *)encrypt); + flush_tlb_kernel_range(start, start + size); + return ret; +} + +static int __set_memory_encrypted(unsigned long addr, + int numpages, + bool encrypt) +{ + if (!is_virtcca_cvm_world()) + return 0; + + WARN_ON(!__is_lm_address(addr)); + return __change_memory_common_cvm(addr, PAGE_SIZE * numpages, encrypt); +} + +int set_cvm_memory_encrypted(unsigned long addr, int numpages) +{ + return __set_memory_encrypted(addr, numpages, true); +} + +int set_cvm_memory_decrypted(unsigned long addr, int numpages) +{ + return __set_memory_encrypted(addr, numpages, false); +} diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 1fa6fba607..52edbd7f63 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -49,11 +49,11 @@ menuconfig KVM

If unsure, say N.

-config CVM_HOST - bool "Enable cvm host feature" +config HISI_VIRTCCA_HOST + bool "Enable virtcca cvm host feature" depends on KVM help - Support CVM based on S-EL2 + Support VIRTCCA CVM based on S-EL2

If unsure, say N.

diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 952eee572e..eadf41417f 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -24,6 +24,9 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o pvsched.o \

kvm-$(CONFIG_VIRT_PLAT_DEV) += vgic/shadow_dev.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o +kvm-$(CONFIG_HISI_VIRTCCA_HOST) += tmi.o +kvm-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm.o +kvm-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_exit.o obj-$(CONFIG_KVM_HISI_VIRT) += hisilicon/

always-y := hyp_constants.h hyp-constants.s diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 43957fce50..205361d1b0 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -18,6 +18,10 @@ #include <asm/kvm_hyp.h> #include <asm/kvm_nested.h>

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif + #include <kvm/arm_vgic.h> #include <kvm/arm_arch_timer.h>

@@ -175,6 +179,71 @@ static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval) } }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +static bool cvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) +{ + return timer_ctx && + ((timer_get_ctl(timer_ctx) & + (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE); +} + +void kvm_cvm_timers_update(struct kvm_vcpu *vcpu) +{ + int i; + u64 cval, now; + bool status, level; + struct arch_timer_context *timer; + struct arch_timer_cpu *arch_timer = &vcpu->arch.timer_cpu; + + for (i = 0; i < NR_KVM_TIMERS; i++) { + timer = &arch_timer->timers[i]; + + if (!timer->loaded) { + if (!cvm_timer_irq_can_fire(timer)) + continue; + cval = timer_get_cval(timer); + now = kvm_phys_timer_read() - timer_get_offset(timer); + level = (cval <= now); + kvm_timer_update_irq(vcpu, level, timer); + } else { + status = timer_get_ctl(timer) & ARCH_TIMER_CTRL_IT_STAT; + level = cvm_timer_irq_can_fire(timer) && status; + if (level != timer->irq.level) + kvm_timer_update_irq(vcpu, level, timer); + } + } +} + +static void set_cvm_timers_loaded(struct kvm_vcpu *vcpu, bool loaded) +{ + int i; + struct arch_timer_cpu *arch_timer = &vcpu->arch.timer_cpu; + + for (i = 0; i < NR_KVM_TIMERS; i++) { + struct arch_timer_context *timer = &arch_timer->timers[i]; + + timer->loaded = loaded; + } +} + +static void kvm_timer_blocking(struct kvm_vcpu *vcpu); +static void kvm_timer_unblocking(struct kvm_vcpu *vcpu); + +static inline void cvm_vcpu_load_timer_callback(struct kvm_vcpu *vcpu) +{ + kvm_cvm_timers_update(vcpu); + kvm_timer_unblocking(vcpu); + set_cvm_timers_loaded(vcpu, true); +} + +static inline void cvm_vcpu_put_timer_callback(struct kvm_vcpu *vcpu) +{ + set_cvm_timers_loaded(vcpu, false); + if (rcuwait_active(kvm_arch_vcpu_get_wait(vcpu))) + kvm_timer_blocking(vcpu); +} +#endif + static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) { if (!ctxt->offset.vm_offset) { @@ -883,6 +952,13 @@ void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + cvm_vcpu_load_timer_callback(vcpu); + return; + } +#endif + if (unlikely(!timer->enabled)) return;

@@ -981,6 +1057,13 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + cvm_vcpu_put_timer_callback(vcpu); + return; + } +#endif + if (unlikely(!timer->enabled)) return;

@@ -1766,6 +1849,15 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) return -EINVAL; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST + /* + * We don't use mapped IRQs for CVM because the TMI doesn't allow + * us setting the LR.HW bit in the VGIC. + */ + if (vcpu_is_tec(vcpu)) + return 0; +#endif + get_timer_map(vcpu, &map);

#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 333c65ced8..24f1d898c4 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -40,6 +40,10 @@ #include <asm/kvm_pkvm.h> #include <asm/kvm_emulate.h> #include <asm/sections.h> +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#include <linux/perf/arm_pmu.h> +#endif

#include <kvm/arm_hypercalls.h> #include <kvm/arm_pmu.h> @@ -132,6 +136,12 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, } mutex_unlock(&kvm->slots_lock); break; +#ifdef CONFIG_HISI_VIRTCCA_HOST + case KVM_CAP_ARM_TMM: + if (static_branch_unlikely(&virtcca_cvm_is_available)) + r = kvm_cvm_enable_cap(kvm, cap); + break; +#endif default: r = -EINVAL; break; @@ -153,6 +163,14 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { int ret;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (kvm_arm_cvm_type(type)) { + ret = kvm_enable_virtcca_cvm(kvm); + if (ret) + return ret; + } +#endif + ret = kvm_sched_affinity_vm_init(kvm); if (ret) return ret; @@ -199,8 +217,20 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)

bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (kvm_arm_cvm_type(type)) { + ret = kvm_init_cvm_vm(kvm); + if (ret) + goto out_free_stage2_pgd; + } +#endif + return 0;

+#ifdef CONFIG_HISI_VIRTCCA_HOST +out_free_stage2_pgd: + kvm_free_stage2_pgd(&kvm->arch.mmu); +#endif err_free_cpumask: free_cpumask_var(kvm->arch.supported_cpus); err_unshare_kvm: @@ -235,6 +265,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_unshare_hyp(kvm, kvm + 1);

kvm_arm_teardown_hypercalls(kvm); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (kvm_is_virtcca_cvm(kvm)) + kvm_destroy_cvm(kvm); +#endif }

int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) @@ -306,7 +340,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = system_supports_mte(); break; case KVM_CAP_STEAL_TIME: - r = kvm_arm_pvtime_supported(); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (kvm && kvm_is_virtcca_cvm(kvm)) + r = 0; + else +#endif + r = kvm_arm_pvtime_supported(); break; case KVM_CAP_ARM_EL1_32BIT: r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1); @@ -346,6 +385,15 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_ARM_VIRT_MSI_BYPASS: r = sdev_enable; break; +#endif +#ifdef CONFIG_HISI_VIRTCCA_HOST + case KVM_CAP_ARM_TMM: + if (!is_armv8_4_sel2_present()) { + r = -ENXIO; + break; + } + r = static_key_enabled(&virtcca_cvm_is_available); + break; #endif default: r = 0; @@ -499,8 +547,23 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)

vcpu->cpu = cpu;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + if (single_task_running()) + vcpu_clear_wfx_traps(vcpu); + else + vcpu_set_wfx_traps(vcpu); + } +#endif kvm_vgic_load(vcpu); kvm_timer_vcpu_load(vcpu); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); + return; + } +#endif if (has_vhe()) kvm_vcpu_load_sysregs_vhe(vcpu); kvm_arch_vcpu_load_fp(vcpu); @@ -531,6 +594,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)

void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) { +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + kvm_cvm_vcpu_put(vcpu); + return; + } +#endif kvm_arch_vcpu_put_debug_state_flags(vcpu); kvm_arch_vcpu_put_fp(vcpu); if (has_vhe()) @@ -705,6 +774,9 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) * Tell the rest of the code that there are userspace irqchip * VMs in the wild. */ +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (!kvm_is_virtcca_cvm(kvm)) +#endif static_branch_inc(&userspace_irqchip_in_use); }

@@ -959,6 +1031,18 @@ static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret) xfer_to_guest_mode_work_pending(); }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +static inline void update_pmu_phys_irq(struct kvm_vcpu *vcpu, bool *pmu_stopped) +{ + struct kvm_pmu *pmu = &vcpu->arch.pmu; + + if (pmu->irq_level) { + *pmu_stopped = true; + arm_pmu_set_phys_irq(false); + } +} +#endif + /* * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while * the vCPU is running. @@ -1011,6 +1095,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) run->exit_reason = KVM_EXIT_UNKNOWN; run->flags = 0; while (ret > 0) { +#ifdef CONFIG_HISI_VIRTCCA_HOST + bool pmu_stopped = false; +#endif /* * Check conditions before entering the guest */ @@ -1038,6 +1125,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_arm_vmid_update(&vcpu->arch.hw_mmu->vmid);

kvm_pmu_flush_hwstate(vcpu); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + update_pmu_phys_irq(vcpu, &pmu_stopped); +#endif

local_irq_disable();

@@ -1076,7 +1167,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) trace_kvm_entry(*vcpu_pc(vcpu)); guest_timing_enter_irqoff();

- ret = kvm_arm_vcpu_enter_exit(vcpu); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + ret = kvm_tec_enter(vcpu); + else +#endif + ret = kvm_arm_vcpu_enter_exit(vcpu);

vcpu->mode = OUTSIDE_GUEST_MODE; vcpu->stat.exits++; @@ -1130,13 +1226,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)

local_irq_enable();

- trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (!vcpu_is_tec(vcpu)) { +#endif + trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));

- /* Exit types that need handling before we can be preempted */ - handle_exit_early(vcpu, ret); + /* Exit types that need handling before we can be preempted */ + handle_exit_early(vcpu, ret);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + } +#endif preempt_enable();

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (pmu_stopped) + arm_pmu_set_phys_irq(true); +#endif /* * The ARMv8 architecture doesn't give the hypervisor * a mechanism to prevent a guest from dropping to AArch32 EL0 @@ -1156,7 +1262,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) ret = ARM_EXCEPTION_IL; }

- ret = handle_exit(vcpu, ret); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + ret = handle_cvm_exit(vcpu, ret); + else +#endif + ret = handle_exit(vcpu, ret); #ifdef CONFIG_ARCH_VCPU_STAT update_vcpu_stat_time(&vcpu->stat); #endif @@ -1669,6 +1780,11 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) struct kvm_device_attr attr;

switch (ioctl) { +#ifdef CONFIG_HISI_VIRTCCA_HOST + case KVM_LOAD_USER_DATA: { + return kvm_load_user_data(kvm, arg); + } +#endif case KVM_CREATE_IRQCHIP: { int ret; if (!vgic_present) @@ -2555,6 +2671,14 @@ static __init int kvm_arm_init(void)

in_hyp_mode = is_kernel_in_hyp_mode();

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (static_branch_unlikely(&virtcca_cvm_is_enable) && in_hyp_mode) { + err = kvm_init_tmm(); + if (err) + return err; + } +#endif + if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || cpus_have_final_cap(ARM64_WORKAROUND_1508412)) kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \ diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index d3161a6838..709c0c9035 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -27,6 +27,10 @@ #include <asm/kvm_nested.h> #include <asm/sigcontext.h>

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif + #include "trace.h"

#ifdef CONFIG_ARCH_VCPU_STAT @@ -875,6 +879,10 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, bool has_esr = events->exception.serror_has_esr; bool ext_dabt_pending = events->exception.ext_dabt_pending;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + return kvm_cvm_vcpu_set_events(vcpu, serror_pending, ext_dabt_pending); +#endif if (serror_pending && has_esr) { if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) return -EINVAL; diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index 3dd38a151d..f2930050c8 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -8,6 +8,10 @@ #include <asm/kvm_emulate.h> #include <trace/events/kvm.h>

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif + #include "trace.h"

void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data) @@ -109,6 +113,12 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) &data); data = vcpu_data_host_to_guest(vcpu, data, len); vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), data); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + ((struct tmi_tec_run *)vcpu->arch.tec.tec_run)-> + tec_entry.gprs[0] = data; + } +#endif }

/* @@ -178,7 +188,12 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) run->mmio.phys_addr = fault_ipa; run->mmio.len = len; vcpu->mmio_needed = 1; - +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + ((struct tmi_tec_run *)vcpu->arch.tec.tec_run)->tec_entry.flags |= + TEC_ENTRY_FLAG_EMUL_MMIO; + } +#endif if (!ret) { /* We handled the access successfully in the kernel. */ if (!is_write) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 121a3d9024..1d034117a4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1013,6 +1013,7 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)

write_lock(&kvm->mmu_lock); pgt = mmu->pgt; + if (pgt) { mmu->pgd_phys = 0; mmu->pgt = NULL; @@ -1391,6 +1392,23 @@ static bool kvm_vma_mte_allowed(struct vm_area_struct *vma) return vma->vm_flags & VM_MTE_ALLOWED; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +static int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, + unsigned long map_size, enum kvm_pgtable_prot prot) +{ + struct page *dst_page = pfn_to_page(pfn); + phys_addr_t dst_phys = page_to_phys(dst_page); + + if (WARN_ON(!(prot & KVM_PGTABLE_PROT_W))) + return -EFAULT; + + if (prot & KVM_PGTABLE_PROT_DEVICE) + return kvm_cvm_map_ipa_mmio(kvm, ipa, dst_phys, map_size); + + return 0; +} +#endif + static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, unsigned long fault_status) @@ -1414,6 +1432,12 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,

fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level); write_fault = kvm_is_write_fault(vcpu); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + write_fault = true; + prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W; + } +#endif exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu); VM_BUG_ON(write_fault && exec_fault); vcpu->stat.mabt_exit_stat++; @@ -1595,6 +1619,12 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, memcache, KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (kvm_is_virtcca_cvm(kvm)) { + ret = kvm_cvm_map_ipa(kvm, fault_ipa, pfn, vma_pagesize, prot); + WARN_ON(ret); + } +#endif

/* Mark the page dirty only if the fault is handled successfully */ if (writable && !ret) { diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 6b066e04dc..cdc023f845 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -15,6 +15,7 @@ #include <kvm/arm_pmu.h> #include <kvm/arm_vgic.h> #include <asm/arm_pmuv3.h> +#include <asm/kvm_tmi.h>

#define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0)

@@ -324,6 +325,14 @@ static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu) { u64 reg = 0;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + + reg = run->tec_exit.pmu_ovf_status; + return reg; + } +#endif if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) { reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0); reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 1f69b66733..c611048fcd 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -16,6 +16,9 @@ #include <kvm/arm_psci.h> #include <kvm/arm_hypercalls.h>

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif /* * This is an implementation of the Power State Coordination Interface * as described in ARM document number ARM DEN 0022A. @@ -79,6 +82,10 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) return PSCI_RET_INVALID_PARAMS;

spin_lock(&vcpu->arch.mp_state_lock); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + cvm_psci_complete(source_vcpu, vcpu); +#endif if (!kvm_arm_vcpu_stopped(vcpu)) { if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) ret = PSCI_RET_ALREADY_ON; @@ -141,7 +148,10 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)

/* Ignore other bits of target affinity */ target_affinity &= target_affinity_mask; - +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + return cvm_psci_vcpu_affinity_info(vcpu, target_affinity, lowest_affinity_level); +#endif /* * If one or more VCPU matching target affinity are running * then ON else OFF diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 7a65a35ee4..cfb01c4917 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -30,6 +30,9 @@ #include <asm/kvm_nested.h> #include <asm/virt.h>

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif /* Maximum phys_shift supported for any VM on this host */ static u32 __ro_after_init kvm_ipa_limit;

@@ -139,6 +142,12 @@ int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature) return -EPERM;

return kvm_vcpu_finalize_sve(vcpu); +#ifdef CONFIG_HISI_VIRTCCA_HOST + case KVM_ARM_VCPU_TEC: + if (!kvm_is_virtcca_cvm(vcpu->kvm)) + return -EINVAL; + return kvm_finalize_vcpu_tec(vcpu); +#endif }

return -EINVAL; @@ -162,6 +171,10 @@ void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu)); kfree(sve_state); kfree(vcpu->arch.ccsidr); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + kvm_destroy_tec(vcpu); +#endif }

static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c new file mode 100644 index 0000000000..262bd2daa5 --- /dev/null +++ b/arch/arm64/kvm/tmi.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#include <linux/arm-smccc.h> +#include <asm/kvm_tmi.h> +#include <asm/memory.h> + +u64 va_to_pa(void *addr) +{ + uint64_t pa, par_el1; + + asm volatile( + "AT S1E1W, %0\n" + ::"r"((uint64_t)(addr)) + ); + isb(); + asm volatile( + "mrs %0, par_el1\n" + :"=r"(par_el1) + ); + + pa = ((uint64_t)(addr) & (PAGE_SIZE - 1)) | + (par_el1 & ULL(0x000ffffffffff000)); + + if (par_el1 & UL(1 << 0)) + return (uint64_t)(addr); + else + return pa; +} +EXPORT_SYMBOL(va_to_pa); + +u64 tmi_version(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_VERSION_REQ, &res); + return res.a1; +} + +u64 tmi_data_create(u64 numa_set, u64 rd, u64 map_addr, u64 src, u64 level) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DATA_CREATE, numa_set, rd, map_addr, src, level, &res); + return res.a1; +} + +u64 tmi_data_destroy(u64 rd, u64 map_addr, u64 level) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DATA_DESTROY, rd, map_addr, level, &res); + return res.a1; +} + +u64 tmi_cvm_activate(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_CVM_ACTIVATE, rd, &res); + return res.a1; +} + +u64 tmi_cvm_create(u64 params_ptr, u64 numa_set) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_CVM_CREATE, params_ptr, numa_set, &res); + return res.a1; +} + +u64 tmi_cvm_destroy(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_CVM_DESTROY, rd, &res); + return res.a1; +} + +u64 tmi_tec_create(u64 numa_set, u64 rd, u64 mpidr, u64 params_ptr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_TEC_CREATE, numa_set, rd, mpidr, params_ptr, &res); + return res.a1; +} + +u64 tmi_tec_destroy(u64 tec) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_TEC_DESTROY, tec, &res); + return res.a1; +} + +u64 tmi_tec_enter(u64 tec, u64 run_ptr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_TEC_ENTER, tec, run_ptr, &res); + return res.a1; +} + +u64 tmi_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_TTT_CREATE, numa_set, rd, map_addr, level, &res); + return res.a1; +} + +u64 tmi_psci_complete(u64 calling_tec, u64 target_tec) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_PSCI_COMPLETE, calling_tec, target_tec, &res); + return res.a1; +} + +u64 tmi_features(u64 index) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_FEATURES, index, &res); + return res.a1; +} + +u64 tmi_mem_info_show(u64 mem_info_addr) +{ + struct arm_smccc_res res; + u64 pa_addr = __pa(mem_info_addr); + + arm_smccc_1_1_smc(TMI_TMM_MEM_INFO_SHOW, pa_addr, &res); + return res.a1; +} +EXPORT_SYMBOL_GPL(tmi_mem_info_show); + +u64 tmi_ttt_map_range(u64 rd, u64 map_addr, u64 size, u64 cur_node, u64 target_node) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_TTT_MAP_RANGE, rd, map_addr, size, cur_node, target_node, &res); + return res.a1; +} + +u64 tmi_ttt_unmap_range(u64 rd, u64 map_addr, u64 size, u64 node_id) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_TTT_UNMAP_RANGE, rd, map_addr, size, node_id, &res); + return res.a1; +} + +u64 tmi_tmm_inf_test(u64 x1, u64 x2, u64 x3, u64 x4, u64 x5) +{ + struct arm_smccc_res res; + u64 vttbr_el2_pa = __pa(x2); + u64 cvm_params_pa = __pa(x3); + u64 tec_params_pa = __pa(x4); + + arm_smccc_1_1_smc(TMI_TMM_INF_TEST, x1, vttbr_el2_pa, cvm_params_pa, tec_params_pa, x5, &res); + return res.a1; +} +EXPORT_SYMBOL_GPL(tmi_tmm_inf_test); + +u64 tmi_smmu_queue_create(u64 params_ptr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_QUEUE_CREATE, params_ptr, &res); + return res.a1; +} + +u64 tmi_smmu_queue_write(uint64_t cmd0, uint64_t cmd1, u64 smmu_id) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_QUEUE_WRITE, cmd0, cmd1, smmu_id, &res); + return res.a1; +} + +u64 tmi_smmu_ste_create(u64 params_ptr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_STE_CREATE, params_ptr, &res); + return res.a1; +} + +u64 tmi_mmio_map(u64 rd, u64 map_addr, u64 level, u64 ttte) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_MAP, rd, map_addr, level, ttte, &res); + return res.a1; +} + +u64 tmi_mmio_unmap(u64 rd, u64 map_addr, u64 level) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_UNMAP, rd, map_addr, level, &res); + return res.a1; +} + +u64 tmi_mmio_write(u64 addr, u64 val, u64 bits, u64 dev_num) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_WRITE, addr, val, bits, dev_num, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_mmio_write); + +u64 tmi_mmio_read(u64 addr, u64 bits, u64 dev_num) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MMIO_READ, addr, bits, dev_num, &res); + return res.a1; +} +EXPORT_SYMBOL(tmi_mmio_read); + +u64 tmi_dev_delegate(u64 params) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DEV_DELEGATE, params, &res); + return res.a1; +} + +u64 tmi_dev_attach(u64 vdev, u64 rd, u64 smmu_id) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DEV_ATTACH, vdev, rd, smmu_id, &res); + return res.a1; +} + +u64 tmi_handle_s_evtq(u64 smmu_id) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_HANDLE_S_EVTQ, smmu_id, &res); + return res.a1; +} + +u64 tmi_smmu_device_reset(u64 params) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_DEVICE_RESET, params, &res); + return res.a1; +} + +u64 tmi_smmu_pcie_core_check(u64 smmu_base) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_PCIE_CORE_CHECK, smmu_base, &res); + return res.a1; +} + +u64 tmi_smmu_write(u64 smmu_base, u64 reg_offset, u64 val, u64 bits) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_WRITE, smmu_base, reg_offset, val, bits, &res); + return res.a1; +} + +u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_SMMU_READ, smmu_base, reg_offset, bits, &res); + return res.a1; +} diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 69ca111e34..f9afb1dd34 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -11,6 +11,10 @@ #include <asm/kvm_mmu.h> #include <asm/kvm_asm.h>

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif + #include "vgic.h"

static bool group0_trap; @@ -680,7 +684,10 @@ int vgic_v3_probe(const struct gic_kvm_info *info) (unsigned long long)info->vcpu.start); } else if (kvm_get_mode() != KVM_MODE_PROTECTED) { kvm_vgic_global_state.vcpu_base = info->vcpu.start; - kvm_vgic_global_state.can_emulate_gicv2 = true; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (!static_branch_unlikely(&virtcca_cvm_is_available)) +#endif + kvm_vgic_global_state.can_emulate_gicv2 = true; ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); if (ret) { kvm_err("Cannot register GICv2 KVM device.\n"); @@ -760,7 +767,13 @@ void vgic_v3_load(struct kvm_vcpu *vcpu) void vgic_v3_vmcr_sync(struct kvm_vcpu *vcpu) { struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) { + cpu_if->vgic_vmcr = + ((struct tmi_tec_run *)vcpu->arch.tec.tec_run)->tec_exit.gicv3_vmcr; + return; + } +#endif if (likely(cpu_if->vgic_sre)) cpu_if->vgic_vmcr = kvm_call_hyp_ret(__vgic_v3_read_vmcr); } diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 2459b0adea..03ba0e01a1 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -12,6 +12,10 @@

#include <asm/kvm_hyp.h>

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif + #include "vgic.h"

#define CREATE_TRACE_POINTS @@ -897,12 +901,44 @@ static inline bool can_access_vgic_from_kernel(void) return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe(); }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +static inline void vgic_tmm_save_state(struct kvm_vcpu *vcpu) +{ + int i; + struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; + struct tmi_tec_run *tec_run = vcpu->arch.tec.tec_run; + + for (i = 0; i < kvm_vgic_global_state.nr_lr; ++i) { + cpu_if->vgic_lr[i] = tec_run->tec_exit.gicv3_lrs[i]; + tec_run->tec_entry.gicv3_lrs[i] = 0; + } +} + +static inline void vgic_tmm_restore_state(struct kvm_vcpu *vcpu) +{ + int i; + struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; + struct tmi_tec_run *tec_run = vcpu->arch.tec.tec_run; + + for (i = 0; i < kvm_vgic_global_state.nr_lr; ++i) { + tec_run->tec_entry.gicv3_lrs[i] = cpu_if->vgic_lr[i]; + tec_run->tec_exit.gicv3_lrs[i] = cpu_if->vgic_lr[i]; + } +} +#endif + static inline void vgic_save_state(struct kvm_vcpu *vcpu) { if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_save_state(vcpu); else - __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + vgic_tmm_save_state(vcpu); + else +#endif + __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); + }

/* Sync back the hardware VGIC state into our emulation after a guest's run. */ @@ -932,7 +968,12 @@ static inline void vgic_restore_state(struct kvm_vcpu *vcpu) if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_restore_state(vcpu); else - __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + vgic_tmm_restore_state(vcpu); + else +#endif + __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); }

/* Flush our emulation state into the GIC hardware before entering the guest. */ @@ -973,7 +1014,10 @@ void kvm_vgic_load(struct kvm_vcpu *vcpu) { if (unlikely(!vgic_initialized(vcpu->kvm))) return; - +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + return; +#endif if (kvm_vgic_global_state.type == VGIC_V2) vgic_v2_load(vcpu); else @@ -984,7 +1028,10 @@ void kvm_vgic_put(struct kvm_vcpu *vcpu) { if (unlikely(!vgic_initialized(vcpu->kvm))) return; - +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + return; +#endif if (kvm_vgic_global_state.type == VGIC_V2) vgic_v2_put(vcpu); else diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c new file mode 100644 index 0000000000..227f70d01a --- /dev/null +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -0,0 +1,1065 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#include <linux/kvm_host.h> +#include <linux/kvm.h> +#include <linux/vfio.h> +#include <asm/kvm_tmi.h> +#include <asm/kvm_pgtable.h> +#include <asm/kvm_emulate.h> +#include <asm/kvm_mmu.h> +#include <asm/stage2_pgtable.h> +#include <linux/arm-smccc.h> +#include <kvm/arm_hypercalls.h> +#include <kvm/arm_psci.h> +#include "../virt/kvm/vfio.h" + +/* Protects access to cvm_vmid_bitmap */ +static DEFINE_SPINLOCK(cvm_vmid_lock); +static unsigned long *cvm_vmid_bitmap; +DEFINE_STATIC_KEY_FALSE(virtcca_cvm_is_available); +DEFINE_STATIC_KEY_FALSE(virtcca_cvm_is_enable); +#define SIMD_PAGE_SIZE 0x3000 + +int kvm_enable_virtcca_cvm(struct kvm *kvm) +{ + if (!static_key_enabled(&virtcca_cvm_is_available)) + return -EFAULT; + + kvm->arch.is_virtcca_cvm = true; + return 0; +} + +static int __init setup_virtcca_cvm_host(char *str) +{ + int ret; + unsigned int val; + + if (!str) + return 0; + + ret = kstrtouint(str, 10, &val); + if (ret) { + pr_warn("Unable to parse cvm_guest.\n"); + } else { + if (val) + static_branch_enable(&virtcca_cvm_is_enable); + } + return ret; +} +early_param("virtcca_cvm_host", setup_virtcca_cvm_host); + +static int cvm_vmid_init(void) +{ + unsigned int vmid_count = 1 << kvm_get_vmid_bits(); + + cvm_vmid_bitmap = bitmap_zalloc(vmid_count, GFP_KERNEL); + if (!cvm_vmid_bitmap) { + kvm_err("%s: Couldn't allocate cvm vmid bitmap\n", __func__); + return -ENOMEM; + } + return 0; +} + +static unsigned long tmm_feat_reg0; + +static bool tmm_supports(unsigned long feature) +{ + return !!u64_get_bits(tmm_feat_reg0, feature); +} + +bool kvm_cvm_supports_sve(void) +{ + return tmm_supports(TMI_FEATURE_REGISTER_0_SVE_EN); +} + +bool kvm_cvm_supports_pmu(void) +{ + return tmm_supports(TMI_FEATURE_REGISTER_0_PMU_EN); +} + +u32 kvm_cvm_ipa_limit(void) +{ + return u64_get_bits(tmm_feat_reg0, TMI_FEATURE_REGISTER_0_S2SZ); +} + +u32 kvm_cvm_get_num_brps(void) +{ + return u64_get_bits(tmm_feat_reg0, TMI_FEATURE_REGISTER_0_NUM_BPS); +} + +u32 kvm_cvm_get_num_wrps(void) +{ + return u64_get_bits(tmm_feat_reg0, TMI_FEATURE_REGISTER_0_NUM_WPS); +} + +static int cvm_vmid_reserve(void) +{ + int ret; + unsigned int vmid_count = 1 << kvm_get_vmid_bits(); + + spin_lock(&cvm_vmid_lock); + ret = bitmap_find_free_region(cvm_vmid_bitmap, vmid_count, 0); + spin_unlock(&cvm_vmid_lock); + + return ret; +} + +static void cvm_vmid_release(unsigned int vmid) +{ + spin_lock(&cvm_vmid_lock); + bitmap_release_region(cvm_vmid_bitmap, vmid, 0); + spin_unlock(&cvm_vmid_lock); +} + +static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) +{ + u64 shift = ARM64_HW_PGTABLE_LEVEL_SHIFT(pgt->start_level - 1); + u64 mask = BIT(pgt->ia_bits) - 1; + + return (addr & mask) >> shift; +} + +static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level) +{ + struct kvm_pgtable pgt = { + .ia_bits = ia_bits, + .start_level = start_level, + }; + return __kvm_pgd_page_idx(&pgt, -1ULL) + 1; +} + +/* + * the configurable physical numa range in QEMU is 0-127, + * but in real scenarios, 0-63 is sufficient. + */ +static u64 kvm_get_host_numa_set_by_vcpu(u64 vcpu, struct kvm *kvm) +{ + int64_t i; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_numa_info *numa_info = &cvm->numa_info; + + for (i = 0; i < numa_info->numa_cnt && i < MAX_NUMA_NODE; i++) { + if (test_bit(vcpu, (unsigned long *)numa_info->numa_nodes[i].cpu_id)) + return numa_info->numa_nodes[i].host_numa_nodes[0]; + } + return NO_NUMA; +} + +static u64 kvm_get_first_binded_numa_set(struct kvm *kvm) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_numa_info *numa_info = &cvm->numa_info; + + if (numa_info->numa_cnt > 0) + return numa_info->numa_nodes[0].host_numa_nodes[0]; + return NO_NUMA; +} + +int cvm_arm_smmu_domain_set_kvm(struct iommu_group *group) +{ + struct arm_smmu_domain *arm_smmu_domain = NULL; + struct iommu_domain *domain; + struct kvm *kvm; + + domain = iommu_group_get_domain(group); + if (!domain) + return -ENXIO; + + arm_smmu_domain = to_smmu_domain(domain); + if (arm_smmu_domain->kvm) + return 0; + + kvm = arm_smmu_get_kvm(arm_smmu_domain); + if (kvm && kvm_is_virtcca_cvm(kvm)) + arm_smmu_domain->kvm = kvm; + + return 0; +} + +int kvm_arm_create_cvm(struct kvm *kvm) +{ + int ret; + struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; + unsigned int pgd_sz; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + /* get affine host numa set by default vcpu 0 */ + u64 numa_set = kvm_get_host_numa_set_by_vcpu(0, kvm); + + if (!kvm_is_virtcca_cvm(kvm) || virtcca_cvm_state(kvm) != CVM_STATE_NONE) + return 0; + + if (!cvm->params) { + ret = -EFAULT; + goto out; + } + + ret = cvm_vmid_reserve(); + if (ret < 0) + goto out; + + cvm->cvm_vmid = ret; + + pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level); + + cvm->params->ttt_level_start = kvm->arch.mmu.pgt->start_level; + cvm->params->ttt_num_start = pgd_sz; + cvm->params->s2sz = VTCR_EL2_IPA(kvm->arch.vtcr); + cvm->params->vmid = cvm->cvm_vmid; + cvm->params->ns_vtcr = kvm->arch.vtcr; + cvm->params->vttbr_el2 = kvm->arch.mmu.pgd_phys; + memcpy(cvm->params->rpv, &cvm->cvm_vmid, sizeof(cvm->cvm_vmid)); + cvm->rd = tmi_cvm_create(__pa(cvm->params), numa_set); + if (!cvm->rd) { + kvm_err("KVM creates cVM failed: %d\n", cvm->cvm_vmid); + ret = -ENOMEM; + goto out; + } + + WRITE_ONCE(cvm->state, CVM_STATE_NEW); + ret = 0; +out: + kfree(cvm->params); + cvm->params = NULL; + if (ret < 0) { + kfree(cvm); + kvm->arch.virtcca_cvm = NULL; + } + return ret; +} + +void kvm_destroy_cvm(struct kvm *kvm) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + uint32_t cvm_vmid; + struct arm_smmu_domain *arm_smmu_domain; + struct list_head smmu_domain_group_list; + + if (!cvm) + return; + + kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); + list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { + if (arm_smmu_domain->kvm && arm_smmu_domain->kvm == kvm) + arm_smmu_domain->kvm = NULL; + } + + cvm_vmid = cvm->cvm_vmid; + kfree(cvm->params); + cvm->params = NULL; + + if (virtcca_cvm_state(kvm) == CVM_STATE_NONE) + return; + + cvm_vmid_release(cvm_vmid); + + WRITE_ONCE(cvm->state, CVM_STATE_DYING); + + if (!tmi_cvm_destroy(cvm->rd)) + kvm_info("KVM has destroyed cVM: %d\n", cvm->cvm_vmid); + + cvm->is_mapped = false; + kfree(cvm); + kvm->arch.virtcca_cvm = NULL; +} + +static int kvm_cvm_ttt_create(struct virtcca_cvm *cvm, + unsigned long addr, + int level, + u64 numa_set) +{ + addr = ALIGN_DOWN(addr, cvm_ttt_level_mapsize(level - 1)); + return tmi_ttt_create(numa_set, cvm->rd, addr, level); +} + +int kvm_cvm_create_ttt_levels(struct kvm *kvm, struct virtcca_cvm *cvm, + unsigned long ipa, + int level, + int max_level, + struct kvm_mmu_memory_cache *mc) +{ + int ret = 0; + if (WARN_ON(level == max_level)) + return 0; + + while (level++ < max_level) { + u64 numa_set = kvm_get_first_binded_numa_set(kvm); + + ret = kvm_cvm_ttt_create(cvm, ipa, level, numa_set); + if (ret) + return -ENXIO; + } + + return 0; +} + +static int kvm_cvm_map_unmap_ipa_internal(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size, uint32_t is_map) +{ + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + phys_addr_t rd = virtcca_cvm->rd; + unsigned long ipa = ipa_base; + unsigned long phys = pa; + unsigned long size; + int map_level = 3; + int ret = 0; + + for (size = 0; size < map_size; size += PAGE_SIZE) { + if (is_map) + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + else + ret = tmi_mmio_unmap(rd, ipa, CVM_TTT_MAX_LEVEL); + + if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { + /* Create missing TTTs and retry */ + int level_fault = TMI_RETURN_INDEX(ret); + + if (is_map) { + ret = kvm_cvm_create_ttt_levels(kvm, virtcca_cvm, ipa, level_fault, + CVM_TTT_MAX_LEVEL, NULL); + if (ret) + goto err; + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + } else { + ret = tmi_mmio_unmap(rd, ipa, level_fault); + } + } + + WARN_ON(ret); + if (ret) + goto err; + + if (size + PAGE_SIZE >= map_size) + break; + ipa += PAGE_SIZE; + phys += PAGE_SIZE; + } + + if (WARN_ON(ret)) + goto err; + return 0; + +err: + while (size > 0) { + phys -= PAGE_SIZE; + size -= PAGE_SIZE; + ipa -= PAGE_SIZE; + + WARN_ON(tmi_data_destroy(rd, ipa, map_level)); + } + return -ENXIO; +} + +int kvm_cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size, uint32_t is_map) +{ + return kvm_cvm_map_unmap_ipa_internal(kvm, ipa_base, pa, map_size, is_map); +} + +int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size) +{ + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + phys_addr_t rd = virtcca_cvm->rd; + unsigned long ipa = ipa_base; + unsigned long phys = pa; + unsigned long size; + int map_level = 3; + int ret = 0; + gfn_t gfn; + kvm_pfn_t pfn; + + if (WARN_ON(!IS_ALIGNED(ipa, map_size))) + return -EINVAL; + + for (size = 0; size < map_size; size += PAGE_SIZE) { + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + if (ret == TMI_ERROR_TTT_CREATED) { + ret = 0; + goto label; + } + if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { + /* Create missing TTTs and retry */ + int level_fault = TMI_RETURN_INDEX(ret); + + ret = kvm_cvm_create_ttt_levels(kvm, virtcca_cvm, ipa, level_fault, + CVM_TTT_MAX_LEVEL, NULL); + + if (ret) + goto err; + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + } + + WARN_ON(ret); + if (ret) + goto err; +label: + if (size + PAGE_SIZE >= map_size) + break; + + ipa += PAGE_SIZE; + gfn = gpa_to_gfn(ipa); + pfn = gfn_to_pfn(kvm, gfn); + kvm_set_pfn_accessed(pfn); + kvm_release_pfn_clean(pfn); + phys = (uint64_t)__pfn_to_phys(pfn); + + } + if (WARN_ON(ret)) + goto err; + + return 0; + +err: + while (size > 0) { + phys -= PAGE_SIZE; + size -= PAGE_SIZE; + ipa -= PAGE_SIZE; + WARN_ON(tmi_data_destroy(rd, ipa, map_level)); + } + return -ENXIO; +} + +static int kvm_cvm_create_protected_data_page(struct kvm *kvm, struct virtcca_cvm *cvm, + unsigned long ipa, int level, struct page *src_page, u64 numa_set) +{ + phys_addr_t src_phys = 0; + int ret; + + if (src_page) + src_phys = page_to_phys(src_page); + ret = tmi_data_create(numa_set, cvm->rd, ipa, src_phys, level); + + if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { + /* Create missing RTTs and retry */ + int level_fault = TMI_RETURN_INDEX(ret); + + ret = kvm_cvm_create_ttt_levels(kvm, cvm, ipa, level_fault, + level, NULL); + if (ret) + goto err; + ret = tmi_data_create(numa_set, cvm->rd, ipa, src_phys, level); + } + if (ret) + goto err; + + return 0; + +err: + kvm_err("Cvm create protected data page fail:%d\n", ret); + return ret; +} + +static u64 cvm_granule_size(u32 level) +{ + return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); +} + +static bool is_data_create_region(phys_addr_t ipa_base, + struct kvm_cap_arm_tmm_populate_region_args *args) +{ + if ((ipa_base >= args->populate_ipa_base1 && + ipa_base < args->populate_ipa_base1 + args->populate_ipa_size1) || + (ipa_base >= args->populate_ipa_base2 && + ipa_base < args->populate_ipa_base2 + args->populate_ipa_size2)) + return true; + return false; +} + +int kvm_cvm_populate_par_region(struct kvm *kvm, u64 numa_set, + phys_addr_t ipa_base, phys_addr_t ipa_end, + struct kvm_cap_arm_tmm_populate_region_args *args) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_memory_slot *memslot; + gfn_t base_gfn, end_gfn; + int idx; + phys_addr_t ipa; + int ret = 0; + int level = TMM_TTT_LEVEL_3; + unsigned long map_size = cvm_granule_size(level); + + base_gfn = gpa_to_gfn(ipa_base); + end_gfn = gpa_to_gfn(ipa_end); + + idx = srcu_read_lock(&kvm->srcu); + memslot = gfn_to_memslot(kvm, base_gfn); + if (!memslot) { + ret = -EFAULT; + goto out; + } + + /* We require the region to be contained within a single memslot */ + if (memslot->base_gfn + memslot->npages < end_gfn) { + ret = -EINVAL; + goto out; + } + + mmap_read_lock(current->mm); + + ipa = ipa_base; + while (ipa < ipa_end) { + struct page *page = NULL; + kvm_pfn_t pfn = 0; + + /* + * FIXME: This causes over mapping, but there's no good + * solution here with the ABI as it stands + */ + ipa = ALIGN_DOWN(ipa, map_size); + + if (is_data_create_region(ipa, args)) { + pfn = gfn_to_pfn_memslot(memslot, gpa_to_gfn(ipa)); + if (is_error_pfn(pfn)) { + ret = -EFAULT; + break; + } + + page = pfn_to_page(pfn); + } + + ret = kvm_cvm_create_protected_data_page(kvm, cvm, ipa, level, page, numa_set); + if (ret) + goto err_release_pfn; + + ipa += map_size; + if (pfn) + kvm_release_pfn_dirty(pfn); +err_release_pfn: + if (ret) { + if (pfn) + kvm_release_pfn_clean(pfn); + break; + } + } + + mmap_read_unlock(current->mm); +out: + srcu_read_unlock(&kvm->srcu, idx); + return ret; +} + +int kvm_finalize_vcpu_tec(struct kvm_vcpu *vcpu) +{ + int ret = 0; + int i; + u64 numa_set; + struct tmi_tec_params *params_ptr = NULL; + struct user_pt_regs *vcpu_regs = vcpu_gp_regs(vcpu); + u64 mpidr = kvm_vcpu_get_mpidr_aff(vcpu); + struct virtcca_cvm *cvm = vcpu->kvm->arch.virtcca_cvm; + struct virtcca_cvm_tec *tec = &vcpu->arch.tec; + + mutex_lock(&vcpu->kvm->lock); + tec->tec_run = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!tec->tec_run) { + ret = -ENOMEM; + goto tec_free; + } + params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!params_ptr) { + ret = -ENOMEM; + goto tec_free; + } + + for (i = 0; i < TEC_CREATE_NR_GPRS; ++i) + params_ptr->gprs[i] = vcpu_regs->regs[i]; + + params_ptr->pc = vcpu_regs->pc; + + if (vcpu->vcpu_id == 0) + params_ptr->flags = TMI_RUNNABLE; + else + params_ptr->flags = TMI_NOT_RUNNABLE; + params_ptr->ram_size = cvm->ram_size; + numa_set = kvm_get_host_numa_set_by_vcpu(vcpu->vcpu_id, vcpu->kvm); + tec->tec = tmi_tec_create(numa_set, cvm->rd, mpidr, __pa(params_ptr)); + + tec->tec_created = true; + kfree(params_ptr); + mutex_unlock(&vcpu->kvm->lock); + return ret; + +tec_free: + kfree(tec->tec_run); + kfree(params_ptr); + mutex_unlock(&vcpu->kvm->lock); + return ret; +} + +static int config_cvm_hash_algo(struct tmi_cvm_params *params, + struct kvm_cap_arm_tmm_config_item *cfg) +{ + switch (cfg->hash_algo) { + case KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA256: + if (!tmm_supports(TMI_FEATURE_REGISTER_0_HASH_SHA_256)) + return -EINVAL; + break; + case KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA512: + if (!tmm_supports(TMI_FEATURE_REGISTER_0_HASH_SHA_512)) + return -EINVAL; + break; + default: + return -EINVAL; + } + params->measurement_algo = cfg->hash_algo; + return 0; +} + +static int config_cvm_sve(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_cvm_params *params; + int max_sve_vq; + + params = cvm->params; + max_sve_vq = u64_get_bits(tmm_feat_reg0, + TMI_FEATURE_REGISTER_0_SVE_VL); + + if (!kvm_cvm_supports_sve()) + return -EINVAL; + + if (cfg->sve_vq > max_sve_vq) + return -EINVAL; + + params->sve_vl = cfg->sve_vq; + params->flags |= TMI_CVM_PARAM_FLAG_SVE; + + return 0; +} + +static int config_cvm_pmu(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_cvm_params *params; + int max_pmu_num_ctrs; + + params = cvm->params; + max_pmu_num_ctrs = u64_get_bits(tmm_feat_reg0, + TMI_FEATURE_REGISTER_0_PMU_NUM_CTRS); + + if (!kvm_cvm_supports_pmu()) + return -EINVAL; + + if (cfg->num_pmu_cntrs > max_pmu_num_ctrs) + return -EINVAL; + + params->pmu_num_cnts = cfg->num_pmu_cntrs; + params->flags |= TMI_CVM_PARAM_FLAG_PMU; + + return 0; +} + +static int kvm_tmm_config_cvm(struct kvm *kvm, struct kvm_enable_cap *cap) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_cap_arm_tmm_config_item cfg; + int r = 0; + + if (virtcca_cvm_state(kvm) != CVM_STATE_NONE) + return -EBUSY; + + if (copy_from_user(&cfg, (void __user *)cap->args[1], sizeof(cfg))) + return -EFAULT; + + switch (cfg.cfg) { + case KVM_CAP_ARM_TMM_CFG_SVE: + r = config_cvm_sve(kvm, &cfg); + break; + case KVM_CAP_ARM_TMM_CFG_PMU: + r = config_cvm_pmu(kvm, &cfg); + break; + case KVM_CAP_ARM_TMM_CFG_HASH_ALGO: + r = config_cvm_hash_algo(cvm->params, &cfg); + break; + default: + r = -EINVAL; + } + + return r; +} + +int kvm_cvm_map_range(struct kvm *kvm) +{ + int ret; + u64 curr_numa_set; + int idx; + u64 l2_granule = cvm_granule_size(TMM_TTT_LEVEL_2); + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_numa_info *numa_info = &cvm->numa_info; + gpa_t gpa; + + curr_numa_set = kvm_get_first_binded_numa_set(kvm); + gpa = round_up(cvm->dtb_end, l2_granule); + for (idx = 0; idx < numa_info->numa_cnt; idx++) { + struct kvm_numa_node *numa_node = &numa_info->numa_nodes[idx]; + + if (idx) + gpa = numa_node->ipa_start; + if (gpa >= numa_node->ipa_start && + gpa < numa_node->ipa_start + numa_node->ipa_size) { + ret = tmi_ttt_map_range(cvm->rd, gpa, + numa_node->ipa_size - gpa + numa_node->ipa_start, + curr_numa_set, numa_node->host_numa_nodes[0]); + if (ret) { + kvm_err("tmi_ttt_map_range failed: %d.\n", ret); + return ret; + } + } + } + + cvm->is_mapped = true; + return ret; +} + +static int kvm_activate_cvm(struct kvm *kvm) +{ + int ret; + struct arm_smmu_domain *arm_smmu_domain; + struct list_head smmu_domain_group_list; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + + if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) + return -EINVAL; + + if (!cvm->is_mapped && kvm_cvm_map_range(kvm)) + return -EFAULT; + + if (kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list)) { + kvm_err("tmi activate cvm: get arm smmu domain failed!\n"); + return -EFAULT; + } + + list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { + if (arm_smmu_domain && arm_smmu_domain->secure) { + ret = arm_smmu_tmi_dev_attach(arm_smmu_domain, kvm); + if (ret) + return ret; + } + } + + if (tmi_cvm_activate(cvm->rd)) { + kvm_err("tmi_cvm_activate failed!\n"); + return -ENXIO; + } + + WRITE_ONCE(cvm->state, CVM_STATE_ACTIVE); + kvm_info("cVM%d is activated!\n", cvm->cvm_vmid); + return 0; +} + +static int kvm_populate_ram_region(struct kvm *kvm, u64 map_size, + phys_addr_t ipa_base, phys_addr_t ipa_end, + struct kvm_cap_arm_tmm_populate_region_args *args) +{ + phys_addr_t gpa; + u64 numa_set = kvm_get_first_binded_numa_set(kvm); + + for (gpa = ipa_base; gpa < ipa_end; gpa += map_size) { + if (kvm_cvm_populate_par_region(kvm, numa_set, gpa, gpa + map_size, args)) { + kvm_err("kvm_cvm_populate_par_region failed: %d\n", -EFAULT); + return -EFAULT; + } + } + return 0; +} + +static int kvm_populate_ipa_cvm_range(struct kvm *kvm, + struct kvm_cap_arm_tmm_populate_region_args *args) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + u64 l2_granule = cvm_granule_size(TMM_TTT_LEVEL_2); + phys_addr_t ipa_base1, ipa_end2; + + if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) + return -EINVAL; + if (!IS_ALIGNED(args->populate_ipa_base1, PAGE_SIZE) || + !IS_ALIGNED(args->populate_ipa_size1, PAGE_SIZE) || + !IS_ALIGNED(args->populate_ipa_base2, PAGE_SIZE) || + !IS_ALIGNED(args->populate_ipa_size2, PAGE_SIZE)) + return -EINVAL; + + if (args->populate_ipa_base1 < cvm->loader_start || + args->populate_ipa_base2 < args->populate_ipa_base1 + args->populate_ipa_size1 || + cvm->dtb_end < args->populate_ipa_base2 + args->populate_ipa_size2) + return -EINVAL; + + if (args->flags & ~TMI_MEASURE_CONTENT) + return -EINVAL; + ipa_base1 = round_down(args->populate_ipa_base1, l2_granule); + ipa_end2 = round_up(args->populate_ipa_base2 + args->populate_ipa_size2, l2_granule); + + return kvm_populate_ram_region(kvm, l2_granule, ipa_base1, ipa_end2, args); +} + +int kvm_cvm_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) +{ + int r = 0; + + mutex_lock(&kvm->lock); + switch (cap->args[0]) { + case KVM_CAP_ARM_TMM_CONFIG_CVM_HOST: + r = kvm_tmm_config_cvm(kvm, cap); + break; + case KVM_CAP_ARM_TMM_CREATE_RD: + r = kvm_arm_create_cvm(kvm); + break; + case KVM_CAP_ARM_TMM_POPULATE_CVM: { + struct kvm_cap_arm_tmm_populate_region_args args; + void __user *argp = u64_to_user_ptr(cap->args[1]); + + if (copy_from_user(&args, argp, sizeof(args))) { + r = -EFAULT; + break; + } + r = kvm_populate_ipa_cvm_range(kvm, &args); + break; + } + case KVM_CAP_ARM_TMM_ACTIVATE_CVM: + r = kvm_activate_cvm(kvm); + break; + default: + r = -EINVAL; + break; + } + mutex_unlock(&kvm->lock); + + return r; +} + +void kvm_destroy_tec(struct kvm_vcpu *vcpu) +{ + struct virtcca_cvm_tec *tec = &vcpu->arch.tec; + + if (!vcpu_is_tec(vcpu)) + return; + + if (tmi_tec_destroy(tec->tec) != 0) + kvm_err("%s vcpu id : %d failed!\n", __func__, vcpu->vcpu_id); + + tec->tec = 0; + kfree(tec->tec_run); +} + +static int tmi_check_version(void) +{ + u64 res; + int version_major; + int version_minor; + + res = tmi_version(); + if (res == SMCCC_RET_NOT_SUPPORTED) + return -ENXIO; + + version_major = TMI_ABI_VERSION_GET_MAJOR(res); + version_minor = TMI_ABI_VERSION_GET_MINOR(res); + + if (version_major != TMI_ABI_VERSION_MAJOR) { + kvm_err("Unsupported TMI_ABI (version %d %d)\n", version_major, + version_minor); + return -ENXIO; + } + + kvm_info("TMI ABI version %d,%d\n", version_major, version_minor); + return 0; +} + +int kvm_tec_enter(struct kvm_vcpu *vcpu) +{ + struct tmi_tec_run *run; + struct virtcca_cvm_tec *tec = &vcpu->arch.tec; + struct virtcca_cvm *cvm = vcpu->kvm->arch.virtcca_cvm; + + if (READ_ONCE(cvm->state) != CVM_STATE_ACTIVE) + return -EINVAL; + + run = tec->tec_run; + /* set/clear TWI TWE flags */ + if (vcpu->arch.hcr_el2 & HCR_TWI) + run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFI; + else + run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFI; + + if (vcpu->arch.hcr_el2 & HCR_TWE) + run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFE; + else + run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFE; + + return tmi_tec_enter(tec->tec, __pa(run)); +} + +int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target) +{ + int ret; + struct virtcca_cvm_tec *calling_tec = &calling->arch.tec; + struct virtcca_cvm_tec *target_tec = &target->arch.tec; + + ret = tmi_psci_complete(calling_tec->tec, target_tec->tec); + if (ret) + return -EINVAL; + return 0; +} + +int kvm_init_tmm(void) +{ + int ret; + + if (PAGE_SIZE != SZ_4K) + return 0; + + if (tmi_check_version()) + return 0; + + ret = cvm_vmid_init(); + if (ret) + return ret; + + tmm_feat_reg0 = tmi_features(0); + kvm_info("TMM feature0: 0x%lx\n", tmm_feat_reg0); + + static_branch_enable(&virtcca_cvm_is_available); + + return 0; +} + +static bool is_numa_ipa_range_valid(struct kvm_numa_info *numa_info) +{ + unsigned long i; + struct kvm_numa_node *numa_node, *prev_numa_node; + + prev_numa_node = NULL; + for (i = 0; i < numa_info->numa_cnt; i++) { + numa_node = &numa_info->numa_nodes[i]; + if (numa_node->ipa_start + numa_node->ipa_size < numa_node->ipa_start) + return false; + if (prev_numa_node && + numa_node->ipa_start < prev_numa_node->ipa_start + prev_numa_node->ipa_size) + return false; + prev_numa_node = numa_node; + } + if (numa_node->ipa_start + numa_node->ipa_size > CVM_IPA_MAX_VAL) + return false; + return true; +} + +int kvm_load_user_data(struct kvm *kvm, unsigned long arg) +{ + struct kvm_user_data user_data; + void __user *argp = (void __user *)arg; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_numa_info *numa_info; + + if (!kvm_is_virtcca_cvm(kvm)) + return -EFAULT; + + if (copy_from_user(&user_data, argp, sizeof(user_data))) + return -EINVAL; + + numa_info = &user_data.numa_info; + if (numa_info->numa_cnt > MAX_NUMA_NODE) + return -EINVAL; + + if (numa_info->numa_cnt > 0) { + unsigned long i, total_size = 0; + struct kvm_numa_node *numa_node = &numa_info->numa_nodes[0]; + unsigned long ipa_end = numa_node->ipa_start + numa_node->ipa_size; + + if (!is_numa_ipa_range_valid(numa_info)) + return -EINVAL; + if (user_data.loader_start < numa_node->ipa_start || + user_data.dtb_end > ipa_end) + return -EINVAL; + for (i = 0; i < numa_info->numa_cnt; i++) + total_size += numa_info->numa_nodes[i].ipa_size; + if (total_size != user_data.ram_size) + return -EINVAL; + } + + if (user_data.image_end <= user_data.loader_start || + user_data.initrd_start < user_data.image_end || + user_data.dtb_end < user_data.initrd_start || + user_data.ram_size < user_data.dtb_end - user_data.loader_start) + return -EINVAL; + + cvm->loader_start = user_data.loader_start; + cvm->image_end = user_data.image_end; + cvm->initrd_start = user_data.initrd_start; + cvm->dtb_end = user_data.dtb_end; + cvm->ram_size = user_data.ram_size; + memcpy(&cvm->numa_info, numa_info, sizeof(struct kvm_numa_info)); + + return 0; +} + +void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + vcpu->cpu = -1; +} + +unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, + unsigned long target_affinity, unsigned long lowest_affinity_level) +{ + struct kvm_vcpu *target_vcpu; + + if (lowest_affinity_level != 0) + return PSCI_RET_INVALID_PARAMS; + + target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, target_affinity); + if (!target_vcpu) + return PSCI_RET_INVALID_PARAMS; + + cvm_psci_complete(vcpu, target_vcpu); + return PSCI_RET_SUCCESS; +} + +int kvm_cvm_vcpu_set_events(struct kvm_vcpu *vcpu, + bool serror_pending, bool ext_dabt_pending) +{ + struct virtcca_cvm_tec *tec = &vcpu->arch.tec; + + if (serror_pending) + return -EINVAL; + + if (ext_dabt_pending) { + if (!(((struct tmi_tec_run *)tec->tec_run)->tec_entry.flags & + TEC_ENTRY_FLAG_EMUL_MMIO)) + return -EINVAL; + + ((struct tmi_tec_run *)tec->tec_run)->tec_entry.flags + &= ~TEC_ENTRY_FLAG_EMUL_MMIO; + ((struct tmi_tec_run *)tec->tec_run)->tec_entry.flags + |= TEC_ENTRY_FLAG_INJECT_SEA; + } + return 0; +} + +int kvm_init_cvm_vm(struct kvm *kvm) +{ + struct tmi_cvm_params *params; + struct virtcca_cvm *cvm; + + if (kvm->arch.virtcca_cvm) { + kvm_info("cvm already create.\n"); + return 0; + } + + cvm = (struct virtcca_cvm *)kzalloc(sizeof(struct virtcca_cvm), GFP_KERNEL_ACCOUNT); + if (!cvm) + return -ENOMEM; + + kvm->arch.virtcca_cvm = cvm; + params = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!params) { + kfree(kvm->arch.virtcca_cvm); + kvm->arch.virtcca_cvm = NULL; + return -ENOMEM; + } + + cvm->params = params; + WRITE_ONCE(cvm->state, CVM_STATE_NONE); + + return 0; +} diff --git a/arch/arm64/kvm/virtcca_cvm_exit.c b/arch/arm64/kvm/virtcca_cvm_exit.c new file mode 100644 index 0000000000..9654375a9c --- /dev/null +++ b/arch/arm64/kvm/virtcca_cvm_exit.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#include <linux/kvm_host.h> +#include <kvm/arm_hypercalls.h> +#include <kvm/arm_psci.h> + +#include <asm/kvm_tmi.h> +#include <asm/kvm_emulate.h> +#include <asm/kvm_mmu.h> + +typedef int (*exit_handler_fn)(struct kvm_vcpu *vcpu); + +static void update_arch_timer_irq_lines(struct kvm_vcpu *vcpu, bool unmask_ctl) +{ + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + + __vcpu_sys_reg(vcpu, CNTV_CTL_EL0) = run->tec_exit.cntv_ctl; + __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0) = run->tec_exit.cntv_cval; + __vcpu_sys_reg(vcpu, CNTP_CTL_EL0) = run->tec_exit.cntp_ctl; + __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = run->tec_exit.cntp_cval; + + /* Because the timer mask is tainted by TMM, we don't know the + * true intent of the guest. Here, we assume mask is always + * cleared during WFI. + */ + if (unmask_ctl) { + __vcpu_sys_reg(vcpu, CNTV_CTL_EL0) &= ~ARCH_TIMER_CTRL_IT_MASK; + __vcpu_sys_reg(vcpu, CNTP_CTL_EL0) &= ~ARCH_TIMER_CTRL_IT_MASK; + } + + kvm_cvm_timers_update(vcpu); +} + +static int tec_exit_reason_notimpl(struct kvm_vcpu *vcpu) +{ + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + + pr_err("[vcpu %d] Unhandled exit reason from cvm (ESR: %#llx)\n", + vcpu->vcpu_id, run->tec_exit.esr); + return -ENXIO; +} + +/* The process is the same as kvm_handle_wfx, + * except the tracing and updating operation for pc, + * we copy kvm_handle_wfx process here + * to avoid changing kvm_handle_wfx function. + */ +static int tec_exit_wfx(struct kvm_vcpu *vcpu) +{ + u64 esr = kvm_vcpu_get_esr(vcpu); + + if (esr & ESR_ELx_WFx_ISS_WFE) { + vcpu->stat.wfe_exit_stat++; + } else { + vcpu->stat.wfi_exit_stat++; + } + + if (esr & ESR_ELx_WFx_ISS_WFxT) { + if (esr & ESR_ELx_WFx_ISS_RV) { + u64 val, now; + + now = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_TIMER_CNT); + val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); + + if (now >= val) + goto out; + } else { + /* Treat WFxT as WFx if RN is invalid */ + esr &= ~ESR_ELx_WFx_ISS_WFxT; + } + } + + if (esr & ESR_ELx_WFx_ISS_WFE) { + kvm_vcpu_on_spin(vcpu, vcpu_mode_priv(vcpu)); + } else { + if (esr & ESR_ELx_WFx_ISS_WFxT) + vcpu_set_flag(vcpu, IN_WFIT); + + kvm_vcpu_wfi(vcpu); + } + +out: + return 1; +} + +static int tec_exit_sys_reg(struct kvm_vcpu *vcpu) +{ + int ret; + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + unsigned long esr = kvm_vcpu_get_esr(vcpu); + int rt = kvm_vcpu_sys_get_rt(vcpu); + bool is_write = !(esr & 1); + + if (is_write) + vcpu_set_reg(vcpu, rt, run->tec_exit.gprs[0]); + + ret = kvm_handle_sys_reg(vcpu); + + if (ret >= 0 && !is_write) + run->tec_entry.gprs[0] = vcpu_get_reg(vcpu, rt); + + return ret; +} + +static int tec_exit_sync_dabt(struct kvm_vcpu *vcpu) +{ + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + + if (kvm_vcpu_dabt_iswrite(vcpu) && kvm_vcpu_dabt_isvalid(vcpu)) { + vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), + run->tec_exit.gprs[0]); + } + return kvm_handle_guest_abort(vcpu); +} + +static int tec_exit_sync_iabt(struct kvm_vcpu *vcpu) +{ + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + + pr_err("[vcpu %d] Unhandled instruction abort (ESR: %#llx).\n", + vcpu->vcpu_id, run->tec_exit.esr); + + return -ENXIO; +} + +static exit_handler_fn tec_exit_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = tec_exit_reason_notimpl, + [ESR_ELx_EC_WFx] = tec_exit_wfx, + [ESR_ELx_EC_SYS64] = tec_exit_sys_reg, + [ESR_ELx_EC_DABT_LOW] = tec_exit_sync_dabt, + [ESR_ELx_EC_IABT_LOW] = tec_exit_sync_iabt +}; + +static int tec_exit_psci(struct kvm_vcpu *vcpu) +{ + int i; + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + + for (i = 0; i < TEC_EXIT_NR_GPRS; ++i) + vcpu_set_reg(vcpu, i, run->tec_exit.gprs[i]); + + return kvm_psci_call(vcpu); +} + +static int tec_exit_host_call(struct kvm_vcpu *vcpu) +{ + int ret, i; + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + + vcpu->stat.hvc_exit_stat++; + + for (i = 0; i < TEC_EXIT_NR_GPRS; ++i) + vcpu_set_reg(vcpu, i, run->tec_exit.gprs[i]); + + ret = kvm_smccc_call_handler(vcpu); + + if (ret < 0) { + vcpu_set_reg(vcpu, 0, ~0UL); + ret = 1; + } + for (i = 0; i < TEC_EXIT_NR_GPRS; ++i) + run->tec_entry.gprs[i] = vcpu_get_reg(vcpu, i); + + return ret; +} + +/* + * Return > 0 to return to guest, < 0 on error, 0(and set exit_reason) on + * proper exit to userspace + */ + +int handle_cvm_exit(struct kvm_vcpu *vcpu, int tec_run_ret) +{ + unsigned long status; + struct tmi_tec_run *run = vcpu->arch.tec.tec_run; + u8 esr_ec = ESR_ELx_EC(run->tec_exit.esr); + bool is_wfx; + + status = TMI_RETURN_STATUS(tec_run_ret); + + if (status == TMI_ERROR_CVM_POWEROFF) { + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SHUTDOWN; + return 0; + } + + if (status == TMI_ERROR_CVM_STATE) { + vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; + return 0; + } + + if (tec_run_ret) + return -ENXIO; + + vcpu->arch.fault.esr_el2 = run->tec_exit.esr; + vcpu->arch.fault.far_el2 = run->tec_exit.far; + vcpu->arch.fault.hpfar_el2 = run->tec_exit.hpfar; + + is_wfx = (run->tec_exit.exit_reason == TMI_EXIT_SYNC) && (esr_ec == ESR_ELx_EC_WFx); + update_arch_timer_irq_lines(vcpu, is_wfx); + + run->tec_entry.flags = 0; + + switch (run->tec_exit.exit_reason) { + case TMI_EXIT_FIQ: + case TMI_EXIT_IRQ: + return 1; + case TMI_EXIT_PSCI: + return tec_exit_psci(vcpu); + case TMI_EXIT_SYNC: + return tec_exit_handlers[esr_ec](vcpu); + case TMI_EXIT_HOST_CALL: + return tec_exit_host_call(vcpu); + } + + kvm_pr_unimpl("Unsupported exit reason : 0x%llx\n", + run->tec_exit.exit_reason); + return 0; +} diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 8c8d7653ba..66a7fff9f3 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -45,6 +45,7 @@ #include <asm/tlb.h> #include <asm/alternative.h> #include <asm/xen/swiotlb-xen.h> +#include <asm/set_memory.h>

#include "internal.h"

@@ -610,6 +611,8 @@ void __init mem_init(void)

swiotlb_init(swiotlb, SWIOTLB_VERBOSE);

+ swiotlb_cvm_update_mem_attributes(); + /* this will put all unused low memory onto the freelists */ memblock_free_all();

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 4142a75a41..31f04f19b6 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -40,6 +40,7 @@ #include <asm/tlbflush.h> #include <asm/pgalloc.h> #include <asm/kfence.h> +#include <asm/set_memory.h>

#define NO_BLOCK_MAPPINGS BIT(0) #define NO_CONT_MAPPINGS BIT(1) @@ -589,7 +590,7 @@ static void __init map_mem(pgd_t *pgdp)

early_kfence_pool = arm64_kfence_alloc_pool();

- if (can_set_direct_map()) + if (can_set_direct_map() || is_virtcca_cvm_world()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;

/* @@ -1343,7 +1344,7 @@ int arch_add_memory(int nid, u64 start, u64 size,

VM_BUG_ON(!mhp_range_allowed(start, size, true));

- if (can_set_direct_map()) + if (can_set_direct_map() || is_virtcca_cvm_world()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;

__create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 0e270a1c51..06e81d1dbc 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -195,6 +195,9 @@ int set_direct_map_default_noflush(struct page *page) #ifdef CONFIG_DEBUG_PAGEALLOC void __kernel_map_pages(struct page *page, int numpages, int enable) { + if (is_virtcca_cvm_world()) + return; + if (!can_set_direct_map()) return;

diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index 54feb1eccc..e4020cafcc 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-objs-y += arm-smmu-v3.o +arm_smmu_v3-objs-$(CONFIG_HISI_VIRTCCA_HOST) += arm-s-smmu-v3.o arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c new file mode 100644 index 0000000000..0b9a16d8be --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#include <linux/crash_dump.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/iopoll.h> +#include <linux/pci.h> +#include <linux/hashtable.h> +#include <asm/kvm_tmi.h> + +#include "arm-smmu-v3.h" +#include "arm-s-smmu-v3.h" +#include "../../dma-iommu.h" + +struct cc_dev_config { + u32 sid; /* BDF number of the device */ + u32 vmid; + u32 root_bd; /* root bus and device number. Multiple sid can have the same root_bd. */ + bool secure; + struct hlist_node node; +}; + +static bool g_smmu_id_map_init = false; + +static DEFINE_HASHTABLE(g_cc_dev_htable, MAX_CC_DEV_NUM_ORDER); +static DECLARE_BITMAP(g_smmu_id_map, ARM_SMMU_MAX_IDS); + +/* Traverse pcie topology to find the root <bus,device> number + * return -1 if error + * return -1 if not pci device + */ +int get_root_bd(struct device *dev) +{ + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) + return -1; + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return -1; + while (pdev->bus->parent != NULL) + pdev = pdev->bus->self; + + return pci_dev_id(pdev) & 0xfff8; +} + +/* if dev is a bridge, get all it's children. + * if dev is a regular device, get itself. + */ +void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, int max_devs, int *ndev) +{ + struct pci_bus *bus = dev->subordinate; + + if (bus) { /* dev is a bridge */ + struct pci_dev *child; + + list_for_each_entry(child, &bus->devices, bus_list) { + get_child_devices_rec(child, devs, max_devs, ndev); + } + } else { /* dev is a regular device */ + uint16_t bdf = pci_dev_id(dev); + int i; + /* check if bdf is already in devs */ + for (i = 0; i < *ndev; i++) { + if (devs[i] == bdf) + return; + } + /* check overflow */ + if (*ndev >= max_devs) { + WARN_ON(1); + return; + } + devs[*ndev] = bdf; + *ndev = *ndev + 1; + } +} + +/* get all devices which share the same root_bd as dev + * return 0 on failure + */ +int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs) +{ + int ndev = 0; + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) + return ndev; + + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return ndev; + + while (pdev->bus->parent != NULL) + pdev = pdev->bus->self; + + get_child_devices_rec(pdev, devs, max_devs, &ndev); + return ndev; +} + +void set_g_cc_dev(u32 sid, u32 vmid, u32 root_bd, bool secure) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj->sid == sid) { + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + return; + } + } + + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) { + WARN_ON(1); + return; + } + + obj->sid = sid; + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + + hash_add(g_cc_dev_htable, &obj->node, sid); +} + +void free_g_cc_dev_htable(void) +{ + int i; + struct cc_dev_config *obj; + struct hlist_node *tmp; + + hash_for_each_safe(g_cc_dev_htable, i, tmp, obj, node) { + hash_del(&obj->node); + kfree(obj); + } +} + +/* Has the root bus device number switched to secure? */ +bool is_cc_root_bd(u32 root_bd) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (obj->root_bd == root_bd && obj->secure) + return true; + } + + return false; +} + +static bool is_cc_vmid(u32 vmid) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (vmid > 0 && obj->vmid == vmid) + return true; + } + + return false; +} + +bool is_cc_dev(u32 sid) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid) + return obj->secure; + } + + return false; +} +EXPORT_SYMBOL(is_cc_dev); + +void s_smmu_cmdq_need_forward(u64 cmd0, u64 cmd1, u64 *forward) +{ + u64 opcode = FIELD_GET(CMDQ_0_OP, cmd0); + + switch (opcode) { + case CMDQ_OP_TLBI_EL2_ALL: + case CMDQ_OP_TLBI_NSNH_ALL: + *forward = 1; + break; + case CMDQ_OP_PREFETCH_CFG: + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_STE: + case CMDQ_OP_CFGI_CD_ALL: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_CFGI_0_SID, cmd0)); + break; + + case CMDQ_OP_CFGI_ALL: + *forward = 1; + break; + case CMDQ_OP_TLBI_NH_VA: + case CMDQ_OP_TLBI_S2_IPA: + case CMDQ_OP_TLBI_NH_ASID: + case CMDQ_OP_TLBI_S12_VMALL: + *forward = (uint64_t)is_cc_vmid(FIELD_GET(CMDQ_TLBI_0_VMID, cmd0)); + break; + case CMDQ_OP_TLBI_EL2_VA: + case CMDQ_OP_TLBI_EL2_ASID: + *forward = 0; + break; + case CMDQ_OP_ATC_INV: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_ATC_0_SID, cmd0)); + break; + case CMDQ_OP_PRI_RESP: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_PRI_0_SID, cmd0)); + break; + case CMDQ_OP_RESUME: + *forward = (uint64_t)is_cc_dev(FIELD_GET(CMDQ_RESUME_0_SID, cmd0)); + break; + case CMDQ_OP_CMD_SYNC: + *forward = 0; + break; + default: + *forward = 0; + } +} + +void s_queue_write(struct arm_smmu_device *smmu, u64 *src, size_t n_dwords) +{ + u64 cmd0, cmd1; + u64 forward = 0; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + if (n_dwords == 2) { + cmd0 = cpu_to_le64(src[0]); + cmd1 = cpu_to_le64(src[1]); + s_smmu_cmdq_need_forward(cmd0, cmd1, &forward); + + /* need forward queue command to TMM */ + if (forward) { + if (tmi_smmu_queue_write(cmd0, cmd1, smmu->id)) + pr_err("tmi_smmu_queue_write err!\n"); + } + } + } +} + +void arm_s_smmu_cmdq_write_entries(struct arm_smmu_device *smmu, u64 *cmds, int n) +{ + int i; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + for (i = 0; i < n; ++i) { + u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS]; + + s_queue_write(smmu, cmd, CMDQ_ENT_DWORDS); + } + } +} + +int arm_s_smmu_init_one_queue(struct arm_smmu_device *smmu, + struct arm_smmu_queue *q, + size_t qsz, const char *name) +{ + if (smmu->id != ARM_SMMU_INVALID_ID) { + struct tmi_smmu_queue_params *params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + + if (!params_ptr) + return -ENOMEM; + + if (!strcmp(name, "cmdq")) { + params_ptr->ns_src = q->base_dma; + params_ptr->smmu_base_addr = smmu->ioaddr; + params_ptr->size = qsz; + params_ptr->smmu_id = smmu->id; + params_ptr->type = TMI_SMMU_CMD_QUEUE; + tmi_smmu_queue_create(__pa(params_ptr)); + } + + if (!strcmp(name, "evtq")) { + params_ptr->ns_src = q->base_dma; + params_ptr->smmu_base_addr = smmu->ioaddr; + params_ptr->size = qsz; + params_ptr->smmu_id = smmu->id; + params_ptr->type = TMI_SMMU_EVT_QUEUE; + tmi_smmu_queue_create(__pa(params_ptr)); + } + + kfree(params_ptr); + } + + return 0; +} + +int arm_smmu_write_s_reg_sync(struct arm_smmu_device *smmu, u32 val, u32 cmp_val, + unsigned int reg_off, unsigned int ack_off) +{ + u32 reg; + + if (tmi_smmu_write(smmu->ioaddr, reg_off, val, 32)) + return -ENXIO; + + return kvm_cvm_read_poll_timeout_atomic(tmi_smmu_read, reg, reg == cmp_val, + 1, ARM_SMMU_POLL_TIMEOUT_US, false, + smmu->ioaddr, ack_off, 32); +} + +int arm_smmu_update_s_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr) +{ + int ret; + u32 reg; + + ret = kvm_cvm_read_poll_timeout_atomic(tmi_smmu_read, reg, !(reg & S_GBPA_UPDATE), + 1, ARM_SMMU_POLL_TIMEOUT_US, false, + smmu->ioaddr, ARM_SMMU_S_GBPA, 32); + if (ret) + return ret; + + reg &= ~clr; + reg |= set; + + ret = tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_GBPA, reg | S_GBPA_UPDATE, 32); + if (ret) + return ret; + + ret = kvm_cvm_read_poll_timeout_atomic(tmi_smmu_read, reg, !(reg & S_GBPA_UPDATE), + 1, ARM_SMMU_POLL_TIMEOUT_US, false, + smmu->ioaddr, ARM_SMMU_S_GBPA, 32); + if (ret) + dev_err(smmu->dev, "S_GBPA not responding to update\n"); + return ret; +} + +irqreturn_t arm_smmu_s_evtq_thread(int irq, void *dev) +{ + struct arm_smmu_device *smmu = dev; + + if (smmu->id != ARM_SMMU_INVALID_ID) + tmi_handle_s_evtq(smmu->id); + + return IRQ_HANDLED; +} + +irqreturn_t arm_smmu_s_gerror_handler(int irq, void *dev) +{ + u32 gerror, gerrorn, active; + u64 ret; + struct arm_smmu_device *smmu = dev; + + ret = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_GERROR, 32); + if (ret >> 32) { + dev_err(smmu->dev, "Get ARM_SMMU_S_GERROR register failed\n"); + return IRQ_NONE; + } + gerror = (u32)ret; + + ret = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_GERRORN, 32); + if (ret >> 32) { + dev_err(smmu->dev, "Get ARM_SMMU_S_GERRORN register failed\n"); + return IRQ_NONE; + } + gerrorn = (u32)ret; + + active = gerror ^ gerrorn; + if (!(active & GERROR_ERR_MASK)) + return IRQ_NONE; /* No errors pending */ + + dev_warn(smmu->dev, + "unexpected secure global error reported, this could be serious, active %x\n", + active); + + if (active & GERROR_SFM_ERR) { + dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); + arm_s_smmu_device_disable(smmu); + } + + if (active & GERROR_MSI_GERROR_ABT_ERR) + dev_warn(smmu->dev, "GERROR MSI write aborted\n"); + + if (active & GERROR_MSI_PRIQ_ABT_ERR) + dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); + + if (active & GERROR_MSI_EVTQ_ABT_ERR) + dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); + + if (active & GERROR_MSI_CMDQ_ABT_ERR) + dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); + + if (active & GERROR_PRIQ_ABT_ERR) + dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); + + if (active & GERROR_EVTQ_ABT_ERR) + dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); + + if (active & GERROR_CMDQ_ERR) + dev_warn(smmu->dev, "CMDQ ERR\n"); + + if (tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_GERRORN, gerror, 32)) { + dev_err(smmu->dev, "SMMU write ARM_SMMU_S_GERRORN failed\n"); + return IRQ_NONE; + } + + return IRQ_HANDLED; +} + +int arm_smmu_disable_s_irq(struct arm_smmu_device *smmu) +{ + int ret; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + ret = arm_smmu_write_s_reg_sync(smmu, 0, 0, + ARM_SMMU_S_IRQ_CTRL, ARM_SMMU_S_IRQ_CTRLACK); + if (ret) { + dev_err(smmu->dev, "failed to disable secure irqs\n"); + return ret; + } + } + + return 0; +} + +int arm_smmu_enable_s_irq(struct arm_smmu_device *smmu, u32 irqen_flags) +{ + int ret; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + ret = arm_smmu_write_s_reg_sync(smmu, irqen_flags, + irqen_flags, ARM_SMMU_S_IRQ_CTRL, ARM_SMMU_S_IRQ_CTRLACK); + if (ret) { + dev_err(smmu->dev, "failed to enable irq for secure evtq\n"); + return ret; + } + } + + return 0; +} + +void arm_s_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) +{ + int irq, ret; + + irq = smmu->s_evtq_irq; + if (irq && smmu->id != ARM_SMMU_INVALID_ID) { + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + arm_smmu_s_evtq_thread, + IRQF_ONESHOT, + "arm-smmu-v3-s_evtq", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable s_evtq irq\n"); + } else { + dev_warn(smmu->dev, "no s_evtq irq - events will not be reported!\n"); + } + + irq = smmu->s_gerr_irq; + if (irq && smmu->id != ARM_SMMU_INVALID_ID) { + ret = devm_request_irq(smmu->dev, irq, arm_smmu_s_gerror_handler, + 0, "arm-smmu-v3-s_gerror", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable s_gerror irq\n"); + } else { + dev_warn(smmu->dev, "no s_gerr irq - errors will not be reported!\n"); + } +} + +void arm_smmu_write_s_msi_msg(struct arm_smmu_device *smmu, phys_addr_t *cfg, + struct msi_msg *msg, phys_addr_t doorbell) +{ + tmi_smmu_write((u64)smmu->ioaddr, cfg[0], doorbell, 64); + tmi_smmu_write((u64)smmu->ioaddr, cfg[1], msg->data, 32); + tmi_smmu_write((u64)smmu->ioaddr, cfg[2], ARM_SMMU_MEMATTR_DEVICE_nGnRE, 32); +} + +void platform_get_s_irq_byname_optional(struct platform_device *pdev, struct arm_smmu_device *smmu) +{ + int irq; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + irq = platform_get_irq_byname_optional(pdev, "s_eventq"); + if (irq > 0) + smmu->s_evtq_irq = irq; + + irq = platform_get_irq_byname_optional(pdev, "s_gerror"); + if (irq > 0) + smmu->s_gerr_irq = irq; + } +} + +int arm_smmu_enable_secure(struct iommu_domain *domain) +{ + int ret = 0; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + mutex_lock(&smmu_domain->init_mutex); + if (smmu_domain->smmu) + ret = -EPERM; + else + smmu_domain->secure = true; + mutex_unlock(&smmu_domain->init_mutex); + + return ret; +} + +u32 arm_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, + struct kvm *kvm) +{ + int ret = -1; + u64 cmd[CMDQ_ENT_DWORDS] = {0}; + unsigned long flags; + int i, j; + struct arm_smmu_master *master; + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + + spin_lock_irqsave(&arm_smmu_domain->devices_lock, flags); + list_for_each_entry(master, &arm_smmu_domain->devices, domain_head) { + if (master && master->num_streams >= 0) { + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + ret = tmi_dev_attach(sid, virtcca_cvm->rd, + arm_smmu_domain->smmu->id); + if (ret) { + kvm_err("dev protected failed!\n"); + ret = -ENXIO; + goto out; + } + cmd[0] |= FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CFGI_STE); + cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); + cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, true); + tmi_smmu_queue_write(cmd[0], cmd[1], arm_smmu_domain->smmu->id); + } + } + } + +out: + spin_unlock_irqrestore(&arm_smmu_domain->devices_lock, flags); + return ret; +} + +int arm_smmu_secure_dev_ste_create(struct arm_smmu_device *smmu, + struct arm_smmu_master *master, u32 sid) +{ + struct tmi_smmu_ste_params *params_ptr; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; + + params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!params_ptr) + return -ENOMEM; + + /* Sync Level 2 STE to TMM */ + params_ptr->ns_src = desc->l2ptr_dma + ((sid & ((1 << STRTAB_SPLIT) - 1)) * STE_ENTRY_SIZE); + params_ptr->sid = sid; + params_ptr->smmu_id = smmu->id; + + if (tmi_smmu_ste_create(__pa(params_ptr)) != 0) + dev_err(smmu->dev, "failed to create STE level 2"); + + kfree(params_ptr); + + return 0; +} + +int arm_smmu_secure_set_dev(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master *master, + struct device *dev) +{ + int i, j; + u64 ret = 0; + uint16_t root_bd = get_root_bd(dev); + + WARN_ON_ONCE(root_bd < 0); + if (!is_cc_root_bd(root_bd)) { + struct tmi_dev_delegate_params *params = kzalloc( + sizeof(struct tmi_dev_delegate_params), GFP_KERNEL); + + params->root_bd = root_bd; + params->num_dev = get_sibling_devices(dev, params->devs, MAX_DEV_PER_PORT); + WARN_ON_ONCE(params->num_dev == 0); + pr_info("Delegate %d devices as %02x:%02x to secure\n", + params->num_dev, root_bd >> 8, (root_bd & 0xff) >> 3); + ret = tmi_dev_delegate(__pa(params)); + if (!ret) { + for (i = 0; i < params->num_dev; i++) + set_g_cc_dev(params->devs[i], 0, root_bd, true); + } + kfree(params); + } + + if (ret) + return ret; + + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + WARN_ON_ONCE(!is_cc_dev(sid)); + set_g_cc_dev(sid, smmu_domain->s2_cfg.vmid, root_bd, true); + } + + return ret; +} + +int arm_smmu_id_alloc(void) +{ + int idx; + + do { + idx = find_first_zero_bit(g_smmu_id_map, ARM_SMMU_MAX_IDS); + if (idx == ARM_SMMU_MAX_IDS) + return -ENOSPC; + } while (test_and_set_bit(idx, g_smmu_id_map)); + + return idx; +} + +void arm_smmu_id_free(int idx) +{ + if (idx != ARM_SMMU_INVALID_ID) + clear_bit(idx, g_smmu_id_map); +} + +void arm_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr) +{ + if (!g_smmu_id_map_init) { + set_bit(0, g_smmu_id_map); + g_smmu_id_map_init = true; + } + smmu->ioaddr = ioaddr; + + if (tmi_smmu_pcie_core_check(ioaddr)) + smmu->id = arm_smmu_id_alloc(); + else + smmu->id = ARM_SMMU_INVALID_ID; + + hash_init(g_cc_dev_htable); +} + +int arm_s_smmu_device_disable(struct arm_smmu_device *smmu) +{ + int ret = 0; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + ret = arm_smmu_write_s_reg_sync(smmu, 0, 0, ARM_SMMU_S_CR0, ARM_SMMU_S_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to clear s_cr0\n"); + return ret; + } + } + + return ret; +} + +int arm_s_smmu_device_reset(struct arm_smmu_device *smmu) +{ + int ret; + u64 rv; + u32 reg, enables; + struct tmi_smmu_cfg_params *params_ptr; + + if (smmu->id == ARM_SMMU_INVALID_ID) + return 0; + + rv = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_CR0, 32); + if (rv >> 32) + return -ENXIO; + + ret = (int)rv; + if (ret & S_CR0_SMMUEN) { + dev_warn(smmu->dev, "Secure SMMU currently enabled! Resetting...\n"); + arm_smmu_update_s_gbpa(smmu, S_GBPA_ABORT, 0); + } + + ret = arm_s_smmu_device_disable(smmu); + if (ret) + return ret; + + /* CR1 (table and queue memory attributes) */ + reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) | + FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) | + FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) | + FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) | + FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) | + FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB); + + ret = tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_CR1, reg, 32); + if (ret) + return ret; + + /* CR2 (random crap) */ + reg = CR2_RECINVSID; + + if (smmu->features & ARM_SMMU_FEAT_E2H) + reg |= CR2_E2H; + + ret = tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_CR2, reg, 32); + if (ret) + return ret; + + params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!params_ptr) + return -ENOMEM; + + params_ptr->is_cmd_queue = 1; + params_ptr->smmu_id = smmu->id; + params_ptr->ioaddr = smmu->ioaddr; + params_ptr->strtab_base_RA_bit = + (smmu->strtab_cfg.strtab_base >> S_STRTAB_BASE_RA_SHIFT) & 0x1; + params_ptr->q_base_RA_WA_bit = + (smmu->cmdq.q.q_base >> S_CMDQ_BASE_RA_SHIFT) & 0x1; + if (tmi_smmu_device_reset(__pa(params_ptr)) != 0) { + dev_err(smmu->dev, "failed to set s cmd queue regs\n"); + return -ENXIO; + } + + enables = CR0_CMDQEN; + ret = arm_smmu_write_s_reg_sync(smmu, enables, enables, ARM_SMMU_S_CR0, + ARM_SMMU_S_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable secure command queue\n"); + return ret; + } + + enables |= CR0_EVTQEN; + + /* Secure event queue */ + memset(params_ptr, 0, sizeof(struct tmi_smmu_ste_params)); + params_ptr->is_cmd_queue = 0; + params_ptr->ioaddr = smmu->ioaddr; + params_ptr->smmu_id = smmu->id; + params_ptr->q_base_RA_WA_bit = + (smmu->evtq.q.q_base >> S_EVTQ_BASE_WA_SHIFT) & 0x1; + params_ptr->strtab_base_RA_bit = + (smmu->strtab_cfg.strtab_base >> S_STRTAB_BASE_RA_SHIFT) & 0x1; + if (tmi_smmu_device_reset(__pa(params_ptr)) != 0) { + dev_err(smmu->dev, "failed to set s event queue regs"); + return -ENXIO; + } + + /* Enable secure eventq */ + ret = arm_smmu_write_s_reg_sync(smmu, enables, enables, ARM_SMMU_S_CR0, + ARM_SMMU_S_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to disable secure event queue\n"); + return ret; + } + + ret = arm_smmu_write_s_reg_sync(smmu, SMMU_S_INIT_INV_ALL, 0, + ARM_SMMU_S_INIT, ARM_SMMU_S_INIT); + if (ret) { + dev_err(smmu->dev, "failed to write S_INIT\n"); + return ret; + } + + return 0; +} + +int arm_s_smmu_device_enable(struct arm_smmu_device *smmu, + u32 enables, bool bypass, bool disable_bypass) +{ + int ret = 0; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + /* Enable the SMMU interface, or ensure bypass */ + if (!bypass || disable_bypass) { + enables |= CR0_SMMUEN; + } else { + ret = arm_smmu_update_s_gbpa(smmu, 0, S_GBPA_ABORT); + if (ret) + return ret; + } + /* Mask BIT1 and BIT4 which are RES0 in SMMU_S_CRO */ + ret = arm_smmu_write_s_reg_sync(smmu, enables & ~SMMU_S_CR0_RESERVED, + enables & ~SMMU_S_CR0_RESERVED, ARM_SMMU_S_CR0, ARM_SMMU_S_CR0ACK); + } + return ret; +} + +int arm_smmu_s_idr1_support_secure(struct arm_smmu_device *smmu) +{ + u32 ret; + u64 rv; + + if (smmu->id != ARM_SMMU_INVALID_ID) { + rv = tmi_smmu_read(smmu->ioaddr, ARM_SMMU_S_IDR1, 32); + if (rv >> 32) { + dev_err(smmu->dev, "Get ARM_SMMU_S_IDR1 register failed!\n"); + return -ENXIO; + } + ret = (u32)rv; + if (!(ret & S_IDR1_SECURE_IMPL)) { + dev_err(smmu->dev, "SMMUv3 does not implement secure state!\n"); + return -ENXIO; + } + + if (!(ret & S_IDR1_SEL2)) { + dev_err(smmu->dev, "SMMUv3: Secure stage2 translation not supported!\n"); + return -ENXIO; + } + } + + return 0; +} diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h new file mode 100644 index 0000000000..fa2061d94a --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ +#ifndef _ARM_S_SMMU_V3_H +#define _ARM_S_SMMU_V3_H + +#include <linux/platform_device.h> + +#define MAX_CC_DEV_NUM_ORDER 8 +#define STE_ENTRY_SIZE 0x40 + +#define ARM_SMMU_MAX_IDS (1 << 5) +#define ARM_SMMU_INVALID_ID ARM_SMMU_MAX_IDS + +/* Secure MMIO registers */ +#define ARM_SMMU_S_IDR0 0x8000 +#define S_IDR0_STALL_MODEL GENMASK(25, 24) +#define S_IDR0_ECMDQ (1 << 31) +#define S_IDR0_MSI (1 << 13) + +#define ARM_SMMU_S_IDR1 0x8004 +#define S_IDR1_SECURE_IMPL (1 << 31) +#define S_IDR1_SEL2 (1 << 29) +#define S_IDR1_SIDSIZE GENMASK(5, 0) + +#define ARM_SMMU_S_IDR3 0x800c +#define S_IDR3_SAMS (1 << 6) + +#define ARM_SMMU_S_CR0 0x8020 +#define S_CR0_SIF (1 << 9) +#define S_CR0_NSSTALLD (1 << 5) +#define S_CR0_CMDQEN (1 << 3) +#define S_CR0_EVTQEN (1 << 2) +#define S_CR0_SMMUEN (1 << 0) + +#define ARM_SMMU_S_CR0ACK 0x8024 + +#define ARM_SMMU_S_CR1 0x8028 +#define S_CR1_TABLE_SH GENMASK(11, 10) +#define S_CR1_TABLE_OC GENMASK(9, 8) +#define S_CR1_TABLE_IC GENMASK(7, 6) +#define S_CR1_QUEUE_SH GENMASK(5, 4) +#define S_CR1_QUEUE_OC GENMASK(3, 2) +#define S_CR1_QUEUE_IC GENMASK(1, 0) + +/* S_CR1 cacheability fields don't quite follow the usual TCR-style encoding */ +#define S_CR1_CACHE_NC 0 +#define S_CR1_CACHE_WB 1 +#define S_CR1_CACHE_WT 2 + +#define ARM_SMMU_S_CR2 0x802c +#define S_CR2_PTM (1 << 2) +#define S_CR2_RECINVSID (1 << 1) +#define S_CR2_E2H (1 << 0) + +#define ARM_SMMU_S_INIT U(0x803c) +/* SMMU_S_INIT register fields */ +#define SMMU_S_INIT_INV_ALL (1UL << 0) + +#define ARM_SMMU_S_GBPA 0x8044 +#define S_GBPA_UPDATE (1 << 31) +#define S_GBPA_ABORT (1 << 20) + +#define ARM_SMMU_S_IRQ_CTRL 0x8050 +#define S_IRQ_CTRL_EVTQ_IRQEN (1 << 2) +#define S_IRQ_CTRL_GERROR_IRQEN (1 << 0) + +#define ARM_SMMU_S_IRQ_CTRLACK 0x8054 + +#define ARM_SMMU_S_GERROR 0x8060 +#define S_GERROR_SFM_ERR (1 << 8) +#define S_GERROR_MSI_GERROR_ABT_ERR (1 << 7) +#define S_GERROR_MSI_EVTQ_ABT_ERR (1 << 5) +#define S_GERROR_MSI_CMDQ_ABT_ERR (1 << 4) +#define S_GERROR_EVTQ_ABT_ERR (1 << 2) +#define S_GERROR_CMDQ_ERR (1 << 0) + +#define ARM_SMMU_S_GERRORN 0x8064 + +#define ARM_SMMU_S_GERROR_IRQ_CFG0 0x8068 +#define ARM_SMMU_S_GERROR_IRQ_CFG1 0x8070 +#define ARM_SMMU_S_GERROR_IRQ_CFG2 0x8074 + +#define ARM_SMMU_S_STRTAB_BASE 0x8080 +#define S_STRTAB_BASE_RA_SHIFT 62 +#define S_STRTAB_BASE_RA (1UL << S_STRTAB_BASE_RA_SHIFT) +#define S_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) + +#define ARM_SMMU_S_STRTAB_BASE_CFG 0x8088 +#define S_STRTAB_BASE_CFG_FMT GENMASK(17, 16) +#define S_STRTAB_BASE_CFG_SPLIT GENMASK(10, 6) +#define S_STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0) + +#define ARM_SMMU_S_CMDQ_BASE 0x8090 +#define ARM_SMMU_S_CMDQ_PROD 0x8098 +#define ARM_SMMU_S_CMDQ_CONS 0x809c +#define S_CMDQ_BASE_ADDR_MASK GENMASK_ULL(51, 5) +#define S_CMDQ_BASE_RA_SHIFT 62 + +#define ARM_SMMU_S_EVTQ_BASE 0x80a0 +#define ARM_SMMU_S_EVTQ_PROD 0x80a8 +#define ARM_SMMU_S_EVTQ_CONS 0x80ac +#define ARM_SMMU_S_EVTQ_IRQ_CFG0 0x80b0 +#define ARM_SMMU_S_EVTQ_IRQ_CFG1 0x80b8 +#define ARM_SMMU_S_EVTQ_IRQ_CFG2 0x80bc +#define S_EVTQ_BASE_ADDR_MASK GENMASK_ULL(51, 5) +#define S_EVTQ_BASE_WA_SHIFT 62 + +/* + * BIT1 is PRIQEN, BIT4 is ATSCHK in SMMU_CRO + * BIT1 and BIT4 are RES0 in SMMU_S_CRO + */ +#define SMMU_S_CR0_RESERVED 0xFFFFFC12 + +int get_root_bd(struct device *dev); +void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, int max_devs, int *ndev); +int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs); +void set_g_cc_dev(u32 sid, u32 vmid, u32 root_bd, bool secure); +void free_g_cc_dev_htable(void); +/* Has the root bus device number switched to secure? */ +bool is_cc_root_bd(u32 root_bd); +bool is_cc_dev(u32 sid); + +void s_smmu_cmdq_need_forward(u64 cmd0, u64 cmd1, u64 *forward); +void s_queue_write(struct arm_smmu_device *smmu, u64 *src, size_t n_dwords); +void arm_s_smmu_cmdq_write_entries(struct arm_smmu_device *smmu, u64 *cmds, int n); +int arm_s_smmu_init_one_queue(struct arm_smmu_device *smmu, + struct arm_smmu_queue *q, + size_t qsz, const char *name); +int arm_smmu_write_s_reg_sync(struct arm_smmu_device *smmu, u32 val, u32 cmp_val, + unsigned int reg_off, unsigned int ack_off); +int arm_smmu_update_s_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr); + +irqreturn_t arm_smmu_s_evtq_thread(int irq, void *dev); +irqreturn_t arm_smmu_s_gerror_handler(int irq, void *dev); +int arm_smmu_disable_s_irq(struct arm_smmu_device *smmu); +int arm_smmu_enable_s_irq(struct arm_smmu_device *smmu, u32 irqen_flags); +void arm_s_smmu_setup_unique_irqs(struct arm_smmu_device *smmu); +void arm_smmu_write_s_msi_msg(struct arm_smmu_device *smmu, phys_addr_t *cfg, + struct msi_msg *msg, phys_addr_t doorbell); +void platform_get_s_irq_byname_optional(struct platform_device *pdev, struct arm_smmu_device *smmu); +int arm_smmu_enable_secure(struct iommu_domain *domain); +u32 arm_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, + struct kvm *kvm); +int arm_smmu_secure_dev_ste_create(struct arm_smmu_device *smmu, + struct arm_smmu_master *master, u32 sid); +int arm_smmu_secure_set_dev(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master *master, + struct device *dev); + +int arm_smmu_id_alloc(void); +void arm_smmu_id_free(int idx); +void arm_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr); +int arm_s_smmu_device_disable(struct arm_smmu_device *smmu); +int arm_s_smmu_device_reset(struct arm_smmu_device *smmu); +int arm_s_smmu_device_enable(struct arm_smmu_device *smmu, + u32 enables, bool bypass, bool disable_bypass); +int arm_smmu_s_idr1_support_secure(struct arm_smmu_device *smmu); + +#endif /* _ARM_S_SMMU_V3_H */ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 16217647fd..157538d22f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -30,6 +30,11 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h"

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#include "arm-s-smmu-v3.h" +#endif + static bool disable_bypass = true; module_param(disable_bypass, bool, 0444); MODULE_PARM_DESC(disable_bypass, @@ -102,6 +107,10 @@ static int arm_smmu_bypass_dev_domain_type(struct device *dev) enum arm_smmu_msi_index { EVTQ_MSI_INDEX, GERROR_MSI_INDEX, +#ifdef CONFIG_HISI_VIRTCCA_HOST + S_EVTQ_MSI_INDEX, + S_GERROR_MSI_INDEX, +#endif PRIQ_MSI_INDEX, ARM_SMMU_MAX_MSIS, }; @@ -117,6 +126,18 @@ static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { ARM_SMMU_GERROR_IRQ_CFG1, ARM_SMMU_GERROR_IRQ_CFG2, }, +#ifdef CONFIG_HISI_VIRTCCA_HOST + [S_EVTQ_MSI_INDEX] = { + ARM_SMMU_S_EVTQ_IRQ_CFG0, + ARM_SMMU_S_EVTQ_IRQ_CFG1, + ARM_SMMU_S_EVTQ_IRQ_CFG2, + }, + [S_GERROR_MSI_INDEX] = { + ARM_SMMU_S_GERROR_IRQ_CFG0, + ARM_SMMU_S_GERROR_IRQ_CFG1, + ARM_SMMU_S_GERROR_IRQ_CFG2, + }, +#endif [PRIQ_MSI_INDEX] = { ARM_SMMU_PRIQ_IRQ_CFG0, ARM_SMMU_PRIQ_IRQ_CFG1, @@ -995,11 +1016,16 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, * Dependency ordering from the cmpxchg() loop above. */ arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); +#ifdef CONFIG_HISI_VIRTCCA_HOST + arm_s_smmu_cmdq_write_entries(smmu, cmds, n); +#endif if (sync) { prod = queue_inc_prod_n(&llq, n); arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod); queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); - + #ifdef CONFIG_HISI_VIRTCCA_HOST + s_queue_write(smmu, cmd_sync, CMDQ_ENT_DWORDS); + #endif /* * In order to determine completion of our CMD_SYNC, we must * ensure that the queue can't wrap twice without us noticing. @@ -2368,6 +2394,12 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu_domain->secure) { + smmu_domain->stage = ARM_SMMU_DOMAIN_S2; + } +#endif + switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; @@ -2588,6 +2620,52 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master) arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +static void arm_smmu_secure_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) +{ + struct arm_smmu_cmdq_ent prefetch_cmd = { + .opcode = CMDQ_OP_PREFETCH_CFG, + .prefetch = { + .sid = sid, + }, + }; + arm_smmu_sync_ste_for_sid(smmu, sid); + + /* It's likely that we'll want to use the new STE soon */ + if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) + arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); +} + +static int arm_smmu_secure_dev_operator(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu, struct arm_smmu_master *master, struct device *dev) +{ + int i, j; + int ret; + + ret = arm_smmu_secure_set_dev(smmu_domain, master, dev); + if (ret) + return ret; + + for (i = 0; i < master->num_streams; ++i) { + u32 sid = master->streams[i].id; + /* Bridged PCI devices may end up with duplicated IDs */ + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + if (arm_smmu_secure_dev_ste_create(smmu, master, sid)) + return -ENOMEM; + + arm_smmu_secure_sync_ste_for_sid(smmu, sid); + } + + dev_info(smmu->dev, "attach confidential dev: %s", dev_name(dev)); + + return ret; +} +#endif + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; @@ -2669,6 +2747,11 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)

arm_smmu_install_ste_for_dev(master);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu_domain->secure) + ret = arm_smmu_secure_dev_operator(smmu_domain, smmu, master, dev); +#endif + arm_smmu_enable_ats(master); return 0;

@@ -3300,6 +3383,9 @@ static struct iommu_ops arm_smmu_ops = { .def_domain_type = arm_smmu_def_domain_type, .pgsize_bitmap = -1UL, /* Restricted during device attach */ .owner = THIS_MODULE, + #ifdef CONFIG_HISI_VIRTCCA_HOST + .iommu_enable_secure = arm_smmu_enable_secure, + #endif .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = arm_smmu_attach_dev, .map_pages = arm_smmu_map_pages, @@ -3362,6 +3448,13 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);

q->llq.prod = q->llq.cons = 0; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (arm_s_smmu_init_one_queue(smmu, q, qsz, name)) { + return -ENOMEM; + }; +#endif + return 0; }

@@ -3665,6 +3758,15 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) desc->msg.data = msg->data; #endif

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if ((desc->msi_index == S_EVTQ_MSI_INDEX || + desc->msi_index == S_GERROR_MSI_INDEX) && + smmu->id != ARM_SMMU_INVALID_ID) { + arm_smmu_write_s_msi_msg(smmu, cfg, msg, doorbell); + return; + } +#endif + writeq_relaxed(doorbell, smmu->base + cfg[0]); writel_relaxed(msg->data, smmu->base + cfg[1]); writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); @@ -3679,6 +3781,15 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu->id != ARM_SMMU_INVALID_ID) { + if (tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_GERROR_IRQ_CFG0, 0, 64)) + return; + if (tmi_smmu_write(smmu->ioaddr, ARM_SMMU_S_EVTQ_IRQ_CFG0, 0, 64)) + return; + } +#endif + if (smmu->features & ARM_SMMU_FEAT_PRI) writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); else @@ -3703,6 +3814,13 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX); smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (smmu->id != ARM_SMMU_INVALID_ID) { + smmu->s_evtq_irq = msi_get_virq(dev, S_EVTQ_MSI_INDEX); + smmu->s_gerr_irq = msi_get_virq(dev, S_GERROR_MSI_INDEX); + } +#endif + /* Add callback to free MSIs on teardown */ devm_add_action(dev, arm_smmu_free_msis, dev); } @@ -3791,6 +3909,11 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu, bool resume dev_warn(smmu->dev, "no priq irq - PRI will be broken\n"); } } + +#ifdef CONFIG_HISI_VIRTCCA_HOST + arm_s_smmu_setup_unique_irqs(smmu); +#endif + }

static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) @@ -3806,6 +3929,12 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) return ret; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_smmu_disable_s_irq(smmu); + if (ret) + return ret; +#endif + irq = smmu->combined_irq; if (irq) { /* @@ -3831,6 +3960,12 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) if (ret) dev_warn(smmu->dev, "failed to enable irqs\n");

+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_smmu_enable_s_irq(smmu, IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN); + if (ret) + return ret; +#endif + return 0; }

@@ -3842,6 +3977,10 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu) if (ret) dev_err(smmu->dev, "failed to clear cr0\n");

+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_s_smmu_device_disable(smmu); +#endif + return ret; }

@@ -3935,6 +4074,12 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool resume) FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB); writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_s_smmu_device_reset(smmu); + if (ret) + return ret; +#endif + /* CR2 (random crap) */ reg = CR2_PTM | CR2_RECINVSID;

@@ -4027,6 +4172,14 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool resume) if (is_kdump_kernel()) enables &= ~(CR0_EVTQEN | CR0_PRIQEN);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_s_smmu_device_enable(smmu, enables, smmu->bypass, disable_bypass); + if (ret) { + dev_err(smmu->dev, "failed to enable secure SMMU interface\n"); + return ret; + } +#endif + /* Enable the SMMU interface, or ensure bypass */ if (!smmu->bypass || disable_bypass) { enables |= CR0_SMMUEN; @@ -4358,6 +4511,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) smmu->features |= ARM_SMMU_FEAT_ECMDQ; #endif

+#ifndef CONFIG_HISI_VIRTCCA_HOST + if (reg & IDR1_ECMDQ) + smmu->features |= ARM_SMMU_FEAT_ECMDQ; +#endif + /* Queue sizes, capped to ensure natural alignment */ smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, reg)); @@ -4390,6 +4548,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (smmu->sid_bits <= STRTAB_SPLIT) smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (arm_smmu_s_idr1_support_secure(smmu)) + return -ENXIO; +#endif + /* IDR3 */ reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3); switch (FIELD_GET(IDR3_BBML, reg)) { @@ -4748,6 +4911,10 @@ static int arm_smmu_device_probe(struct platform_device *pdev) } ioaddr = res->start;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + arm_smmu_map_init(smmu, ioaddr); +#endif + /* * Don't map the IMPLEMENTATION DEFINED regions, since they may contain * the PMCG registers which are reserved by the PMU driver. @@ -4782,6 +4949,11 @@ static int arm_smmu_device_probe(struct platform_device *pdev) irq = platform_get_irq_byname_optional(pdev, "gerror"); if (irq > 0) smmu->gerr_irq = irq; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + platform_get_s_irq_byname_optional(pdev, smmu); +#endif + } /* Probe the h/w */ ret = arm_smmu_device_hw_probe(smmu); @@ -4829,6 +5001,10 @@ static void arm_smmu_device_remove(struct platform_device *pdev) arm_smmu_device_disable(smmu); iopf_queue_free(smmu->evtq.iopf); ida_destroy(&smmu->vmid_map); +#ifdef CONFIG_HISI_VIRTCCA_HOST + free_g_cc_dev_htable(); + arm_smmu_id_free(smmu->id); +#endif }

static void arm_smmu_device_shutdown(struct platform_device *pdev) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 859d217298..43121c17a2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -13,6 +13,9 @@ #include <linux/kernel.h> #include <linux/mmzone.h> #include <linux/sizes.h> +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <linux/kvm_host.h> +#endif

/* MMIO registers */ #define ARM_SMMU_IDR0 0x0 @@ -750,6 +753,13 @@ struct arm_smmu_device { struct mutex streams_mutex;

bool bypass; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + int s_evtq_irq; + int s_gerr_irq; + resource_size_t ioaddr; + uint64_t id; +#endif };

struct arm_smmu_stream { @@ -802,6 +812,12 @@ struct arm_smmu_domain { spinlock_t devices_lock;

struct list_head mmu_notifiers; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + bool secure; + struct list_head node; + struct kvm *kvm; +#endif };

static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 79aa11d88c..0f46148871 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -22,6 +22,16 @@

#include "io-pgtable-arm.h"

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#include <asm/kvm_emulate.h> +#include <asm/kvm_tmm.h> +#include <linux/kvm_host.h> +#include <linux/kvm.h> +#include <asm/kvm.h> +#include "../virt/kvm/vfio.h" +#endif + #define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 @@ -493,6 +503,41 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, return pte; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +static int arm_lpae_cvm_map_unmap_pages(struct arm_lpae_io_pgtable *data, unsigned long iova, + phys_addr_t paddr, size_t size, uint32_t is_map) +{ + struct arm_smmu_domain *smmu_domain = (struct arm_smmu_domain *)data->iop.cookie; + int ret = 0; + struct kvm *kvm; + u64 loader_start; + u64 ram_size; + static bool smmu_mapped = false; + + kvm = smmu_domain->kvm; + if (kvm && kvm_is_virtcca_cvm(kvm) && virtcca_cvm_state(kvm) != CVM_STATE_DYING) { + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + + loader_start = virtcca_cvm->loader_start; + ram_size = virtcca_cvm->ram_size; + if (!smmu_mapped && iova >= loader_start && + iova < loader_start + ram_size && + is_map && !virtcca_cvm->is_mapped) { + ret = kvm_cvm_map_range(kvm); + if (!ret) + smmu_mapped = true; + WARN_ON(ret); + } else if (iova < loader_start) { + if (iova == CVM_MSI_ORIG_IOVA) + iova += CVM_MSI_IOVA_OFFSET; + ret = kvm_cvm_map_unmap_ipa_range(kvm, iova, paddr, size, is_map); + WARN_ON(ret); + } + } + return ret; +} +#endif + static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int iommu_prot, gfp_t gfp, size_t *mapped) @@ -517,6 +562,11 @@ static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova, return 0;

prot = arm_lpae_prot_to_pte(data, iommu_prot); + +#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = arm_lpae_cvm_map_unmap_pages(data, iova, paddr, pgsize * pgcount, true); +#endif + ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl, ptep, gfp, mapped); /* @@ -703,6 +753,16 @@ static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iov if (WARN_ON(iaext)) return 0;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + int ret; + + ret = arm_lpae_cvm_map_unmap_pages(data, iova, 0, pgsize * pgcount, false); + WARN_ON(ret); + if (ret) + pr_err("%s %d failed to unmap pages, iova %lx, size %lx\n", + __func__, __LINE__, iova, pgsize); +#endif + return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount, data->start_level, ptep); } diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 28f63ad432..07a084c9d9 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -592,6 +592,17 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list return ret; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +struct iommu_domain *iommu_group_get_domain(struct iommu_group *iommu_group) +{ + if (iommu_group) + return iommu_group->domain; + + return NULL; +} +EXPORT_SYMBOL_GPL(iommu_group_get_domain); +#endif + int iommu_probe_device(struct device *dev) { const struct iommu_ops *ops; diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 0fa706a478..05700ae898 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -29,6 +29,10 @@ #include <linux/percpu.h> #include <linux/slab.h> #include <linux/syscore_ops.h> +#ifdef CONFIG_HISI_VIRTCCA_GUEST +#include <linux/swiotlb.h> +#include <asm/virtcca_cvm_guest.h> +#endif

#include <linux/irqchip.h> #include <linux/irqchip/arm-gic-v3.h> @@ -399,6 +403,93 @@ static int build_devid_pools(void) } #endif

+#ifdef CONFIG_HISI_VIRTCCA_GUEST + +static struct device cvm_alloc_device; +static LIST_HEAD(cvm_its_nodes); +static raw_spinlock_t cvm_its_lock; + +struct its_device_order { + struct its_device *dev; + struct list_head entry; + int itt_order; +}; + +static inline struct page *its_alloc_shared_pages_node(int node, gfp_t gfp, + unsigned int order) +{ + return swiotlb_alloc(&cvm_alloc_device, (1 << order) * PAGE_SIZE); +} + +static inline struct page *its_alloc_shared_pages(gfp_t gfp, unsigned int order) +{ + return its_alloc_shared_pages_node(NUMA_NO_NODE, gfp, order); +} + +static void its_free_shared_pages(void *addr, int order) +{ + if (order < 0) + return; + + swiotlb_free(&cvm_alloc_device, (struct page *)addr, (1 << order) * PAGE_SIZE); +} + +static int add_its_device_order(struct its_device *dev, int itt_order) +{ + struct its_device_order *new; + unsigned long flags; + + new = kmalloc(sizeof(struct its_device_order), GFP_KERNEL); + if (!new) + return -ENOMEM; + new->dev = dev; + new->itt_order = itt_order; + raw_spin_lock_irqsave(&cvm_its_lock, flags); + list_add_tail(&new->entry, &cvm_its_nodes); + raw_spin_unlock_irqrestore(&cvm_its_lock, flags); + return 0; +} + +/* get its device order and then free its device order */ +static int get_its_device_order(struct its_device *dev) +{ + struct its_device_order *pos, *tmp; + unsigned long flags; + int itt_order = -1; + + raw_spin_lock_irqsave(&cvm_its_lock, flags); + list_for_each_entry_safe(pos, tmp, &cvm_its_nodes, entry) { + if (pos->dev == dev) { + itt_order = pos->itt_order; + list_del(&pos->entry); + kfree(pos); + goto found; + } + } +found: + raw_spin_unlock_irqrestore(&cvm_its_lock, flags); + return itt_order; +} + +static void *its_alloc_shared_page_address(struct its_device *dev, + struct its_node *its, int sz) +{ + struct page *page; + int itt_order; + + itt_order = get_order(sz); + if (add_its_device_order(dev, itt_order)) + return NULL; + + page = its_alloc_shared_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, + itt_order); + if (!page) + return NULL; + return (void *)page_address(page); +} + +#endif + /* * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we * always have vSGIs mapped. @@ -2441,7 +2532,13 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags) { struct page *prop_page;

- prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + prop_page = its_alloc_shared_pages(gfp_flags, + get_order(LPI_PROPBASE_SZ)); + else +#endif + prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); if (!prop_page) return NULL;

@@ -2452,8 +2549,14 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags)

static void its_free_prop_table(struct page *prop_page) { - free_pages((unsigned long)page_address(prop_page), - get_order(LPI_PROPBASE_SZ)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(page_address(prop_page), + get_order(LPI_PROPBASE_SZ)); + else +#endif + free_pages((unsigned long)page_address(prop_page), + get_order(LPI_PROPBASE_SZ)); }

static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size) @@ -2575,7 +2678,13 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, order = get_order(GITS_BASER_PAGES_MAX * psz); }

- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + page = its_alloc_shared_pages_node(its->numa_node, + GFP_KERNEL | __GFP_ZERO, order); + else +#endif + page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); if (!page) return -ENOMEM;

@@ -2588,7 +2697,12 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, /* 52bit PA is supported only when PageSize=64K */ if (psz != SZ_64K) { pr_err("ITS: no 52bit PA support when psz=%d\n", psz); - free_pages((unsigned long)base, order); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(base, order); + else +#endif + free_pages((unsigned long)base, order); return -ENXIO; }

@@ -2644,7 +2758,12 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", &its->phys_base, its_base_type_string[type], val, tmp); - free_pages((unsigned long)base, order); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(base, order); + else +#endif + free_pages((unsigned long)base, order); return -ENXIO; }

@@ -2783,8 +2902,14 @@ static void its_free_tables(struct its_node *its)

for (i = 0; i < GITS_BASER_NR_REGS; i++) { if (its->tables[i].base) { - free_pages((unsigned long)its->tables[i].base, - its->tables[i].order); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(its->tables[i].base, + its->tables[i].order); + else +#endif + free_pages((unsigned long)its->tables[i].base, + its->tables[i].order); its->tables[i].base = NULL; } } @@ -3054,7 +3179,13 @@ static bool allocate_vpe_l2_table(int cpu, u32 id)

/* Allocate memory for 2nd level table */ if (!table[idx]) { - page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + page = its_alloc_shared_pages(GFP_KERNEL | __GFP_ZERO, + get_order(psz)); + else +#endif + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); if (!page) return false;

@@ -3173,7 +3304,13 @@ static int allocate_vpe_l1_table(void)

pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", np, npg, psz, epp, esz); - page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + page = its_alloc_shared_pages(GFP_ATOMIC | __GFP_ZERO, + get_order(np * PAGE_SIZE)); + else +#endif + page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); if (!page) return -ENOMEM;

@@ -3219,8 +3356,14 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags) { struct page *pend_page;

- pend_page = alloc_pages(gfp_flags | __GFP_ZERO, - get_order(LPI_PENDBASE_SZ)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + pend_page = its_alloc_shared_pages(gfp_flags | __GFP_ZERO, + get_order(LPI_PENDBASE_SZ)); + else +#endif + pend_page = alloc_pages(gfp_flags | __GFP_ZERO, + get_order(LPI_PENDBASE_SZ)); if (!pend_page) return NULL;

@@ -3232,7 +3375,13 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags)

static void its_free_pending_table(struct page *pt) { - free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(page_address(pt), + get_order(LPI_PENDBASE_SZ)); + else +#endif + free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); }

/* @@ -3567,8 +3716,15 @@ static bool its_alloc_table_entry(struct its_node *its,

/* Allocate memory for 2nd level table */ if (!table[idx]) { - page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, - get_order(baser->psz)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + page = its_alloc_shared_pages_node(its->numa_node, + GFP_KERNEL | __GFP_ZERO, + get_order(baser->psz)); + else +#endif + page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, + get_order(baser->psz)); if (!page) return false;

@@ -3671,7 +3827,12 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, nr_ites = max(2, nvecs); sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; - itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + itt = its_alloc_shared_page_address(dev, its, sz); + else +#endif + itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); if (alloc_lpis) { lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis); if (lpi_map) @@ -3685,7 +3846,12 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id,

if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { kfree(dev); - kfree(itt); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(itt, get_order(sz)); + else +#endif + kfree(itt); bitmap_free(lpi_map); kfree(col_map); return NULL; @@ -3722,7 +3888,12 @@ static void its_free_device(struct its_device *its_dev) list_del(&its_dev->entry); raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); kfree(its_dev->event_map.col_map); - kfree(its_dev->itt); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(its_dev->itt, get_its_device_order(its_dev)); + else +#endif + kfree(its_dev->itt);

#ifdef CONFIG_VIRT_PLAT_DEV if (its_dev->is_vdev) { @@ -5465,8 +5636,15 @@ static int __init its_probe_one(struct its_node *its) } }

- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, - get_order(ITS_CMD_QUEUE_SZ)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + page = its_alloc_shared_pages_node(its->numa_node, + GFP_KERNEL | __GFP_ZERO, + get_order(ITS_CMD_QUEUE_SZ)); + else +#endif + page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, + get_order(ITS_CMD_QUEUE_SZ)); if (!page) { err = -ENOMEM; goto out_unmap_sgir; @@ -5530,7 +5708,12 @@ static int __init its_probe_one(struct its_node *its) out_free_tables: its_free_tables(its); out_free_cmd: - free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); +#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) + its_free_shared_pages(its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); + else +#endif + free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); out_unmap_sgir: if (its->sgir_base) iounmap(its->sgir_base); @@ -6019,6 +6202,12 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, #endif int err;

+#ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) { + device_initialize(&cvm_alloc_device); + raw_spin_lock_init(&cvm_its_lock); + } +#endif gic_rdists = rdists;

its_parent = parent_domain; diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 6554a2e89d..813198abfc 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -7,6 +7,10 @@

#include "pci.h"

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif + /* * This interrupt-safe spinlock protects all accesses to PCI * configuration space. @@ -86,6 +90,19 @@ int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_cc_dev((bus->number << 8) | devfn)) { + if (size == 1) + *val = tmi_mmio_read(va_to_pa(addr), 8, ((bus->number << 8) | devfn)); + else if (size == 2) + *val = tmi_mmio_read(va_to_pa(addr), 16, ((bus->number << 8) | devfn)); + else + *val = tmi_mmio_read(va_to_pa(addr), 32, ((bus->number << 8) | devfn)); + + return PCIBIOS_SUCCESSFUL; + } +#endif + if (size == 1) *val = readb(addr); else if (size == 2) @@ -106,6 +123,22 @@ int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_cc_dev((bus->number << 8) | devfn)) { + if (size == 1) + WARN_ON(tmi_mmio_write(va_to_pa(addr), val, + 8, ((bus->number << 8) | devfn))); + else if (size == 2) + WARN_ON(tmi_mmio_write(va_to_pa(addr), val, + 16, ((bus->number << 8) | devfn))); + else + WARN_ON(tmi_mmio_write(va_to_pa(addr), val, + 32, ((bus->number << 8) | devfn))); + + return PCIBIOS_SUCCESSFUL; + } +#endif + if (size == 1) writeb(val, addr); else if (size == 2) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 7078200be1..c98fb0cbe1 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -13,6 +13,11 @@ #include "../pci.h" #include "msi.h"

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmm.h> +#include <asm/kvm_tmi.h> +#endif + int pci_msi_enable = 1; int pci_msi_ignore_mask;

@@ -159,9 +164,23 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual)) return;

- msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); - msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); - msg->data = readl(base + PCI_MSIX_ENTRY_DATA); +#ifdef CONFIG_HISI_VIRTCCA_HOST + u64 pbase = va_to_pa(base); + + if (dev != NULL && is_cc_dev(pci_dev_id(dev))) { + msg->address_lo = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + 32, pci_dev_id(dev)); + msg->address_hi = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + 32, pci_dev_id(dev)); + msg->data = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_DATA, 32, pci_dev_id(dev)); + } else { +#endif + msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); + msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); + msg->data = readl(base + PCI_MSIX_ENTRY_DATA); +#ifdef CONFIG_HISI_VIRTCCA_HOST + } +#endif } else { int pos = dev->msi_cap; u16 data; @@ -221,15 +240,41 @@ static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg if (unmasked) pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT);

- writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); - writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); - writel(msg->data, base + PCI_MSIX_ENTRY_DATA); +#ifdef CONFIG_HISI_VIRTCCA_HOST + u64 pbase = va_to_pa(base); + + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev != NULL && is_cc_dev(pci_dev_id(pdev))) { + u64 addr = (u64)msg->address_lo | ((u64)msg->address_hi << 32); + + addr += CVM_MSI_IOVA_OFFSET; + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + lower_32_bits(addr), 32, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + upper_32_bits(addr), 32, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_DATA, + msg->data, 32, pci_dev_id(pdev)); + } else { +#endif + writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); + writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); + writel(msg->data, base + PCI_MSIX_ENTRY_DATA); +#ifdef CONFIG_HISI_VIRTCCA_HOST + } +#endif

if (unmasked) pci_msix_write_vector_ctrl(desc, ctrl);

/* Ensure that the writes are visible in the device */ - readl(base + PCI_MSIX_ENTRY_DATA); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (pdev != NULL && is_cc_dev(pci_dev_id(pdev))) + tmi_mmio_read(va_to_pa(pbase + PCI_MSIX_ENTRY_DATA), 32, pci_dev_id(pdev)); + else +#endif + readl(base + PCI_MSIX_ENTRY_DATA); }

void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) @@ -632,7 +677,12 @@ void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc)

if (desc->pci.msi_attrib.can_mask) { void __iomem *addr = pci_msix_desc_addr(desc); - +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_cc_dev(pci_dev_id(dev))) + desc->pci.msix_ctrl = tmi_mmio_read(va_to_pa(addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + 32, pci_dev_id(dev)); + else +#endif desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } } @@ -684,6 +734,23 @@ static void msix_mask_all(void __iomem *base, int tsize) writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL); }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +static void msix_mask_all_cc(void __iomem *base, int tsize, u64 dev_num) +{ + u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; + int i; + u64 pbase = va_to_pa(base); + + if (pci_msi_ignore_mask) + return; + + for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) { + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, + ctrl, 32, dev_num); + } +} +#endif + static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { @@ -770,7 +837,12 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, * which takes the MSI-X mask bits into account even * when MSI-X is disabled, which prevents MSI delivery. */ - msix_mask_all(dev->msix_base, tsize); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_cc_dev(pci_dev_id(dev))) + msix_mask_all_cc(dev->msix_base, tsize, pci_dev_id(dev)); + else +#endif + msix_mask_all(dev->msix_base, tsize); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);

pcibios_free_irq(dev); diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c index d712a19e47..3f6a998667 100644 --- a/drivers/perf/arm_pmu.c +++ b/drivers/perf/arm_pmu.c @@ -736,6 +736,23 @@ static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) return 0; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +void arm_pmu_set_phys_irq(bool enable) +{ + int cpu = get_cpu(); + struct arm_pmu *pmu = per_cpu(cpu_armpmu, cpu); + int irq; + + irq = armpmu_get_cpu_irq(pmu, cpu); + if (irq && !enable) + per_cpu(cpu_irq_ops, cpu)->disable_pmuirq(irq); + else if (irq && enable) + per_cpu(cpu_irq_ops, cpu)->enable_pmuirq(irq); + + put_cpu(); +} +#endif + #ifdef CONFIG_CPU_PM static void cpu_pm_pmu_setup(struct arm_pmu *armpmu, unsigned long cmd) { diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 1929103ee5..0f3976692b 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -975,6 +975,11 @@ static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev, if (vdev->reset_works) info.flags |= VFIO_DEVICE_FLAGS_RESET;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_cc_dev(pci_dev_id(vdev->pdev))) + info.flags |= VFIO_DEVICE_FLAGS_SECURE; +#endif + info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS;

diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index e27de61ac9..e4deb868dc 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -18,6 +18,9 @@ #include <linux/vgaarb.h>

#include "vfio_pci_priv.h" +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_tmi.h> +#endif

#ifdef __LITTLE_ENDIAN #define vfio_ioread64 ioread64 @@ -37,6 +40,34 @@ #define vfio_ioread8 ioread8 #define vfio_iowrite8 iowrite8

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#define VFIO_IOWRITE(size) \ +static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size val, void __iomem *io) \ +{ \ + struct pci_dev *pdev = vdev->pdev; \ + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); \ + \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + if (cc_dev) { \ + WARN_ON(tmi_mmio_write(va_to_pa(io), val, size, pci_dev_id(pdev))); \ + } else { \ + vfio_iowrite##size(val, io); \ + } \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} +#else /* not CONFIG_HISI_VIRTCCA_HOST */ #define VFIO_IOWRITE(size) \ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io) \ @@ -56,6 +87,7 @@ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ \ return 0; \ } +#endif /* CONFIG_HISI_VIRTCCA_HOST */

VFIO_IOWRITE(8) VFIO_IOWRITE(16) @@ -64,6 +96,34 @@ VFIO_IOWRITE(32) VFIO_IOWRITE(64) #endif

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#define VFIO_IOREAD(size) \ +static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size *val, void __iomem *io) \ +{ \ + struct pci_dev *pdev = vdev->pdev; \ + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); \ + \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + if (cc_dev) { \ + *val = tmi_mmio_read(va_to_pa(io), size, pci_dev_id(pdev)); \ + } else { \ + *val = vfio_ioread##size(io); \ + } \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} +#else /* not CONFIG_HISI_VIRTCCA_HOST */ #define VFIO_IOREAD(size) \ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size *val, void __iomem *io) \ @@ -83,6 +143,7 @@ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ \ return 0; \ } +#endif

VFIO_IOREAD(8) VFIO_IOREAD(16) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 9c4adf11db..021f054088 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -77,6 +77,9 @@ struct vfio_iommu { bool dirty_page_tracking; struct list_head emulated_iommu_groups; bool dirty_log_get_no_clear; +#ifdef CONFIG_HISI_VIRTCCA_HOST + bool secure; +#endif };

struct vfio_domain { @@ -2454,6 +2457,11 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, goto out_domain; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (iommu->secure) + domain->domain->owner->iommu_enable_secure(domain->domain); +#endif + ret = iommu_attach_group(domain->domain, group->iommu_group); if (ret) goto out_domain; @@ -2807,6 +2815,12 @@ static void *vfio_iommu_type1_open(unsigned long arg) case VFIO_TYPE1v2_IOMMU: iommu->v2 = true; break; +#ifdef CONFIG_HISI_VIRTCCA_HOST + case VFIO_TYPE1v2_S_IOMMU: + iommu->v2 = true; + iommu->secure = true; + break; +#endif default: kfree(iommu); return ERR_PTR(-EINVAL); @@ -2898,6 +2912,9 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, switch (arg) { case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: +#ifdef CONFIG_HISI_VIRTCCA_HOST + case VFIO_TYPE1v2_S_IOMMU: +#endif case VFIO_TYPE1_NESTING_IOMMU: case VFIO_UNMAP_ALL: return 1; diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index a96d97da36..0d45ed89ab 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -318,6 +318,24 @@ static int __vfio_register_dev(struct vfio_device *device, return ret; }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +bool vfio_iommu_group(struct vfio_group *vfio_group, struct iommu_group *iommu_group) +{ + if (vfio_group->iommu_group == iommu_group) + return true; + return false; +} +EXPORT_SYMBOL_GPL(vfio_iommu_group); + +struct iommu_group *vfio_get_iommu_group(struct vfio_group *vfio_group) +{ + if (vfio_group) + return vfio_group->iommu_group; + return NULL; +} +EXPORT_SYMBOL_GPL(vfio_get_iommu_group); +#endif + int vfio_register_group_dev(struct vfio_device *device) { return __vfio_register_dev(device, VFIO_IOMMU); diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index fee20f66da..9d3f034bd8 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -172,4 +172,8 @@ static inline bool has_cntpoff(void) return (has_vhe() && cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF)); }

+#ifdef CONFIG_HISI_VIRTCCA_HOST +/* Needed for S-EL2 */ +void kvm_cvm_timers_update(struct kvm_vcpu *vcpu); +#endif #endif diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1d70dd0d0c..a957310861 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -635,7 +635,11 @@ struct iommu_ops { struct iommu_domain *blocked_domain; struct iommu_domain *default_domain;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + KABI_USE(1, int (*iommu_enable_secure)(struct iommu_domain *domain)) +#else KABI_RESERVE(1) +#endif KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) @@ -1148,6 +1152,11 @@ static inline void *dev_iommu_priv_get(struct device *dev) void dev_iommu_priv_set(struct device *dev, void *priv);

extern struct mutex iommu_probe_device_lock; + +#ifdef CONFIG_HISI_VIRTCCA_HOST +struct iommu_domain *iommu_group_get_domain(struct iommu_group *iommu_group); +#endif + int iommu_probe_device(struct device *dev);

int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h index 19a7b00baf..865938aebf 100644 --- a/include/linux/iopoll.h +++ b/include/linux/iopoll.h @@ -113,6 +113,44 @@ (cond) ? 0 : -ETIMEDOUT; \ })

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#define kvm_cvm_read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, \ + delay_before_read, args...) \ +({ \ + u64 __timeout_us = (timeout_us); \ + u64 rv = 0; \ + int result = 0; \ + unsigned long __delay_us = (delay_us); \ + ktime_t __timeout = ktime_add_us(ktime_get(), __timeout_us); \ + if (delay_before_read && __delay_us) \ + udelay(__delay_us); \ + for (;;) { \ + rv = op(args); \ + if (rv >> 32) { \ + result = -ENXIO; \ + break; \ + } \ + (val) = (u32)rv; \ + if (cond) \ + break; \ + if (__timeout_us && \ + ktime_compare(ktime_get(), __timeout) > 0) { \ + rv = op(args); \ + if (rv >> 32) { \ + result = -ENXIO; \ + break; \ + } \ + (val) = (u32)rv; \ + break; \ + } \ + if (__delay_us) \ + udelay(__delay_us); \ + cpu_relax(); \ + } \ + result ? result : ((cond) ? 0 : -ETIMEDOUT); \ +}) +#endif + /** * readx_poll_timeout - Periodically poll an address until a condition is met or a timeout occurs * @op: accessor function (takes @addr as its only argument) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 4557f47353..000ae1ba06 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -543,6 +543,28 @@ static __always_inline void guest_state_exit_irqoff(void) instrumentation_end(); }

+#ifdef CONFIG_HISI_VIRTCCA_HOST + +#define KVM_TYPE_CVM_BIT 8 +#define CVM_MAX_HALT_POLL_NS 100000 + +DECLARE_STATIC_KEY_FALSE(virtcca_cvm_is_available); + +static __always_inline bool vcpu_is_tec(struct kvm_vcpu *vcpu) +{ + if (static_branch_unlikely(&virtcca_cvm_is_available)) + return vcpu->arch.tec.tec_run; + + return false; +} + +static inline bool kvm_arm_cvm_type(unsigned long type) +{ + return type & (1UL << KVM_TYPE_CVM_BIT); +} + +#endif + static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) { /* diff --git a/include/linux/pci.h b/include/linux/pci.h index dcc442ca8b..1e5240d13e 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2727,4 +2727,8 @@ void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type); WARN_ONCE(condition, "%s %s: " fmt, \ dev_driver_string(&(pdev)->dev), pci_name(pdev), ##arg)

+#ifdef CONFIG_HISI_VIRTCCA_HOST +bool is_cc_dev(u32 sid); +#endif + #endif /* LINUX_PCI_H */ diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 143fbc10ec..c50f292363 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -167,6 +167,10 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn); static inline int arm_pmu_acpi_probe(armpmu_init_fn init_fn) { return 0; } #endif

+#ifdef CONFIG_HISI_VIRTCCA_HOST +void arm_pmu_set_phys_irq(bool enable); +#endif + #ifdef CONFIG_KVM void kvm_host_pmu_init(struct arm_pmu *pmu); #else diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 5ac5f182ce..d231eb5bf7 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -123,6 +123,11 @@ struct vfio_device_ops { void __user *arg, size_t argsz); };

+#ifdef CONFIG_HISI_VIRTCCA_HOST +extern bool vfio_iommu_group(struct vfio_group *vfio_group, struct iommu_group *iommu_group); +extern struct iommu_group *vfio_get_iommu_group(struct vfio_group *vfio_group); +#endif + #if IS_ENABLED(CONFIG_IOMMUFD) struct iommufd_ctx *vfio_iommufd_device_ictx(struct vfio_device *vdev); int vfio_iommufd_get_dev_id(struct vfio_device *vdev, struct iommufd_ctx *ictx); diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index cb63d56d8b..15012e7132 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1509,6 +1509,19 @@ struct kvm_numa_info { #define KVM_SET_TSS_ADDR _IO(KVMIO, 0x47) #define KVM_SET_IDENTITY_MAP_ADDR _IOW(KVMIO, 0x48, __u64)

+#define KVM_LOAD_USER_DATA _IOW(KVMIO, 0x49, struct kvm_user_data) + +#define KVM_CAP_ARM_TMM 300 /* FIXME: Large number to prevent conflicts */ + +struct kvm_user_data { + __u64 loader_start; + __u64 image_end; + __u64 initrd_start; + __u64 dtb_end; + __u64 ram_size; + struct kvm_numa_info numa_info; +}; + /* enable ucontrol for s390 */ struct kvm_s390_ucas_mapping { __u64 user_addr; diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 2b78d03c0c..c95f03258a 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -25,6 +25,8 @@ #define VFIO_TYPE1_IOMMU 1 #define VFIO_SPAPR_TCE_IOMMU 2 #define VFIO_TYPE1v2_IOMMU 3 +#define VFIO_TYPE1v2_S_IOMMU 12 + /* * IOMMU enforces DMA cache coherence (ex. PCIe NoSnoop stripping). This * capability is subject to change as groups are added or removed. @@ -224,6 +226,7 @@ struct vfio_device_info { #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6) /* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7) /* Info supports caps */ #define VFIO_DEVICE_FLAGS_CDX (1 << 8) /* vfio-cdx device */ +#define VFIO_DEVICE_FLAGS_SECURE (1 << 9) __u32 num_regions; /* Max region index + 1 */ __u32 num_irqs; /* Max IRQ index + 1 */ __u32 cap_offset; /* Offset within info struct of first cap */ diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index a7d5fb473b..5c10d1b7c3 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1734,4 +1734,20 @@ static int __init rmem_swiotlb_setup(struct reserved_mem *rmem) }

RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup); + +#ifdef CONFIG_HISI_VIRTCCA_GUEST +void __init swiotlb_cvm_update_mem_attributes(void) +{ + void *vaddr; + unsigned long bytes; + + if (!is_virtcca_cvm_world() || !is_swiotlb_allocated()) + return; + vaddr = phys_to_virt(io_tlb_default_mem.defpool.start); + bytes = PAGE_ALIGN(io_tlb_default_mem.defpool.nslabs << IO_TLB_SHIFT); + set_cvm_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT); + memset(vaddr, 0, bytes); + io_tlb_default_mem.for_alloc = true; +} +#endif #endif /* CONFIG_DMA_RESTRICTED_POOL */ diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 62f19b4b96..3ccb5f2bc3 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -3550,6 +3550,10 @@ static unsigned int kvm_vcpu_max_halt_poll_ns(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm;

+#ifdef CONFIG_HISI_VIRTCCA_HOST + if (vcpu_is_tec(vcpu)) + return CVM_MAX_HALT_POLL_NS; +#endif if (kvm->override_halt_poll_ns) { /* * Ensure kvm->max_halt_poll_ns is not read before diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index ca24ce1209..a62f72606f 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -17,6 +17,10 @@ #include <linux/vfio.h> #include "vfio.h"

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <asm/kvm_emulate.h> +#endif + #ifdef CONFIG_SPAPR_TCE_IOMMU #include <asm/kvm_ppc.h> #endif @@ -80,7 +84,7 @@ static bool kvm_vfio_file_is_valid(struct file *file) return ret; }

-#ifdef CONFIG_SPAPR_TCE_IOMMU +#if defined CONFIG_SPAPR_TCE_IOMMU || defined CONFIG_HISI_VIRTCCA_HOST static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file) { struct iommu_group *(*fn)(struct file *file); @@ -96,7 +100,9 @@ static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file)

return ret; } +#endif

+#ifdef ONFIG_SPAPR_TCE_IOMMU static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm, struct kvm_vfio_file *kvf) { @@ -142,6 +148,9 @@ static void kvm_vfio_update_coherency(struct kvm_device *dev)

static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) { +#ifdef CONFIG_HISI_VIRTCCA_HOST + struct iommu_group *iommu_group; +#endif struct kvm_vfio *kv = dev->private; struct kvm_vfio_file *kvf; struct file *filp; @@ -179,6 +188,18 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvm_vfio_file_set_kvm(kvf->file, dev->kvm); kvm_vfio_update_coherency(dev);

+#ifdef CONFIG_HISI_VIRTCCA_HOST + iommu_group = kvm_vfio_file_iommu_group(filp); + if (!iommu_group) { + ret = -ENXIO; + goto out_unlock; + } + if (cvm_arm_smmu_domain_set_kvm(iommu_group)) { + ret = -ENXIO; + goto out_unlock; + } +#endif + out_unlock: mutex_unlock(&kv->lock); out_fput: @@ -392,3 +413,101 @@ void kvm_vfio_ops_exit(void) { kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO); } + +#ifdef CONFIG_HISI_VIRTCCA_HOST +struct kvm *arm_smmu_get_kvm(struct arm_smmu_domain *domain) +{ + int ret = -1; + struct kvm *kvm; + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + struct iommu_group *iommu_group; + + unsigned long flags; + struct arm_smmu_master *master; + + spin_lock_irqsave(&domain->devices_lock, flags); + list_for_each_entry(master, &domain->devices, domain_head) { + if (master && master->num_streams >= 0) { + ret = 0; + break; + } + } + spin_unlock_irqrestore(&domain->devices_lock, flags); + if (ret) + return NULL; + + ret = -1; + iommu_group = master->dev->iommu_group; + mutex_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) { + mutex_lock(&kvm->lock); + list_for_each_entry(dev, &kvm->devices, vm_node) { + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") == 0) { + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + if (kvm_vfio_file_iommu_group(kvf->file) == iommu_group) { + ret = 0; + break; + } + } + mutex_unlock(&kv->lock); + if (!ret) + break; + } + } + mutex_unlock(&kvm->lock); + if (!ret) + break; + } + mutex_unlock(&kvm_lock); + + if (ret) + return NULL; + return kvm; +} + +void find_arm_smmu_domain(struct kvm_vfio_file *kvf, struct list_head *smmu_domain_group_list) +{ + struct iommu_group *iommu_group; + int ret = 0; + struct arm_smmu_domain *arm_smmu_domain = NULL; + struct arm_smmu_domain *arm_smmu_domain_node = NULL; + + iommu_group = kvm_vfio_file_iommu_group(kvf->file); + arm_smmu_domain = to_smmu_domain(iommu_group_get_domain(iommu_group)); + list_for_each_entry(arm_smmu_domain_node, + smmu_domain_group_list, node) { + if (arm_smmu_domain_node == arm_smmu_domain) { + ret = -1; + break; + } + } + if (!ret) + list_add_tail(&arm_smmu_domain->node, smmu_domain_group_list); +} + +int kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list) +{ + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + + INIT_LIST_HEAD(smmu_domain_group_list); + + list_for_each_entry(dev, &kvm->devices, vm_node) { + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") == 0) { + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + find_arm_smmu_domain(kvf, smmu_domain_group_list); + } + mutex_unlock(&kv->lock); + } + } + + return 0; +} +#endif diff --git a/virt/kvm/vfio.h b/virt/kvm/vfio.h index e130a4a035..43c303bcf6 100644 --- a/virt/kvm/vfio.h +++ b/virt/kvm/vfio.h @@ -2,6 +2,12 @@ #ifndef __KVM_VFIO_H #define __KVM_VFIO_H

+#ifdef CONFIG_HISI_VIRTCCA_HOST +#include <linux/kvm_host.h> +#include "../drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h" +#include "../drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h" +#endif + #ifdef CONFIG_KVM_VFIO int kvm_vfio_ops_init(void); void kvm_vfio_ops_exit(void); @@ -15,4 +21,8 @@ static inline void kvm_vfio_ops_exit(void) } #endif

+#ifdef CONFIG_HISI_VIRTCCA_HOST +struct kvm *arm_smmu_get_kvm(struct arm_smmu_domain *domain); +int kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list); +#endif #endif