virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBPH85 ------------------------------------------------------------------------ IPIV optimizes unicast scenarios and does not support multicast or broadcast. Instead, it converts multicast and broadcast traffic into multiple unicast streams. To prevent Guest OSes from generating multicast and broadcast traffic, the MPIDR of vCPUs is modified to ensure that the [aff3, aff2, aff1] fields are uniquely assigned for each vCPU within a virtual machine, while all aff0 fields are set to zero. This configuration guarantees the uniqueness of vCPU affinity identifiers at the architecture level, thereby suppressing the generation of vSGI multicast and broadcast signals by the virtualized environment. Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- arch/arm64/kvm/sys_regs.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 0cfa8d93e2ee..4d9bf04c1023 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -711,20 +711,30 @@ static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) return actlr; } +extern struct static_key_false ipiv_enable; + static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 mpidr; - /* - * Map the vcpu_id into the first three affinity level fields of - * the MPIDR. We limit the number of VCPUs in level 0 due to a - * limitation to 16 CPUs in that level in the ICC_SGIxR registers - * of the GICv3 to be able to address each CPU directly when - * sending IPIs. - */ - mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); - mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); - mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); + if (static_branch_unlikely(&ipiv_enable)) { + /* + * To avoid sending multi-SGIs in guest OS, make aff1/aff2 unique + */ + mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(1); + mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(2); + } else { + /* + * Map the vcpu_id into the first three affinity level fields of + * the MPIDR. We limit the number of VCPUs in level 0 due to a + * limitation to 16 CPUs in that level in the ICC_SGIxR registers + * of the GICv3 to be able to address each CPU directly when + * sending IPIs. + */ + mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); + mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); + mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); + } mpidr |= (1ULL << 31); vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1); -- 2.33.0

From: Xiang Chen <chenxiang66@hisilicon.com> virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBPH85 ------------------------------------------------------------------------ When the index of VM table exceeds the supported range or guest sends SGI with IRM=1, it triggers a exception interrupt (PPI interrupt). Just printk exception info in interrupt handler. Signed-off-by: Xiang Chen <chenxiang66@hisilicon.com> Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- arch/arm64/kvm/vgic/vgic-init.c | 44 +++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index db5db9402c3a..113accd6cd0b 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -3,6 +3,7 @@ * Copyright (C) 2015, 2016 ARM Ltd. */ +#include <linux/acpi.h> #include <linux/uaccess.h> #include <linux/interrupt.h> #include <linux/cpu.h> @@ -531,17 +532,30 @@ int kvm_vgic_map_resources(struct kvm *kvm) return ret; } +extern struct static_key_false ipiv_enable; +static int ipiv_irq; + /* GENERIC PROBE */ void kvm_vgic_cpu_up(void) { enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0); + if (static_branch_unlikely(&ipiv_enable)) + enable_percpu_irq(ipiv_irq, 0); } void kvm_vgic_cpu_down(void) { disable_percpu_irq(kvm_vgic_global_state.maint_irq); + if (static_branch_unlikely(&ipiv_enable)) + disable_percpu_irq(ipiv_irq); +} + +static irqreturn_t vgic_ipiv_irq_handler(int irq, void *data) +{ + kvm_info("IPIV irq handler!\n"); + return IRQ_HANDLED; } static irqreturn_t vgic_maintenance_handler(int irq, void *data) @@ -615,6 +629,15 @@ int kvm_vgic_hyp_init(void) kvm_vgic_global_state.no_hw_deactivation = true; } + if (static_branch_unlikely(&ipiv_enable)) { + ipiv_irq = acpi_register_gsi(NULL, 18, ACPI_EDGE_SENSITIVE, + ACPI_ACTIVE_HIGH); + if (ipiv_irq < 0) { + kvm_err("No ipiv exception irq\n"); + return -ENXIO; + } + } + kvm_vgic_global_state.flags = gic_kvm_info->flags; switch (gic_kvm_info->type) { case GIC_V2: @@ -637,7 +660,7 @@ int kvm_vgic_hyp_init(void) gic_kvm_info = NULL; if (ret) - return ret; + goto out_unregister_gsi; if (!has_mask && !kvm_vgic_global_state.maint_irq) return 0; @@ -648,9 +671,26 @@ int kvm_vgic_hyp_init(void) if (ret) { kvm_err("Cannot register interrupt %d\n", kvm_vgic_global_state.maint_irq); - return ret; + goto out_unregister_gsi; } kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq); + + if (static_branch_unlikely(&ipiv_enable)) { + ret = request_percpu_irq(ipiv_irq, vgic_ipiv_irq_handler, + "ipiv exception", kvm_get_running_vcpus()); + if (ret) { + kvm_err("Cannot register interrupt %d\n", ipiv_irq); + goto out_free_irq; + } + } + return 0; +out_free_irq: + free_percpu_irq(kvm_vgic_global_state.maint_irq, + kvm_get_running_vcpus()); +out_unregister_gsi: + if (static_branch_unlikely(&ipiv_enable)) + acpi_unregister_gsi(18); + return ret; } -- 2.33.0

From: Xiang Chen <chenxiang66@hisilicon.com> virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBPH85 ------------------------------------------------------------------------ IPIV is an virtualization extension on HIP12, which allows IPIs on guest directly sending by hardware to other vcpu stead of trapping to EL2. It will bring IPI interrupt optimization on guest. Introduce the method to detect and enable the feature, and also add a kernel command parameter "kvm-arm.ipiv_enabled" (default is 0) so that users can disable or enable the feature. The feature is based on GICv4p1. Signed-off-by: Xiang Chen <chenxiang66@hisilicon.com> Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- .../admin-guide/kernel-parameters.txt | 3 ++ arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/arm.c | 8 +++++ arch/arm64/kvm/hisilicon/hisi_virt.c | 33 +++++++++++++++++++ arch/arm64/kvm/hisilicon/hisi_virt.h | 11 +++++++ arch/arm64/kvm/vgic/vgic-mmio-v3.c | 1 + drivers/irqchip/irq-gic-v3.c | 30 +++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 12 +++++++ 8 files changed, 99 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 78a30d4453ad..6d8ad9c0e1fd 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2709,6 +2709,9 @@ [KVM,ARM] Allow use of GICv4 for direct injection of LPIs. + kvm-arm.ipiv_enabled= + [KVM,ARM] Allow use of HiSilicon ipiv on GICv4.1 + kvm-arm.dvmbm_enabled= [KVM,ARM] Allow use of HiSilicon DVMBM capability. Default: 0 diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ac4e59256f8e..bf884f95e6e6 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1286,5 +1286,6 @@ bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu); extern bool force_wfi_trap; extern bool kvm_ncsnp_support; extern bool kvm_dvmbm_support; +extern bool kvm_ipiv_support; #endif /* __ARM64_KVM_HOST_H__ */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 996a7117d709..f3fa3a74e041 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -67,6 +67,9 @@ bool kvm_ncsnp_support; /* Capability of DVMBM */ bool kvm_dvmbm_support; +/* Capability of IPIV */ +bool kvm_ipiv_support; + static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized); bool is_kvm_arm_initialised(void) @@ -2701,12 +2704,17 @@ static __init int kvm_arm_init(void) probe_hisi_cpu_type(); kvm_ncsnp_support = hisi_ncsnp_supported(); kvm_dvmbm_support = hisi_dvmbm_supported(); + kvm_ipiv_support = hisi_ipiv_supported(); kvm_info("KVM ncsnp %s\n", kvm_ncsnp_support ? "enabled" : "disabled"); kvm_info("KVM dvmbm %s\n", kvm_dvmbm_support ? "enabled" : "disabled"); + kvm_info("KVM ipiv %s\n", kvm_ipiv_support ? "enabled" : "disabled"); if (kvm_dvmbm_support) kvm_get_pg_cfg(); + if (kvm_ipiv_support) + ipiv_gicd_init(); + in_hyp_mode = is_kernel_in_hyp_mode(); #ifdef CONFIG_HISI_VIRTCCA_HOST diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.c b/arch/arm64/kvm/hisilicon/hisi_virt.c index d95b96ee7237..fd604ac9c11a 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.c +++ b/arch/arm64/kvm/hisilicon/hisi_virt.c @@ -12,6 +12,7 @@ static enum hisi_cpu_type cpu_type = UNKNOWN_HI_TYPE; static bool dvmbm_enabled; +static bool ipiv_enabled; static const char * const hisi_cpu_type_str[] = { "Hisi1612", @@ -157,6 +158,38 @@ static void hardware_disable_dvmbm(void *data) write_sysreg_s(val, SYS_LSUDVM_CTRL_EL2); } +static int __init early_ipiv_enable(char *buf) +{ + return strtobool(buf, &ipiv_enabled); +} +early_param("kvm-arm.ipiv_enabled", early_ipiv_enable); + +bool hisi_ipiv_supported(void) +{ + /* Determine whether IPIV is supported by the hardware */ + if (!(read_sysreg(aidr_el1) & AIDR_EL1_IPIV_MASK)) { + kvm_info("Hisi ipiv not supported by the hardware\n"); + return false; + } else + kvm_info("Hisi ipiv detected on the hardware\n"); + + /* User provided kernel command-line parameter */ + if (!ipiv_enabled || !is_kernel_in_hyp_mode()) + return false; + + /* Enable IPIV feature if necessary */ + if (!is_gicv4p1()) { + kvm_info("Need to enable GICv4p1!\n"); + return false; + } + return true; +} + +void ipiv_gicd_init(void) +{ + gic_dist_enable_ipiv(); +} + bool hisi_dvmbm_supported(void) { if (cpu_type != HI_IP10 && cpu_type != HI_IP10C && diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.h b/arch/arm64/kvm/hisilicon/hisi_virt.h index c45d319e7b41..f0b47c020261 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.h +++ b/arch/arm64/kvm/hisilicon/hisi_virt.h @@ -18,6 +18,8 @@ enum hisi_cpu_type { UNKNOWN_HI_TYPE }; +/* HIP12 */ +#define AIDR_EL1_IPIV_MASK GENMASK_ULL(17, 16) /* HIP10 */ #define AIDR_EL1_DVMBM_MASK GENMASK_ULL(13, 12) #define SYS_LSUDVM_CTRL_EL2 sys_reg(3, 4, 15, 7, 4) @@ -75,7 +77,9 @@ enum hisi_cpu_type { void probe_hisi_cpu_type(void); bool hisi_ncsnp_supported(void); bool hisi_dvmbm_supported(void); +bool hisi_ipiv_supported(void); void kvm_get_pg_cfg(void); +void ipiv_gicd_init(void); int kvm_sched_affinity_vcpu_init(struct kvm_vcpu *vcpu); void kvm_sched_affinity_vcpu_destroy(struct kvm_vcpu *vcpu); @@ -94,7 +98,12 @@ static inline bool hisi_dvmbm_supported(void) { return false; } +static inline bool hisi_ipiv_supported(void) +{ + return false; +} static inline void kvm_get_pg_cfg(void) {} +static inline void ipiv_gicd_init(void) {} static inline int kvm_sched_affinity_vcpu_init(struct kvm_vcpu *vcpu) { @@ -111,4 +120,6 @@ static inline void kvm_tlbi_dvmbm_vcpu_put(struct kvm_vcpu *vcpu) {} static inline void kvm_hisi_reload_lsudvmbm(struct kvm *kvm) {} #endif /* CONFIG_KVM_HISI_VIRT */ +extern bool gic_dist_enable_ipiv(void); +extern bool is_gicv4p1(void); #endif /* __HISI_VIRT_H__ */ diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index d55607dcd226..0edea419baf3 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -342,6 +342,7 @@ static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu, u64 value; value = (u64)(mpidr & GENMASK(23, 0)) << 32; + value |= MPIDR_AFFINITY_LEVEL(mpidr, 3) << 56; value |= ((target_vcpu_id & 0xffff) << 8); if (vgic_has_its(vcpu->kvm)) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 8e80eed7513c..7a64d77be181 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1514,6 +1514,36 @@ static int gic_dist_supports_lpis(void) !gicv3_nolpi); } +bool is_gicv4p1(void) +{ + if (!gic_data.rdists.has_rvpeid) + return false; + + return true; +} +EXPORT_SYMBOL(is_gicv4p1); + +void gic_dist_enable_ipiv(void) +{ + u32 val; + + val = readl_relaxed(gic_data.dist_base + GICD_MISC_CTRL); + val |= GICD_MISC_CTRL_CFG_IPIV_EN; + writel_relaxed(val, gic_data.dist_base + GICD_MISC_CTRL); + static_branch_enable(&ipiv_enable); + + val = (0 << GICD_IPIV_CTRL_AFF_DIRECT_VPEID_SHIFT) | + (0 << GICD_IPIV_CTRL_AFF1_LEFT_SHIFT_SHIFT) | + (4 << GICD_IPIV_CTRL_AFF2_LEFT_SHIFT_SHIFT) | + (7 << GICD_IPIV_CTRL_VM_TABLE_INNERCACHE_SHIFT) | + (2 << GICD_IPIV_CTRL_VM_TABLE_SHAREABILITY_SHIFT); + writel_relaxed(val, gic_data.dist_base + GICD_IPIV_CTRL); + + /* Set target ITS address of IPIV feature */ + writel_relaxed(0x4880, gic_data.dist_base + GICD_IPIV_ITS_TA_BASE); +} +EXPORT_SYMBOL(gic_dist_enable_ipiv); + static void gic_cpu_init(void) { void __iomem *rbase; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 074b347e8fcc..5fdc3ae4cd99 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -113,6 +113,18 @@ #define GIC_PAGE_SIZE_64K 2ULL #define GIC_PAGE_SIZE_MASK 3ULL +#define GICD_MISC_CTRL 0x2084 +#define GICD_MISC_CTRL_CFG_IPIV_EN (1U << 19) + +/* IPIV private register */ +#define GICD_IPIV_CTRL 0xc05c +#define GICD_IPIV_CTRL_AFF_DIRECT_VPEID_SHIFT 4 +#define GICD_IPIV_CTRL_AFF1_LEFT_SHIFT_SHIFT 8 +#define GICD_IPIV_CTRL_AFF2_LEFT_SHIFT_SHIFT 12 +#define GICD_IPIV_CTRL_VM_TABLE_INNERCACHE_SHIFT 16 +#define GICD_IPIV_CTRL_VM_TABLE_SHAREABILITY_SHIFT 19 +#define GICD_IPIV_ITS_TA_BASE 0xc010 + /* * Re-Distributor registers, offsets from RD_base */ -- 2.33.0

From: Xiang Chen <chenxiang66@hisilicon.com> virt inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBPH85 ------------------------------------------------------------------------ There are two modes for IPIV feature: - indirect mode, access vm table to retrieve vpeid; - direct mode, directly calcute vpeid according to mpidr; Add support for ipiv direct mode. Signed-off-by: Xiang Chen <chenxiang66@hisilicon.com> Signed-off-by: Jinqian Yang <yangjinqian1@huawei.com> --- .../admin-guide/kernel-parameters.txt | 5 ++ arch/arm64/kvm/arm.c | 13 +++-- arch/arm64/kvm/hisilicon/hisi_virt.c | 11 +++- arch/arm64/kvm/hisilicon/hisi_virt.h | 2 +- drivers/irqchip/irq-gic-v3-its.c | 50 ++++++++++++------- drivers/irqchip/irq-gic-v3.c | 18 ++++--- 6 files changed, 68 insertions(+), 31 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6d8ad9c0e1fd..a5a76107d294 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2712,6 +2712,11 @@ kvm-arm.ipiv_enabled= [KVM,ARM] Allow use of HiSilicon ipiv on GICv4.1 + kvm-arm.ipiv_direct= + [KVM,ARM] Use different modes of ipiv if ipiv enabled + Default is 0 which using vmtable in memory to retrieve aff-vpeid + Directly calculate vpeid according to aff if set 1 + kvm-arm.dvmbm_enabled= [KVM,ARM] Allow use of HiSilicon DVMBM capability. Default: 0 diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index f3fa3a74e041..43ef7742be82 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -316,6 +316,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) } extern struct static_key_false ipiv_enable; +extern struct static_key_false ipiv_direct; int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) { @@ -442,10 +443,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; #endif case KVM_CAP_ARM_IPIV_MODE: - if (static_branch_unlikely(&ipiv_enable)) - r = 1; - else - r = 0; + if (static_branch_unlikely(&ipiv_enable)) { + if (static_branch_unlikely(&ipiv_direct)) + r = 2; /* direct mode */ + else + r = 1; /* indirect mode */ + } else { + r = 0; /* don't enable IPIV */ + } break; default: r = 0; diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.c b/arch/arm64/kvm/hisilicon/hisi_virt.c index fd604ac9c11a..29d0a0179739 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.c +++ b/arch/arm64/kvm/hisilicon/hisi_virt.c @@ -13,6 +13,7 @@ static enum hisi_cpu_type cpu_type = UNKNOWN_HI_TYPE; static bool dvmbm_enabled; static bool ipiv_enabled; +static bool ipiv_direct; static const char * const hisi_cpu_type_str[] = { "Hisi1612", @@ -164,6 +165,12 @@ static int __init early_ipiv_enable(char *buf) } early_param("kvm-arm.ipiv_enabled", early_ipiv_enable); +static int __init early_ipiv_direct(char *buf) +{ + return kstrtobool(buf, &ipiv_direct); +} +early_param("kvm-arm.ipiv_direct", early_ipiv_direct); + bool hisi_ipiv_supported(void) { /* Determine whether IPIV is supported by the hardware */ @@ -182,12 +189,14 @@ bool hisi_ipiv_supported(void) kvm_info("Need to enable GICv4p1!\n"); return false; } + + kvm_info("Enable Hisi ipiv with %s mode\n", ipiv_direct ? "direct" : "indirect"); return true; } void ipiv_gicd_init(void) { - gic_dist_enable_ipiv(); + gic_dist_enable_ipiv(ipiv_direct); } bool hisi_dvmbm_supported(void) diff --git a/arch/arm64/kvm/hisilicon/hisi_virt.h b/arch/arm64/kvm/hisilicon/hisi_virt.h index f0b47c020261..09f33f7cc18e 100644 --- a/arch/arm64/kvm/hisilicon/hisi_virt.h +++ b/arch/arm64/kvm/hisilicon/hisi_virt.h @@ -120,6 +120,6 @@ static inline void kvm_tlbi_dvmbm_vcpu_put(struct kvm_vcpu *vcpu) {} static inline void kvm_hisi_reload_lsudvmbm(struct kvm *kvm) {} #endif /* CONFIG_KVM_HISI_VIRT */ -extern bool gic_dist_enable_ipiv(void); +extern bool gic_dist_enable_ipiv(bool direct); extern bool is_gicv4p1(void); #endif /* __HISI_VIRT_H__ */ diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 5cfd680ebcbb..dc45213b5587 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -389,6 +389,7 @@ static int alloc_devid_from_rsv_pools(struct rsv_devid_pool **devid_pool, #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) extern struct static_key_false ipiv_enable; +extern struct static_key_false ipiv_direct; #ifdef CONFIG_VIRT_PLAT_DEV /* @@ -4596,17 +4597,23 @@ static void its_vpe_4_1_schedule(struct its_vpe *vpe, /* wait gicr_ipiv_busy */ WARN_ON_ONCE(readl_relaxed_poll_timeout_atomic(vlpi_base + GICR_IPIV_ST, val, !(val & GICR_IPIV_ST_IPIV_BUSY), 1, 500)); - vpe_addr = virt_to_phys(page_address(vm->vpe_page)); - writel_relaxed(lower_32_bits(vpe_addr), - vlpi_base + GICR_VM_TABLE_BAR_L); - writel_relaxed(upper_32_bits(vpe_addr), - vlpi_base + GICR_VM_TABLE_BAR_H); - - /* setup gicr_vcpu_entry_num_max and gicr_ipiv_its_ta_sel */ - nr_vpes = vpe->its_vm->nr_vpes; - val = ((nr_vpes - 1) << GICR_IPIV_CTRL_VCPU_ENTRY_NUM_MAX_SHIFT) | - (0 << GICR_IPIV_CTRL_IPIV_ITS_TA_SEL_SHIFT); - writel_relaxed(val, vlpi_base + GICR_IPIV_CTRL); + if (!static_branch_unlikely(&ipiv_direct)) { + vpe_addr = virt_to_phys(page_address(vm->vpe_page)); + writel_relaxed(lower_32_bits(vpe_addr), + vlpi_base + GICR_VM_TABLE_BAR_L); + writel_relaxed(upper_32_bits(vpe_addr), + vlpi_base + GICR_VM_TABLE_BAR_H); + + /* setup gicr_vcpu_entry_num_max and gicr_ipiv_its_ta_sel */ + nr_vpes = vpe->its_vm->nr_vpes; + val = ((nr_vpes - 1) << GICR_IPIV_CTRL_VCPU_ENTRY_NUM_MAX_SHIFT) | + (0 << GICR_IPIV_CTRL_IPIV_ITS_TA_SEL_SHIFT); + writel_relaxed(val, vlpi_base + GICR_IPIV_CTRL); + } else { + /* setup gicr_ipiv_its_ta_sel */ + val = (0 << GICR_IPIV_CTRL_IPIV_ITS_TA_SEL_SHIFT); + writel_relaxed(val, vlpi_base + GICR_IPIV_CTRL); + } ipiv_disable_vsgi_trap(val); } else { @@ -4661,11 +4668,13 @@ static void its_vpe_4_1_deschedule(struct its_vpe *vpe, if (static_branch_unlikely(&ipiv_enable) && vm->nassgireq) { - /* wait gicr_ipiv_busy */ - WARN_ON_ONCE(readl_relaxed_poll_timeout_atomic(vlpi_base + GICR_IPIV_ST, - val, !(val & GICR_IPIV_ST_IPIV_BUSY), 1, 500)); - writel_relaxed(0, vlpi_base + GICR_VM_TABLE_BAR_L); - writel_relaxed(0, vlpi_base + GICR_VM_TABLE_BAR_H); + if (!static_branch_unlikely(&ipiv_direct)) { + /* wait gicr_ipiv_busy */ + WARN_ON_ONCE(readl_relaxed_poll_timeout_atomic(vlpi_base + GICR_IPIV_ST, + val, !(val & GICR_IPIV_ST_IPIV_BUSY), 1, 500)); + writel_relaxed(0, vlpi_base + GICR_VM_TABLE_BAR_L); + writel_relaxed(0, vlpi_base + GICR_VM_TABLE_BAR_H); + } ipiv_enable_vsgi_trap(val); } @@ -5079,7 +5088,8 @@ static void its_vpe_irq_domain_free(struct irq_domain *domain, if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) { its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); its_free_prop_table(vm->vprop_page); - if (static_branch_unlikely(&ipiv_enable)) { + if (static_branch_unlikely(&ipiv_enable) && + !static_branch_unlikely(&ipiv_direct)) { free_pages((unsigned long)page_address(vm->vpe_page), get_order(nr_irqs * 2)); } @@ -5120,7 +5130,8 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq if (gic_rdists->has_rvpeid) { irqchip = &its_vpe_4_1_irq_chip; - if (static_branch_unlikely(&ipiv_enable)) { + if (static_branch_unlikely(&ipiv_enable) && + !static_branch_unlikely(&ipiv_direct)) { vpe_page = alloc_pages(GFP_KERNEL, get_order(nr_irqs * 2)); if (!vpe_page) { its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); @@ -5137,7 +5148,8 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq err = its_vpe_init(vm->vpes[i]); if (err) break; - if (static_branch_unlikely(&ipiv_enable)) { + if (static_branch_unlikely(&ipiv_enable) && + !static_branch_unlikely(&ipiv_direct)) { vpe_entry = (u16 *)vpe_table_va + i; *(u16 *)vpe_entry = vm->vpes[i]->vpe_id; } diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index d1c2e2315e42..b1956e1c6273 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1526,7 +1526,7 @@ bool is_gicv4p1(void) } EXPORT_SYMBOL(is_gicv4p1); -void gic_dist_enable_ipiv(void) +void gic_dist_enable_ipiv(bool direct) { u32 val; @@ -1535,11 +1535,17 @@ void gic_dist_enable_ipiv(void) writel_relaxed(val, gic_data.dist_base + GICD_MISC_CTRL); static_branch_enable(&ipiv_enable); - val = (0 << GICD_IPIV_CTRL_AFF_DIRECT_VPEID_SHIFT) | - (0 << GICD_IPIV_CTRL_AFF1_LEFT_SHIFT_SHIFT) | - (4 << GICD_IPIV_CTRL_AFF2_LEFT_SHIFT_SHIFT) | - (7 << GICD_IPIV_CTRL_VM_TABLE_INNERCACHE_SHIFT) | - (2 << GICD_IPIV_CTRL_VM_TABLE_SHAREABILITY_SHIFT); + val = readl_relaxed(gic_data.dist_base + GICD_IPIV_CTRL); + if (direct) { + val |= (1U << GICD_IPIV_CTRL_AFF_DIRECT_VPEID_SHIFT); + static_branch_enable(&ipiv_direct); + } else { + val = (0 << GICD_IPIV_CTRL_AFF_DIRECT_VPEID_SHIFT) | + (0 << GICD_IPIV_CTRL_AFF1_LEFT_SHIFT_SHIFT) | + (4 << GICD_IPIV_CTRL_AFF2_LEFT_SHIFT_SHIFT) | + (7 << GICD_IPIV_CTRL_VM_TABLE_INNERCACHE_SHIFT) | + (2 << GICD_IPIV_CTRL_VM_TABLE_SHAREABILITY_SHIFT); + } writel_relaxed(val, gic_data.dist_base + GICD_IPIV_CTRL); /* Set target ITS address of IPIV feature */ -- 2.33.0