hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/release-management/issues/IBV2E4 -------------------------------- Support hardware xint. Hardware xint provides a separate entry for interrupt handling, so we can use it to customize and respond to interrupts relatively quickly. Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com> --- v2: - Reuse the xint set and check logic in xint 1.0. - Introduce fast_handle_xint(). - Rename enable_xint_xcall() to enable_xfunc(). - Rename test_has_xint_xcall() to test_has_xfunc(). - Fix a typo, Harware -> Hardware. - Add a blank line after declarations in el0t_64_xint_handler(). --- arch/arm64/Kconfig | 7 +++ arch/arm64/include/asm/sysreg.h | 4 ++ arch/arm64/kernel/cpufeature.c | 75 ++++++++++++++++++++++++- arch/arm64/kernel/entry-common.c | 35 +++++++++--- arch/arm64/kernel/entry.S | 40 +++++++++++++- arch/arm64/tools/cpucaps | 2 +- drivers/irqchip/irq-gic-v3.c | 89 ++++++++++++++++++++++++++++-- include/linux/irqchip/arm-gic-v3.h | 6 +- kernel/irq/debugfs.c | 6 +- kernel/irq/internals.h | 3 +- 10 files changed, 244 insertions(+), 23 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 9da9d58f1c02..57d05cfbd29e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1717,6 +1717,13 @@ config ARCH_DEFAULT_KEXEC_IMAGE_VERIFY_SIG config ARCH_SUPPORTS_CRASH_DUMP def_bool y +config ARCH_SUPPORTS_XINT + bool "Hardware xint support" + default n + depends on ARM64_NMI + depends on ARM_GIC_V3 + depends on !COMPAT + config TRANS_TABLE def_bool y depends on HIBERNATION || KEXEC_CORE diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 435634a703c6..b34e8cc4476b 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -269,6 +269,10 @@ #define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6) #define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1) + +#define ACTLR_ELx_XINT_SHIFT 21 +#define ACTLR_ELx_XINT (BIT(ACTLR_ELx_XINT_SHIFT)) + #define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5) #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index a1736e9044da..b4de05241f7d 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2406,8 +2406,12 @@ static bool has_xcall_support(const struct arm64_cpu_capabilities *entry, int __ } #endif -#ifdef CONFIG_FAST_IRQ +#if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ARCH_SUPPORTS_XINT) bool is_xint_support; +bool hw_xint_support; +#endif + +#ifdef CONFIG_FAST_IRQ static int __init xint_setup(char *str) { if (!cpus_have_cap(ARM64_HAS_GIC_CPUIF_SYSREGS)) @@ -2424,6 +2428,66 @@ static bool has_xint_support(const struct arm64_cpu_capabilities *entry, int __u } #endif +#ifdef CONFIG_ARCH_SUPPORTS_XINT +static bool test_has_xfunc(void) +{ + u64 new, old = read_sysreg(actlr_el1); + + write_sysreg(old | ACTLR_ELx_XINT, actlr_el1); + isb(); + new = read_sysreg(actlr_el1); + if (new & ACTLR_ELx_XINT) { + write_sysreg(old, actlr_el1); + hw_xint_support = true; + return true; + } + + return false; +} + +static void enable_xfunc(void) +{ + u64 actlr_el1, actlr_el2; + u64 el; + + el = read_sysreg(CurrentEL); + if (el == CurrentEL_EL2) { + actlr_el2 = read_sysreg(actlr_el2); + actlr_el2 |= ACTLR_ELx_XINT; + write_sysreg(actlr_el2, actlr_el2); + isb(); + actlr_el2 = read_sysreg(actlr_el2); + pr_info("actlr_el2: %llx, cpu:%d\n", actlr_el2, smp_processor_id()); + } + + actlr_el1 = read_sysreg(actlr_el1); + actlr_el1 |= ACTLR_ELx_XINT; + write_sysreg(actlr_el1, actlr_el1); + isb(); + actlr_el1 = read_sysreg(actlr_el1); + pr_info("actlr_el1: %llx, cpu:%d\n", actlr_el1, smp_processor_id()); +} + +static bool test_has_xint(const struct arm64_cpu_capabilities *entry, int scope) +{ + if (!IS_ENABLED(CONFIG_ARM64_NMI)) + pr_info("CONFIG_ARM64_NMI disabled, using XINTs for guests only\n"); +#ifdef CONFIG_ARM64_PSEUDO_NMI + else if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && enable_pseudo_nmi) { + pr_info("Pseudo NMI enabled, not using architected XINT\n"); + return false; + } +#endif + + return test_has_xfunc(); +} + +static void xint_enable(const struct arm64_cpu_capabilities *__unused) +{ + enable_xfunc(); +} +#endif + static const struct arm64_cpu_capabilities arm64_features[] = { { .capability = ARM64_ALWAYS_BOOT, @@ -2971,6 +3035,15 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_xint_support, }, +#endif +#ifdef CONFIG_ARCH_SUPPORTS_XINT + { + .desc = "Hardware xint Support", + .capability = ARM64_HAS_HW_XINT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = test_has_xint, + .cpu_enable = xint_enable, + }, #endif {}, }; diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 4602c107c40a..d09029dfcf02 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -6,6 +6,9 @@ */ #include <linux/context_tracking.h> +#ifdef CONFIG_ARCH_SUPPORTS_XINT +#include <linux/irqchip/arm-gic-v3.h> +#endif #include <linux/kasan.h> #include <linux/linkage.h> #include <linux/lockdep.h> @@ -607,7 +610,7 @@ static void noinstr el0_xint(struct pt_regs *regs, u64 nmi_flag, } -asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) +asmlinkage void noinstr el0t_64_sw_xint_handler(struct pt_regs *regs) { el0_xint(regs, ISR_EL1_IS, handle_arch_irq, handle_arch_nmi_irq); } @@ -966,6 +969,30 @@ asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) __el0_error_handler_common(regs); } +#ifdef CONFIG_ARCH_SUPPORTS_XINT +asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) +{ + u32 irqnr = read_sysreg_s(SYS_ICC_HPPIR1_EL1); + + if (gic_irqnr_is_special(irqnr)) + return; + + if (is_xint(irqnr)) + fast_handle_xint(regs, irqnr); + else + el0t_64_irq_handler(regs); +} +#else +#ifdef CONFIG_AARCH32_EL0 +asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) +{ + __el0_irq_handler_common(regs); +} +#else /* CONFIG_AARCH32_EL0 */ +UNHANDLED(el0t, 32, irq) +#endif +#endif + #ifdef CONFIG_AARCH32_EL0 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) { @@ -1028,11 +1055,6 @@ asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) } } -asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) -{ - __el0_irq_handler_common(regs); -} - asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) { __el0_fiq_handler_common(regs); @@ -1044,7 +1066,6 @@ asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) } #else /* CONFIG_AARCH32_EL0 */ UNHANDLED(el0t, 32, sync) -UNHANDLED(el0t, 32, irq) UNHANDLED(el0t, 32, fiq) UNHANDLED(el0t, 32, error) #endif /* CONFIG_AARCH32_EL0 */ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index da3809632f0f..046225fa2f90 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -565,7 +565,11 @@ SYM_CODE_START(vectors) kernel_ventry 0, t, 64, error // Error 64-bit EL0 kernel_ventry 0, t, 32, sync // Synchronous 32-bit EL0 +#ifdef CONFIG_ARCH_SUPPORTS_XINT + kernel_ventry 0, t, 64, xint // XINT 64-bit EL0 +#else kernel_ventry 0, t, 32, irq // IRQ 32-bit EL0 +#endif kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 kernel_ventry 0, t, 32, error // Error 32-bit EL0 SYM_CODE_END(vectors) @@ -696,7 +700,7 @@ SYM_CODE_END(__bad_stack) kernel_entry 0, 64 #endif mov x0, sp - bl el0t_64_xint_handler + bl el0t_64_sw_xint_handler #ifdef CONFIG_SECURITY_FEATURE_BYPASS kernel_exit 0, xint #else @@ -729,6 +733,18 @@ SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) check_xint_pre_kernel_entry .Lskip_check_xint\@: .endif +#endif +#ifdef CONFIG_ARCH_SUPPORTS_XINT + .if \el == 0 && \regsize == 64 && \label == xint + alternative_if_not ARM64_HAS_HW_XINT + b .Lskip_hw_xint\@ + alternative_else_nop_endif + kernel_entry 0, 64, xint + mov x0, sp + bl el0t_64_xint_handler + kernel_exit 0, xint +.Lskip_hw_xint\@: + .endif #endif kernel_entry \el, \regsize mov x0, sp @@ -760,7 +776,11 @@ SYM_CODE_END(el\el\ht\()_\regsize\()_\label) entry_handler 0, t, 64, error entry_handler 0, t, 32, sync +#ifdef CONFIG_ARCH_SUPPORTS_XINT + entry_handler 0, t, 64, xint +#else entry_handler 0, t, 32, irq +#endif entry_handler 0, t, 32, fiq entry_handler 0, t, 32, error @@ -905,7 +925,14 @@ alternative_else_nop_endif .rept 4 tramp_ventry .Lvector_start\@, 64, \kpti, \bhb .endr - .rept 4 + +#ifdef CONFIG_ARCH_SUPPORTS_XINT + tramp_ventry .Lvector_start\@, 64, \kpti, \bhb +#else + tramp_ventry .Lvector_start\@, 32, \kpti, \bhb +#endif + + .rept 3 tramp_ventry .Lvector_start\@, 32, \kpti, \bhb .endr .endm @@ -955,7 +982,14 @@ SYM_CODE_END(tramp_exit) .rept 4 tramp_ventry .Lvector_start\@, 64, 0, \bhb .endr - .rept 4 + +#ifdef CONFIG_ARCH_SUPPORTS_XINT + tramp_ventry .Lvector_start\@, 64, 0, \bhb +#else + tramp_ventry .Lvector_start\@, 32, 0, \bhb +#endif + + .rept 3 tramp_ventry .Lvector_start\@, 32, 0, \bhb .endr .endm diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index f2ddced689b5..b1f109f17e4f 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -110,7 +110,7 @@ WORKAROUND_HISI_HIP08_RU_PREFETCH WORKAROUND_HISILICON_1980005 HAS_XCALL HAS_XINT -KABI_RESERVE_3 +HAS_HW_XINT KABI_RESERVE_4 KABI_RESERVE_5 KABI_RESERVE_6 diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 4cc8b95d533f..32ffdf06f61e 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -35,7 +35,7 @@ #include "irq-gic-common.h" -#ifdef CONFIG_FAST_IRQ +#if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ARCH_SUPPORTS_XINT) #include "../../../kernel/irq/internals.h" #endif @@ -828,7 +828,7 @@ static bool gic_rpr_is_nmi_prio(void) return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI)); } -static bool gic_irqnr_is_special(u32 irqnr) +bool gic_irqnr_is_special(u32 irqnr) { return irqnr >= 1020 && irqnr <= 1023; } @@ -993,7 +993,11 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs __gic_handle_irq_from_irqson(regs); } -#ifdef CONFIG_FAST_IRQ +#ifdef CONFIG_ARCH_SUPPORTS_XINT +DECLARE_BITMAP(irqnr_nmi_map, 1024); +#endif + +#if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ARCH_SUPPORTS_XINT) DECLARE_BITMAP(irqnr_xint_map, 1024); static bool can_set_xint(unsigned int hwirq) @@ -1002,12 +1006,18 @@ static bool can_set_xint(unsigned int hwirq) __get_intid_range(hwirq) == SPI_RANGE) return true; +#ifdef CONFIG_ARCH_SUPPORTS_XINT + if (hw_xint_support && __get_intid_range(hwirq) == PPI_RANGE) + return true; +#endif + return false; } static bool xint_transform(int irqno, enum xint_op op) { struct irq_data *data = irq_get_irq_data(irqno); + struct irq_desc *desc; int hwirq; while (data->parent_data) @@ -1018,14 +1028,29 @@ static bool xint_transform(int irqno, enum xint_op op) if (!can_set_xint(hwirq)) return false; + desc = irq_data_to_desc(data); + switch (op) { case IRQ_TO_XINT: set_bit(hwirq, irqnr_xint_map); xint_add_debugfs_entry(irqno); +#ifdef CONFIG_ARCH_SUPPORTS_XINT + if (has_v3_3_nmi() && hw_xint_support && !irq_is_nmi(desc)) { + gic_irq_enable_nmi(data); + set_bit(hwirq, irqnr_nmi_map); + } +#endif return true; case XINT_TO_IRQ: clear_bit(hwirq, irqnr_xint_map); xint_remove_debugfs_entry(irqno); +#ifdef CONFIG_ARCH_SUPPORTS_XINT + if (has_v3_3_nmi() && hw_xint_support && irq_is_nmi(desc) && + test_bit(hwirq, irqnr_nmi_map)) { + gic_irq_disable_nmi(data); + clear_bit(hwirq, irqnr_nmi_map); + } +#endif return false; case XINT_SET_CHECK: return test_bit(hwirq, irqnr_xint_map); @@ -1096,7 +1121,7 @@ static const struct proc_ops xint_proc_ops = { void register_irqchip_proc(struct irq_desc *desc, void *irqp) { - if (!is_xint_support) + if (!is_xint_support && !hw_xint_support) return; /* create /proc/irq/<irq>/xint */ @@ -1105,12 +1130,63 @@ void register_irqchip_proc(struct irq_desc *desc, void *irqp) void unregister_irqchip_proc(struct irq_desc *desc) { - if (!is_xint_support) + if (!is_xint_support && !hw_xint_support) return; remove_proc_entry("xint", desc->dir); } -#endif /* CONFIG_FAST_IRQ */ +#endif + +#ifdef CONFIG_ARCH_SUPPORTS_XINT +bool is_xint(unsigned long hwirq) +{ + return test_bit(hwirq, irqnr_xint_map); +} + +static bool is_spi(unsigned long hwirq) +{ + if (__get_intid_range(hwirq) == SPI_RANGE || + __get_intid_range(hwirq) == ESPI_RANGE) + return true; + + return false; +} + +void fast_handle_xint(struct pt_regs *regs, u32 irqnr) +{ + struct pt_regs *old_regs; + struct irq_domain *domain; + struct irqaction *action; + struct irq_desc *desc; + struct irq_data *data; + + arch_nmi_enter(); + BUG_ON(in_nmi() == NMI_MASK); + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); + old_regs = set_irq_regs(regs); + + domain = irq_get_default_host(); + data = radix_tree_lookup(&domain->revmap_tree, irqnr); + + desc = irq_data_to_desc(data); + action = desc->action; + + gic_read_nmiar(); + write_gicreg(irqnr, ICC_EOIR1_EL1); + isb(); + + if (is_spi(irqnr)) + action->handler(data->irq, action->dev_id); + else + action->handler(data->irq, raw_cpu_ptr(action->percpu_dev_id)); + gic_write_dir(irqnr); + + set_irq_regs(old_regs); + BUG_ON(!in_nmi()); + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); + arch_nmi_exit(); +} +#endif static u32 gic_get_pribits(void) { @@ -2358,6 +2434,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, goto out_free; } + irq_set_default_host(gic_data.domain); irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); gic_data.has_rss = !!(typer & GICD_TYPER_RSS); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 548b8a5c46cf..3c0f04b86c15 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -721,8 +721,9 @@ static inline enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) } } -#ifdef CONFIG_FAST_IRQ +#if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ARCH_SUPPORTS_XINT) extern bool is_xint_support; +extern bool hw_xint_support; enum xint_op { XINT_TO_IRQ, @@ -733,6 +734,9 @@ enum xint_op { void register_irqchip_proc(struct irq_desc *desc, void *irqp); void unregister_irqchip_proc(struct irq_desc *desc); +bool gic_irqnr_is_special(u32 irqnr); +bool is_xint(unsigned long hwirq); +void fast_handle_xint(struct pt_regs *regs, u32 irqnr); #endif #endif diff --git a/kernel/irq/debugfs.c b/kernel/irq/debugfs.c index dc94c360b54b..2152125f8ae2 100644 --- a/kernel/irq/debugfs.c +++ b/kernel/irq/debugfs.c @@ -242,7 +242,7 @@ void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc) &dfs_irq_ops); } -#ifdef CONFIG_FAST_IRQ +#if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ARCH_SUPPORTS_XINT) static struct dentry *xint_dir; void xint_add_debugfs_entry(unsigned int irq) @@ -281,8 +281,8 @@ static int __init irq_debugfs_init(void) irq_dir = debugfs_create_dir("irqs", root_dir); -#ifdef CONFIG_FAST_IRQ - if (is_xint_support) +#if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ARCH_SUPPORTS_XINT) + if (is_xint_support || hw_xint_support) xint_dir = debugfs_create_dir("xints", root_dir); #endif diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 2fb08139a9e4..1c64854102f5 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -506,8 +506,9 @@ static inline void irq_remove_debugfs_entry(struct irq_desc *desc) kfree(desc->dev_name); } -#ifdef CONFIG_FAST_IRQ +#if defined(CONFIG_FAST_IRQ) || defined(CONFIG_ARCH_SUPPORTS_XINT) extern bool is_xint_support; +extern bool hw_xint_support; void xint_add_debugfs_entry(unsigned int irq); void xint_remove_debugfs_entry(unsigned int irq); -- 2.34.1