From: gaojuxin gaojuxin@loongson.cn
LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAZ38J
--------------------------------
Signed-off-by: gaojuxin gaojuxin@loongson.cn --- arch/loongarch/include/asm/io.h | 149 ++++++++++++++++++++++++++++++++ arch/loongarch/kernel/smp.c | 20 ++++- 2 files changed, 168 insertions(+), 1 deletion(-)
diff --git a/arch/loongarch/include/asm/io.h b/arch/loongarch/include/asm/io.h index 4a8adcca329b..dccb1921dfd2 100644 --- a/arch/loongarch/include/asm/io.h +++ b/arch/loongarch/include/asm/io.h @@ -73,6 +73,155 @@ extern void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t
#define __io_aw() mmiowb()
+#include <linux/spinlock.h> +extern spinlock_t lcl_node_lock[16]; +#define __raw_readb __raw_readb +static inline u8 __raw_readb(const volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + u8 val; + + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u8 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + /* read barrier */ + rmb(); + return val; +} + +#define __raw_readw __raw_readw +static inline u16 __raw_readw(const volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + u16 val; + + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u16 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + /* read barrier */ + rmb(); + return val; +} + +#define __raw_readl __raw_readl +static inline u32 __raw_readl(const volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + u32 val; + + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u32 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + /* read barrier */ + rmb(); + return val; +} + +#ifdef CONFIG_64BIT +#define __raw_readq __raw_readq +static inline u64 __raw_readq(const volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + u64 val; + + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u64 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + /* read barrier */ + rmb(); + return val; +} +#endif /* CONFIG_64BIT */ + +#define __raw_writeb __raw_writeb +static inline void __raw_writeb(u8 value, volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + + /* write barrier */ + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); + *(volatile u8 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); +} + +#define __raw_writew __raw_writew +static inline void __raw_writew(u16 value, volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + + /* write barrier */ + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); + *(volatile u16 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); +} + +#define __raw_writel __raw_writel +static inline void __raw_writel(u32 value, volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + + /* write barrier */ + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); + *(volatile u32 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); +} + +#ifdef CONFIG_64BIT +#define __raw_writeq __raw_writeq +static inline void __raw_writeq(u64 value, volatile void __iomem *addr) +{ + unsigned long dst_node, node; + unsigned long irq_flag; + + /* write barrier */ + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); + *(volatile u64 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); +} +#endif /* CONFIG_64BIT */ #include <asm-generic/io.h>
#define ARCH_HAS_VALID_PHYS_ADDR_RANGE diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index ea69606fd9f5..3e904356df3c 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -35,6 +35,8 @@ #include <asm/time.h> #include "legacy_boot.h"
+spinlock_t lcl_node_lock[16]; +EXPORT_SYMBOL(lcl_node_lock); int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ EXPORT_SYMBOL(__cpu_number_map);
@@ -196,7 +198,19 @@ static void ipi_write_action(int cpu, u32 action)
static void loongson_send_ipi_single(int cpu, unsigned int action) { + unsigned int curr_cpu = cpu_logical_map(smp_processor_id()); + unsigned int t_cpu = cpu_logical_map(cpu); + int flag = -1; + unsigned long irq_flag; + + if ((curr_cpu / 32) != (t_cpu / 32)) { + flag = curr_cpu / 32; + spin_lock_irqsave(&lcl_node_lock[flag], irq_flag); + asm ("dbar 0x0"); + } ipi_write_action(cpu_logical_map(cpu), (u32)action); + if (flag >= 0) + spin_unlock_irqrestore(&lcl_node_lock[flag], irq_flag); }
static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) @@ -204,7 +218,7 @@ static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int acti unsigned int i;
for_each_cpu(i, mask) - ipi_write_action(cpu_logical_map(i), (u32)action); + loongson_send_ipi_single(i, (u32)action); }
void arch_send_call_function_single_ipi(int cpu) @@ -305,6 +319,10 @@ static void __init fdt_smp_setup(void)
void __init loongson_smp_setup(void) { + int i; + + for (i = 0; i < 16; i++) + spin_lock_init(&lcl_node_lock[i]); fdt_smp_setup();
cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package;