From: gaojuxin gaojuxin@loongson.cn
LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAZ38J
--------------------------------
Signed-off-by: gaojuxin gaojuxin@loongson.cn --- arch/loongarch/kernel/smp.c | 20 +++++++- include/asm-generic/io.h | 91 +++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 5 deletions(-)
diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index ea69606fd9f5..3e904356df3c 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -35,6 +35,8 @@ #include <asm/time.h> #include "legacy_boot.h"
+spinlock_t lcl_node_lock[16]; +EXPORT_SYMBOL(lcl_node_lock); int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ EXPORT_SYMBOL(__cpu_number_map);
@@ -196,7 +198,19 @@ static void ipi_write_action(int cpu, u32 action)
static void loongson_send_ipi_single(int cpu, unsigned int action) { + unsigned int curr_cpu = cpu_logical_map(smp_processor_id()); + unsigned int t_cpu = cpu_logical_map(cpu); + int flag = -1; + unsigned long irq_flag; + + if ((curr_cpu / 32) != (t_cpu / 32)) { + flag = curr_cpu / 32; + spin_lock_irqsave(&lcl_node_lock[flag], irq_flag); + asm ("dbar 0x0"); + } ipi_write_action(cpu_logical_map(cpu), (u32)action); + if (flag >= 0) + spin_unlock_irqrestore(&lcl_node_lock[flag], irq_flag); }
static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) @@ -204,7 +218,7 @@ static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int acti unsigned int i;
for_each_cpu(i, mask) - ipi_write_action(cpu_logical_map(i), (u32)action); + loongson_send_ipi_single(i, (u32)action); }
void arch_send_call_function_single_ipi(int cpu) @@ -305,6 +319,10 @@ static void __init fdt_smp_setup(void)
void __init loongson_smp_setup(void) { + int i; + + for (i = 0; i < 16; i++) + spin_lock_init(&lcl_node_lock[i]); fdt_smp_setup();
cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index bac63e874c7b..0f80b79cc640 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -16,9 +16,11 @@ #include <asm-generic/iomap.h> #endif
+#include <linux/spinlock.h> #include <asm/mmiowb.h> #include <asm-generic/pci_iomap.h>
+extern spinlock_t lcl_node_lock[16]; #ifndef __io_br #define __io_br() barrier() #endif @@ -113,7 +115,19 @@ static inline void log_post_read_mmio(u64 val, u8 width, const volatile void __i #define __raw_readb __raw_readb static inline u8 __raw_readb(const volatile void __iomem *addr) { - return *(const volatile u8 __force *)addr; + unsigned long dst_node, node; + unsigned long irq_flag; + u8 val; + + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u8 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + rmb(); + return val; } #endif
@@ -121,7 +135,18 @@ static inline u8 __raw_readb(const volatile void __iomem *addr) #define __raw_readw __raw_readw static inline u16 __raw_readw(const volatile void __iomem *addr) { - return *(const volatile u16 __force *)addr; + unsigned long dst_node, node; + unsigned long irq_flag; + u16 val; + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u16 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + rmb(); + return val; } #endif
@@ -129,7 +154,18 @@ static inline u16 __raw_readw(const volatile void __iomem *addr) #define __raw_readl __raw_readl static inline u32 __raw_readl(const volatile void __iomem *addr) { - return *(const volatile u32 __force *)addr; + unsigned long dst_node, node; + unsigned long irq_flag; + u32 val; + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u32 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + rmb(); + return val; } #endif
@@ -138,7 +174,18 @@ static inline u32 __raw_readl(const volatile void __iomem *addr) #define __raw_readq __raw_readq static inline u64 __raw_readq(const volatile void __iomem *addr) { - return *(const volatile u64 __force *)addr; + unsigned long dst_node, node; + unsigned long irq_flag; + u64 val; + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node + 8], irq_flag); + val = *(const volatile u64 __force *)addr; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node + 8], irq_flag); + rmb(); + return val; } #endif #endif /* CONFIG_64BIT */ @@ -147,7 +194,16 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) #define __raw_writeb __raw_writeb static inline void __raw_writeb(u8 value, volatile void __iomem *addr) { + unsigned long dst_node, node; + unsigned long irq_flag; + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); *(volatile u8 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); } #endif
@@ -155,7 +211,16 @@ static inline void __raw_writeb(u8 value, volatile void __iomem *addr) #define __raw_writew __raw_writew static inline void __raw_writew(u16 value, volatile void __iomem *addr) { + unsigned long dst_node, node; + unsigned long irq_flag; + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); *(volatile u16 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); } #endif
@@ -163,7 +228,16 @@ static inline void __raw_writew(u16 value, volatile void __iomem *addr) #define __raw_writel __raw_writel static inline void __raw_writel(u32 value, volatile void __iomem *addr) { + unsigned long dst_node, node; + unsigned long irq_flag; + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); *(volatile u32 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); } #endif
@@ -172,7 +246,16 @@ static inline void __raw_writel(u32 value, volatile void __iomem *addr) #define __raw_writeq __raw_writeq static inline void __raw_writeq(u64 value, volatile void __iomem *addr) { + unsigned long dst_node, node; + unsigned long irq_flag; + wmb(); + dst_node = ((unsigned long)addr >> 44) & 0xf; + node = get_csr_cpuid() / 32; + if (node != dst_node) + spin_lock_irqsave(&lcl_node_lock[node], irq_flag); *(volatile u64 __force *)addr = value; + if (node != dst_node) + spin_unlock_irqrestore(&lcl_node_lock[node], irq_flag); } #endif #endif /* CONFIG_64BIT */