[PATCH v2 OLK-5.10] iommu/iova: decrease cpu rcaches to improve iova rcache performance - Kernel

13 Jun 2023

hulk inclusion
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/I7ASVH
CVE: NA
---------------------------------------
The logic of cpu rcaches could cause severe performance problem. It is
because that the logic of iova rcache would create percpu rcache
which will be used only by local cpu, free iovas in cpu_rcache won't be
shared among different cpus. For example, platform which have 256 cpus
with 256 threads will have 417792 entiries( 6 * (256 * 2 + 32) * 128)
to cache iova, but every thread running on a cpu can only use 1/256 of
total cache size, and this could cause serve performance problem. We
limit the max cpu size to 64 to fix this problem, and each iovad could
save about 2MB memory.
In our test FIO with 256 threads:
Jobs: 12 (f=2): [/(1),R(2),/(9)][16.7%][r=2KiB/s][r=0 IOPS][eta 00m:30s]
Jobs: 12 (f=12): [R(12)][20.0%][r=1091MiB/s][r=279k IOPS][eta 00m:28s]
Jobs: 12 (f=12): [R(12)][22.2%][r=1426MiB/s][r=365k IOPS][eta 00m:28s]
Jobs: 12 (f=12): [R(12)][25.0%][r=1607MiB/s][r=411k IOPS][eta 00m:27s]
Jobs: 12 (f=12): [R(12)][27.8%][r=1501MiB/s][r=384k IOPS][eta 00m:26s]
Jobs: 12 (f=12): [R(12)][30.6%][r=1486MiB/s][r=380k IOPS][eta 00m:25s]
Jobs: 12 (f=12): [R(12)][33.3%][r=1393MiB/s][r=357k IOPS][eta 00m:24s]
Jobs: 12 (f=12): [R(12)][36.1%][r=1550MiB/s][r=397k IOPS][eta 00m:23s]
Jobs: 12 (f=12): [R(12)][38.9%][r=1485MiB/s][r=380k IOPS][eta 00m:22s]
After this patch:
bs: 10 (f=10): [R(10)][98.4%][r=7414MiB/s][r=1898k IOPS][eta 00m:15s]
Jobs: 10 (f=10): [R(10)][98.5%][r=7495MiB/s][r=1919k IOPS][eta 00m:14s]
Jobs: 10 (f=10): [R(10)][98.6%][r=7497MiB/s][r=1919k IOPS][eta 00m:13s]
Jobs: 10 (f=10): [R(10)][98.7%][r=7497MiB/s][r=1919k IOPS][eta 00m:12s]
Jobs: 10 (f=10): [R(10)][98.8%][r=7471MiB/s][r=1913k IOPS][eta 00m:11s]
Jobs: 10 (f=10): [R(10)][98.9%][r=7483MiB/s][r=1916k IOPS][eta 00m:10s]
Jobs: 10 (f=10): [R(10)][99.0%][r=7491MiB/s][r=1918k IOPS][eta 00m:09s]
Jobs: 10 (f=10): [R(10)][99.1%][r=7436MiB/s][r=1904k IOPS][eta 00m:08s]
Jobs: 10 (f=10): [R(10)][99.2%][r=7462MiB/s][r=1910k IOPS][eta 00m:07s]
can increase about 500% of IOPS.
Signed-off-by: Zhang Zekun zhangzekun11@huawei.com
Reviewed-by: Weilong Chen chenweilong@huawei.com
---
v2:
-add hulk inclusion
drivers/iommu/Kconfig | 10 ++++++++++
 drivers/iommu/iova.c  | 33 +++++++++++++++++++++++++++++++++
 include/linux/iova.h  |  5 +++++
 3 files changed, 48 insertions(+)

diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index f04a2bde0018..c71eb20bc702 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -343,6 +343,16 @@ config ARM_SMMU_V3_PM
    help
      Add support for suspend and resume support for arm smmu v3.
+config ARM_SMMU_V3_REDUCE_CPURCACHE
+	bool "Add support for cpu rcache"
+	depends on ARM_SMMU_V3
+	default n
+	help
+	  Add support for reducing the total amount of cpu rcache. When
+	  num of cpus is large, iova rcache will have serve performance
+	  problem. Reduce the amount of cpu rcaches to improve cache hit
+	  rate.
+
 config S390_IOMMU
    def_bool y if S390 && PCI
    depends on S390 && PCI
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 1246e8f8bf08..ed86bb09db4f 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -881,11 +881,19 @@ static void init_iova_rcaches(struct iova_domain *iovad)
    	rcache = &iovad->rcaches[i];
    	spin_lock_init(&rcache->lock);
    	rcache->depot_size = 0;
+#ifdef CONFIG_ARM_SMMU_V3_REDUCE_CPURCACHE
+		for (cpu = 0; cpu < MAX_CPU_SIZE; cpu++) {
+			rcache->cpu_rcaches[cpu] = kmalloc(sizeof(*cpu_rcache), GFP_KERNEL);
+			if (WARN_ON(!rcache->cpu_rcaches[cpu]))
+				continue;
+			cpu_rcache = rcache->cpu_rcaches[cpu];
+#else
    	rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
    	if (WARN_ON(!rcache->cpu_rcaches))
    		continue;
    	for_each_possible_cpu(cpu) {
    		cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
+#endif
    		spin_lock_init(&cpu_rcache->lock);
    		cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
    		cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
@@ -907,8 +915,14 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
    struct iova_cpu_rcache *cpu_rcache;
    bool can_insert = false;
    unsigned long flags;
+#ifdef CONFIG_ARM_SMMU_V3_REDUCE_CPURCACHE
+	int cpu;
+	cpu = raw_smp_processor_id();
+	cpu_rcache = rcache->cpu_rcaches[cpu % MAX_CPU_SIZE];
+#else
    cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
+#endif
    spin_lock_irqsave(&cpu_rcache->lock, flags);
if (!iova_magazine_full(cpu_rcache->loaded)) {
@@ -970,8 +984,14 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
    unsigned long iova_pfn = 0;
    bool has_pfn = false;
    unsigned long flags;
+#ifdef CONFIG_ARM_SMMU_V3_REDUCE_CPURCACHE
+	int cpu;
+	cpu = raw_smp_processor_id();
+	cpu_rcache = rcache->cpu_rcaches[cpu % MAX_CPU_SIZE];
+#else
    cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
+#endif
    spin_lock_irqsave(&cpu_rcache->lock, flags);
if (!iova_magazine_empty(cpu_rcache->loaded)) {
@@ -1026,12 +1046,21 @@ static void free_iova_rcaches(struct iova_domain *iovad)
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
    	rcache = &iovad->rcaches[i];
+#ifdef CONFIG_ARM_SMMU_V3_REDUCE_CPURCACHE
+		for (cpu = 0; cpu < MAX_CPU_SIZE; cpu++) {
+			cpu_rcache = rcache->cpu_rcaches[cpu];
+			iova_magazine_free(cpu_rcache->loaded);
+			iova_magazine_free(cpu_rcache->prev);
+			kfree(cpu_rcache);
+		}
+#else
    	for_each_possible_cpu(cpu) {
    		cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
    		iova_magazine_free(cpu_rcache->loaded);
    		iova_magazine_free(cpu_rcache->prev);
    	}
    	free_percpu(rcache->cpu_rcaches);
+#endif
    	for (j = 0; j < rcache->depot_size; ++j)
    		iova_magazine_free(rcache->depot[j]);
    }
@@ -1049,7 +1078,11 @@ void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
    	rcache = &iovad->rcaches[i];
+#ifdef CONFIG_ARM_SMMU_V3_REDUCE_CPURCACHE
+		cpu_rcache = rcache->cpu_rcaches[cpu % MAX_CPU_SIZE];
+#else
    	cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
+#endif
    	spin_lock_irqsave(&cpu_rcache->lock, flags);
    	iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
    	iova_magazine_free_pfns(cpu_rcache->prev, iovad);
diff --git a/include/linux/iova.h b/include/linux/iova.h
index dfa51ae49666..06897fc970eb 100644
--- a/include/linux/iova.h
+++ b/include/linux/iova.h
@@ -27,12 +27,17 @@ struct iova_cpu_rcache;
#define IOVA_RANGE_CACHE_MAX_SIZE 6	/* log of max cached IOVA range size (in pages) */
 #define MAX_GLOBAL_MAGS 32	/* magazines per bin */
+#define MAX_CPU_SIZE (NR_CPUS > 64 ? 64 : NR_CPUS)	/* max cpu rcaches */
struct iova_rcache {
    spinlock_t lock;
    unsigned long depot_size;
    struct iova_magazine *depot[MAX_GLOBAL_MAGS];
+#ifdef CONFIG_ARM_SMMU_V3_REDUCE_CPURCACHE
+	struct iova_cpu_rcache *cpu_rcaches[MAX_CPU_SIZE];
+#else
    struct iova_cpu_rcache __percpu *cpu_rcaches;
+#endif
 };
struct iova_domain;
-- 
2.17.1


    