From: Yu Kuai yukuai3@huawei.com
hulk inclusion category: performance bugzilla: 187597, https://gitee.com/openeuler/kernel/issues/I5QK5M CVE: NA
--------------------------------
If CONFIG_BLK_BIO_DISPATCH_ASYNC is enabled, and driver support QUEUE_FLAG_DISPATCH_ASYNC, bios will be dispatched asynchronously to specific CPUs to avoid across nodes memory access in driver.
Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Jason Yan yanaijie@huawei.com Signed-off-by: Yongqiang Liu liuyongqiang13@huawei.com --- block/Kconfig | 13 +++++++++++++ block/blk-core.c | 21 ++++++++++++++++++++- block/blk-sysfs.c | 4 ++++ block/blk.h | 8 ++++++++ include/linux/blkdev.h | 7 +++++++ 5 files changed, 52 insertions(+), 1 deletion(-)
diff --git a/block/Kconfig b/block/Kconfig index 1f2469a0123c..da71e56f8682 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -200,6 +200,19 @@ config BLK_SED_OPAL Enabling this option enables users to setup/unlock/lock Locking ranges for SED devices using the Opal protocol.
+config BLK_BIO_DISPATCH_ASYNC + bool "Dispatch bios asynchronously on specific cpus" + default n + depends on BLOCK=y + help + If there are multiple nodes, memory access across nodes is rather bad + compare to local node. And if some drivers are using internal spin + locks, io performance will be bad if bios are issued concurrently from + different nodes. This feature will dispatch bio asynchronously to the + specific CPUs to avoid across nodes memory access in driver, noted this + feature will require special care in the driver to work. If unsure, + say N here. + menu "Partition Types"
source "block/partitions/Kconfig" diff --git a/block/blk-core.c b/block/blk-core.c index 08e7a875b229..2ce4ad4619c1 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -35,7 +35,6 @@ #include <linux/blk-cgroup.h> #include <linux/debugfs.h> #include <linux/bpf.h> -#include <linux/arch_topology.h>
#define CREATE_TRACE_POINTS #include <trace/events/block.h> @@ -86,6 +85,9 @@ struct kmem_cache *blk_requestq_cachep; */ static struct workqueue_struct *kblockd_workqueue;
+#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC +#include <linux/arch_topology.h> + #define BIO_DISPATCH_MAX_LOOP 16 /* the minimum of cpus that dispatch async can be enabled */ #define MIN_DISPATCH_ASYNC_CPUS 16 @@ -309,6 +311,23 @@ void queue_init_dispatch_async_cpus(struct request_queue *q, int node) cpumask_setall(dispatch_async_cpus); } EXPORT_SYMBOL_GPL(queue_init_dispatch_async_cpus); +#else +static int blk_alloc_queue_dispatch_async(struct request_queue *q) +{ + return 0; +} + +static blk_qc_t blk_queue_do_make_request(struct bio *bio) +{ + struct request_queue *q = bio->bi_disk->queue; + + return q->make_request_fn(q, bio); +} + +static void init_blk_queue_async_dispatch(void) +{ +} +#endif
/** * blk_queue_flag_set - atomically set a queue flag diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index cbd7de22c463..5bdae7c8fa27 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -696,6 +696,7 @@ static struct queue_sysfs_entry queue_wb_lat_entry = { .store = queue_wb_lat_store, };
+#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC static ssize_t queue_dispatch_async_cpus_show(struct request_queue *q, char *page) { @@ -731,6 +732,7 @@ static struct queue_sysfs_entry queue_dispatch_async_entry = { .attr = {.name = "dispatch_async", .mode = 0444 }, .show = queue_show_dispatch_async, }; +#endif
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW static struct queue_sysfs_entry throtl_sample_time_entry = { @@ -774,8 +776,10 @@ static struct attribute *default_attrs[] = { &queue_dax_entry.attr, &queue_wb_lat_entry.attr, &queue_poll_delay_entry.attr, +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC &queue_dispatch_async_cpus_entry.attr, &queue_dispatch_async_entry.attr, +#endif #ifdef CONFIG_BLK_DEV_THROTTLING_LOW &throtl_sample_time_entry.attr, #endif diff --git a/block/blk.h b/block/blk.h index cb1c9057d788..985e20980aff 100644 --- a/block/blk.h +++ b/block/blk.h @@ -49,9 +49,11 @@ struct request_queue_wrapper { struct mutex mq_freeze_lock; int mq_freeze_depth;
+#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC /* used when QUEUE_FLAG_DISPATCH_ASYNC is set */ struct cpumask dispatch_async_cpus; int __percpu *last_dispatch_cpu; +#endif };
#define queue_to_wrapper(q) \ @@ -464,6 +466,12 @@ extern int blk_iolatency_init(struct request_queue *q); static inline int blk_iolatency_init(struct request_queue *q) { return 0; } #endif
+#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC extern void blk_free_queue_dispatch_async(struct request_queue *q); +#else +static inline void blk_free_queue_dispatch_async(struct request_queue *q) +{ +} +#endif
#endif /* BLK_INTERNAL_H */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 996e00575042..41d235ee579a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -785,7 +785,14 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
extern void blk_set_pm_only(struct request_queue *q); extern void blk_clear_pm_only(struct request_queue *q); +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC extern void queue_init_dispatch_async_cpus(struct request_queue *q, int node); +#else +static inline void queue_init_dispatch_async_cpus(struct request_queue *q, + int node) +{ +} +#endif
static inline int queue_in_flight(struct request_queue *q) {