From: Yu Kuai yukuai3@huawei.com

In certain environments, specific CPUs handle a large number of tasks and become bottlenecks, affecting overall system performance. This commit introduces a new feature that enables asynchronous I/O dispatch to designated CPUs, thereby relieving the pressure on the busy CPUs.

Signed-off-by: Yu Kuai yukuai3@huawei.com Signed-off-by: Li Nan linan122@huawei.com --- block/blk.h | 7 ++ include/linux/blk_types.h | 1 + include/linux/blkdev.h | 8 ++ block/blk-core.c | 229 +++++++++++++++++++++++++++++++++++++- block/blk-mq-debugfs.c | 1 + block/blk-sysfs.c | 58 ++++++++++ block/Kconfig | 12 ++ 7 files changed, 315 insertions(+), 1 deletion(-)

diff --git a/block/blk.h b/block/blk.h index 4bbcc971d4f7..5e7c00356ddc 100644 --- a/block/blk.h +++ b/block/blk.h @@ -450,4 +450,11 @@ int bio_add_hw_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned int max_sectors, bool *same_page);

+#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC +void blk_free_queue_dispatch_async(struct request_queue *q); +#else +static inline void blk_free_queue_dispatch_async(struct request_queue *q) +{ +} +#endif #endif /* BLK_INTERNAL_H */ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 1853ec569b72..865e1198e837 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -301,6 +301,7 @@ enum { * of this bio. */ BIO_CGROUP_ACCT, /* has been accounted to a cgroup */ BIO_TRACKED, /* set if bio goes through the rq_qos path */ + BIO_ASYNC, BIO_FLAG_LAST };

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 50b4fd0a0687..453bae093b65 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -601,6 +601,12 @@ struct request_queue { #define BLK_MAX_WRITE_HINTS 5 u64 write_hints[BLK_MAX_WRITE_HINTS];

+ +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC + /* used when QUEUE_FLAG_DISPATCH_ASYNC is set */ + struct cpumask dispatch_async_cpus; + int __percpu *last_dispatch_cpu; +#endif KABI_REPLACE(unsigned long dtag_wait_time, struct blk_mq_tags *shared_sbitmap_tags) KABI_RESERVE(1) @@ -643,6 +649,8 @@ struct request_queue { #define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */ /*at least one blk-mq hctx can't get driver tag */ #define QUEUE_FLAG_HCTX_WAIT 30 +/* support to dispatch bio asynchronously */ +#define QUEUE_FLAG_DISPATCH_ASYNC 31

#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ (1 << QUEUE_FLAG_SAME_COMP) | \ diff --git a/block/blk-core.c b/block/blk-core.c index f91f8e8be482..03375d28d664 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -87,6 +87,223 @@ struct kmem_cache *blk_requestq_cachep; */ static struct workqueue_struct *kblockd_workqueue;

+#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC + +#define BIO_DISPATCH_MAX_LOOP 16 + +struct async_bio { + struct bio_list list; + spinlock_t lock; +} ____cacheline_aligned_in_smp; + +struct bio_dispatch_async_ctl { + /* + * Vector size is nr_cpu_ids, list stores bio dispatched from other cpu, + * such bio will be dispatched asynchronously to the cpu this structure + * is serviced. + */ + struct async_bio *bios; + /* kthread to handle bio dispatched from other cpu. */ + struct task_struct *thread; + wait_queue_head_t wait; +}; + +static struct bio_dispatch_async_ctl __percpu *bio_dispatch_async_ctl; + +static int blk_alloc_queue_dispatch_async(struct request_queue *q) +{ + int cpu; + + q->last_dispatch_cpu = alloc_percpu(int); + if (!q->last_dispatch_cpu) + return -ENOMEM; + + cpumask_setall(&q->dispatch_async_cpus); + for_each_possible_cpu(cpu) + *per_cpu_ptr(q->last_dispatch_cpu, cpu) = cpu; + + return 0; +} + +void blk_free_queue_dispatch_async(struct request_queue *q) +{ + free_percpu(q->last_dispatch_cpu); + q->last_dispatch_cpu = NULL; +} + +static int get_dispatch_cpu(struct request_queue *q, int cpu) +{ + int *last_dispatch_cpu = + per_cpu_ptr(q->last_dispatch_cpu, cpu); + + cpu = cpumask_next(*last_dispatch_cpu, &q->dispatch_async_cpus); + if (cpu >= nr_cpu_ids) + cpu = cpumask_first(&q->dispatch_async_cpus); + + *last_dispatch_cpu = cpu; + + return cpu; +} + +static bool __submit_bio_noacct_async(struct bio *bio) +{ + int cpu = smp_processor_id(); + int dispatch_cpu = get_dispatch_cpu(bio->bi_disk->queue, cpu); + struct bio_dispatch_async_ctl *ctl; + + if (dispatch_cpu >= nr_cpu_ids || dispatch_cpu == cpu) + return false; + + ctl = per_cpu_ptr(bio_dispatch_async_ctl, dispatch_cpu); + spin_lock_irq(&ctl->bios[cpu].lock); + bio_list_add(&ctl->bios[cpu].list, bio); + spin_unlock_irq(&ctl->bios[cpu].lock); + + if (wq_has_sleeper(&ctl->wait)) + wake_up(&ctl->wait); + + return true; +} + +static bool submit_bio_noacct_async(struct bio *bio) +{ + struct request_queue *q; + + if (bio_flagged(bio, BIO_ASYNC)) + return false; + + bio_set_flag(bio, BIO_ASYNC); + /* + * Don't dispatch bio asynchronously in following cases: + * + * - QUEUE_FLAG_DISPATCH_ASYNC is not set; + * - current cpu is the target cpu; + * - bio is flagged no wait; + * - io polling is enabled; + */ + q = bio->bi_disk->queue; + if (!test_bit(QUEUE_FLAG_DISPATCH_ASYNC, &q->queue_flags) || + test_bit(QUEUE_FLAG_POLL, &q->queue_flags) || + cpumask_test_cpu(smp_processor_id(), &q->dispatch_async_cpus) || + bio->bi_opf & REQ_NOWAIT) + return false; + + return __submit_bio_noacct_async(bio); +} + +static bool collect_bio(struct bio_dispatch_async_ctl *ctl, + struct bio_list *list) +{ + bool has_bio = false; + int cpu; + + for_each_possible_cpu(cpu) { + struct async_bio *abio = &ctl->bios[cpu]; + + if (bio_list_empty(&abio->list)) + continue; + + has_bio = true; + + spin_lock_irq(&abio->lock); + bio_list_merge(list, &abio->list); + bio_list_init(&abio->list); + spin_unlock_irq(&abio->lock); + } + + return has_bio; +} + +static int bio_dispatch_work(void *data) +{ + int loop_count = 0; + struct bio_list bio_list_on_stack; + struct blk_plug plug; + struct bio_dispatch_async_ctl *ctl; + + bio_list_init(&bio_list_on_stack); + ctl = this_cpu_ptr(bio_dispatch_async_ctl); + + for (;; loop_count++) { + struct bio *bio; + bool has_bio = collect_bio(ctl, &bio_list_on_stack); + + if (!has_bio) { + DEFINE_WAIT(wait); + + for (;;) { + prepare_to_wait(&ctl->wait, &wait, + TASK_INTERRUPTIBLE); + has_bio = collect_bio(ctl, &bio_list_on_stack); + if (has_bio) + break; + schedule(); + loop_count = 0; + } + finish_wait(&ctl->wait, &wait); + } + + blk_start_plug(&plug); + while ((bio = bio_list_pop(&bio_list_on_stack))) + submit_bio_noacct(bio); + blk_finish_plug(&plug); + + /* prevent soft lockup. */ + if (loop_count >= BIO_DISPATCH_MAX_LOOP) { + loop_count = 0; + cond_resched(); + } + } + + return 0; +} + +static void init_blk_queue_async_dispatch(void) +{ + int cpu; + + bio_dispatch_async_ctl = alloc_percpu(struct bio_dispatch_async_ctl); + if (!bio_dispatch_async_ctl) + panic("Failed to alloc bio_dispatch_async_ctl\n"); + + for_each_possible_cpu(cpu) { + int i; + struct bio_dispatch_async_ctl *ctl = + per_cpu_ptr(bio_dispatch_async_ctl, cpu); + + init_waitqueue_head(&ctl->wait); + ctl->bios = kmalloc_array(nr_cpu_ids, sizeof(struct async_bio), + GFP_KERNEL | __GFP_NOFAIL); + for (i = 0; i < nr_cpu_ids; ++i) { + bio_list_init(&ctl->bios[i].list); + spin_lock_init(&ctl->bios[i].lock); + } + + ctl->thread = + kthread_create_on_cpu(bio_dispatch_work, NULL, cpu, + "bio_dispatch_work_%u"); + if (IS_ERR_OR_NULL(ctl->thread)) + panic("Failed to create bio dispatch thread\n"); + + wake_up_process(ctl->thread); + } +} +#else +static int blk_alloc_queue_dispatch_async(struct request_queue *q) +{ + return 0; +} + +static bool submit_bio_noacct_async(struct bio *bio) +{ + return false; +} + +static void init_blk_queue_async_dispatch(void) +{ +} +#endif + /** * blk_queue_flag_set - atomically set a queue flag * @flag: flag to be set @@ -539,9 +756,12 @@ struct request_queue *blk_alloc_queue(int node_id)

q->last_merge = NULL;

+ if (blk_alloc_queue_dispatch_async(q)) + goto fail_q; + q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL); if (q->id < 0) - goto fail_q; + goto fail_dispatch_async;

ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); if (ret) @@ -606,6 +826,8 @@ struct request_queue *blk_alloc_queue(int node_id) bioset_exit(&q->bio_split); fail_id: ida_simple_remove(&blk_queue_ida, q->id); +fail_dispatch_async: + blk_free_queue_dispatch_async(q); fail_q: kmem_cache_free(blk_requestq_cachep, q); return NULL; @@ -1055,6 +1277,9 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) */ blk_qc_t submit_bio_noacct(struct bio *bio) { + if (submit_bio_noacct_async(bio)) + return BLK_QC_T_NONE; + if (!submit_bio_checks(bio)) return BLK_QC_T_NONE;

@@ -1905,5 +2130,7 @@ int __init blk_dev_init(void)

blk_debugfs_root = debugfs_create_dir("block", NULL);

+ init_blk_queue_async_dispatch(); + return 0; } diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index a879f94782e4..b5b17c6ee650 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -131,6 +131,7 @@ static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(RQ_ALLOC_TIME), QUEUE_FLAG_NAME(HCTX_ACTIVE), QUEUE_FLAG_NAME(NOWAIT), + QUEUE_FLAG_NAME(DISPATCH_ASYNC), }; #undef QUEUE_FLAG_NAME

diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index c95be9626a09..e363222858b4 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -288,6 +288,9 @@ QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1); QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0); QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0); QUEUE_SYSFS_BIT_FNS(stable_writes, STABLE_WRITES, 0); +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC +QUEUE_SYSFS_BIT_FNS(dispatch_async, DISPATCH_ASYNC, 0); +#endif #undef QUEUE_SYSFS_BIT_FNS

static ssize_t queue_zoned_show(struct request_queue *q, char *page) @@ -619,6 +622,56 @@ QUEUE_RW_ENTRY(queue_iostats, "iostats"); QUEUE_RW_ENTRY(queue_random, "add_random"); QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");

+#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC + +static ssize_t queue_dispatch_async_cpus_show(struct request_queue *q, + char *page) +{ + + return sprintf(page, "%*pb\n", nr_cpu_ids, + cpumask_bits(&q->dispatch_async_cpus)); +} + +static ssize_t queue_dispatch_async_cpus_store(struct request_queue *q, + const char *page, size_t count) +{ + cpumask_var_t cpumask; + ssize_t ret; + + if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) + return -ENOMEM; + + ret = bitmap_parse(page, count, cpumask_bits(cpumask), + nr_cpumask_bits); + if (ret < 0) + goto out; + + if (cpumask_empty(cpumask) || + !cpumask_subset(cpumask, cpu_online_mask)) + return -EINVAL; + + blk_mq_freeze_queue(q); + blk_mq_quiesce_queue(q); + + cpumask_copy(&q->dispatch_async_cpus, cpumask); + + blk_mq_unquiesce_queue(q); + blk_mq_unfreeze_queue(q); + ret = count; +out: + free_cpumask_var(cpumask); + return ret; +} + +static struct queue_sysfs_entry queue_dispatch_async_cpus_entry = { + .attr = {.name = "dispatch_async_cpus", .mode = 0644 }, + .show = queue_dispatch_async_cpus_show, + .store = queue_dispatch_async_cpus_store, +}; + +QUEUE_RW_ENTRY(queue_dispatch_async, "dispatch_async"); +#endif + static struct attribute *queue_attrs[] = { &queue_requests_entry.attr, &queue_ra_entry.attr, @@ -659,6 +712,10 @@ static struct attribute *queue_attrs[] = { &queue_wb_lat_entry.attr, &queue_poll_delay_entry.attr, &queue_io_timeout_entry.attr, +#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC + &queue_dispatch_async_cpus_entry.attr, + &queue_dispatch_async_entry.attr, +#endif #ifdef CONFIG_BLK_DEV_THROTTLING_LOW &blk_throtl_sample_time_entry.attr, #endif @@ -795,6 +852,7 @@ static void blk_release_queue(struct kobject *kobj) blk_stat_remove_callback(q, q->poll_cb); blk_stat_free_callback(q->poll_cb);

+ blk_free_queue_dispatch_async(q); blk_free_queue_stats(q->stats);

blk_exit_queue(q); diff --git a/block/Kconfig b/block/Kconfig index e5c965f1ea25..bda77b006352 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -209,6 +209,18 @@ config BLK_DEV_DUMPINFO Dump info when open an write opened block device exclusively or open an exclusive opened device for write

+config BLK_BIO_DISPATCH_ASYNC + bool "Dispatch bios asynchronously on specific cpus" + default n + help + If there are multiple nodes, memory access across nodes is rather bad + compare to local node. And if some drivers are using internal spin + locks, io performance will be bad if bios are issued concurrently from + different nodes. This feature will dispatch bio asynchronously to the + specific CPUs to avoid across nodes memory access in driver, noted this + feature will require special care in the driver to work. If unsure, + say N here. + menu "Partition Types"

source "block/partitions/Kconfig"