From: Keith Busch <kbusch@kernel.org> stable inclusion from stable-v6.18.56 commit 790730e4d95de8356a573d4b4864486bba731863 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/20493 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=... -------------------------------- [ Upstream commit ab6c756f28c741d704a7c3e04814bfc3adf6f818 ] The cached requests are allocated for one operation but can be handed out for another. A passthrough command has RQF_USE_SCHED cleared, so using those flags for a subsequent read/write bio will insert it into the scheduler without ->prepare_request() and frees it without ->finish_request(). For kyber, this leaks the domain token acquired at dispatch and stalls the queue. Don't set RQF_USE_SCHED based on the first operation the batch happened to be allocated for. Instead, set it after the request is claimed by an operation. Introduce a helper function, blk_mq_rq_late_init() for this, and absorb blk_mq_rq_time_init() since the two run together. Fixes: 4b6a5d9cea91 ("block: enable batched allocation for blk_mq_alloc_request()") Link: https://lore.kernel.org/linux-block/20260916151655.2588-1-15815827059@163.co... Reported-by: Henry Hu <huhai@kylinos.cn> Signed-off-by: Keith Busch <kbusch@kernel.org> Reviewed-by: Christoph Hellwig <hch@lst.de> Link: https://patch.msgid.link/20260922172711.1573186-1-kbusch@meta.com Signed-off-by: Jens Axboe <axboe@kernel.dk> Signed-off-by: Sasha Levin <sashal@kernel.org> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- block/blk-mq.c | 64 +++++++++++++++++++++++++++++---------------- block/mq-deadline.c | 2 +- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index ed2ddd17841b..9724e5bce3f0 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -398,20 +398,10 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, INIT_LIST_HEAD(&rq->queuelist); /* tag was already set */ WRITE_ONCE(rq->deadline, 0); req_ref_set(rq, 1); - if (rq->rq_flags & RQF_USE_SCHED) { - struct elevator_queue *e = data->q->elevator; - - INIT_HLIST_NODE(&rq->hash); - RB_CLEAR_NODE(&rq->rb_node); - - if (e->type->ops.prepare_request) - e->type->ops.prepare_request(rq); - } - return rq; } static inline struct request * __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) @@ -442,10 +432,44 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) data->nr_tags -= nr; return rq_list_pop(data->cached_rq); } +static bool blk_op_bypass_sched(blk_opf_t opf) +{ + return (opf & REQ_OP_MASK) == REQ_OP_FLUSH || + blk_op_is_passthrough(opf); +} + +/* + * Finish initializing a request once it has been claimed for an operation. + * Cached requests are allocated before that operation is known. + */ +static void blk_mq_rq_late_init(struct request *rq, u64 alloc_time_ns) +{ + struct elevator_queue *e; + + if (!(rq->rq_flags & RQF_SCHED_TAGS) || (rq->rq_flags & RQF_RESV) || + blk_op_bypass_sched(rq->cmd_flags)) { + blk_mq_rq_time_init(rq, alloc_time_ns); + return; + } + + /* + * Set this before initializing the time stamps because + * blk_mq_rq_time_init() looks at RQF_USE_SCHED. + */ + rq->rq_flags |= RQF_USE_SCHED; + blk_mq_rq_time_init(rq, alloc_time_ns); + INIT_HLIST_NODE(&rq->hash); + RB_CLEAR_NODE(&rq->rb_node); + + e = rq->q->elevator; + if (e->type->ops.prepare_request) + e->type->ops.prepare_request(rq); +} + static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) { struct request_queue *q = data->q; u64 alloc_time_ns = 0; struct request *rq; @@ -471,17 +495,15 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) /* * Flush/passthrough requests are special and go directly to the * dispatch list. */ - if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH && - !blk_op_is_passthrough(data->cmd_flags)) { + if (!blk_op_bypass_sched(data->cmd_flags)) { struct elevator_mq_ops *ops = &q->elevator->type->ops; WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); - data->rq_flags |= RQF_USE_SCHED; if (ops->limit_depth) ops->limit_depth(data->cmd_flags, data); } } else { blk_mq_tag_busy(data->hctx); @@ -494,11 +516,11 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) * Try batched alloc if we want more than 1 tag. */ if (data->nr_tags > 1) { rq = __blk_mq_alloc_requests_batch(data); if (rq) { - blk_mq_rq_time_init(rq, alloc_time_ns); + blk_mq_rq_late_init(rq, alloc_time_ns); return rq; } data->nr_tags = 1; } @@ -520,11 +542,11 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) msleep(3); goto retry; } rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag); - blk_mq_rq_time_init(rq, alloc_time_ns); + blk_mq_rq_late_init(rq, alloc_time_ns); return rq; } static struct request *blk_mq_rq_cache_fill(struct request_queue *q, struct blk_plug *plug, @@ -563,12 +585,10 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, if (rq_list_empty(plug->cached_rq)) { if (plug->nr_ios == 1) return NULL; rq = blk_mq_rq_cache_fill(q, plug, opf, flags); - if (!rq) - return NULL; } else { rq = rq_list_peek(&plug->cached_rq); if (!rq || rq->q != q) return NULL; @@ -576,15 +596,15 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, return NULL; if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) return NULL; plug->cached_rq = rq_list_next(rq); - blk_mq_rq_time_init(rq, 0); + rq->cmd_flags = opf; + INIT_LIST_HEAD(&rq->queuelist); + blk_mq_rq_late_init(rq, 0); } - rq->cmd_flags = opf; - INIT_LIST_HEAD(&rq->queuelist); return rq; } struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, blk_mq_req_flags_t flags) @@ -679,11 +699,11 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, ret = -EWOULDBLOCK; tag = blk_mq_get_tag(&data); if (tag == BLK_MQ_NO_TAG) goto out_queue_exit; rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag); - blk_mq_rq_time_init(rq, alloc_time_ns); + blk_mq_rq_late_init(rq, alloc_time_ns); rq->__data_len = 0; rq->__sector = (sector_t) -1; rq->bio = rq->biotail = NULL; return rq; @@ -3036,13 +3056,13 @@ static bool blk_mq_can_use_cached_rq(struct request *rq, struct blk_plug *plug, * before we throttle. */ plug->cached_rq = rq_list_next(rq); rq_qos_throttle(rq->q, bio); - blk_mq_rq_time_init(rq, 0); rq->cmd_flags = bio->bi_opf; INIT_LIST_HEAD(&rq->queuelist); + blk_mq_rq_late_init(rq, 0); return true; } /** * blk_mq_submit_bio - Create and send a request to block device. diff --git a/block/mq-deadline.c b/block/mq-deadline.c index a9cf8e19f9d1..433b9cecdb6c 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -906,11 +906,11 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, rq_list_hierarchy_end_io_acct(&free, STAGE_DEADLINE); blk_mq_free_requests(&free); } -/* Callback from inside blk_mq_rq_ctx_init(). */ +/* Callback from inside blk_mq_rq_late_init(). */ static void dd_prepare_request(struct request *rq) { rq->elv.priv[0] = NULL; } -- 2.52.0