From: Yu Kuai yukuai3@huawei.com

hulk inclusion category: bugfix bugzilla: 34280, https://gitee.com/openeuler/kernel/issues/I4AKY4 CVE: NA

-----------------------------------------------

This reverts commit 15c87cc96c186fc0927d2e0dac0887643293c73c.

Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Hou Tao houtao1@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- block/blk-mq-tag.c | 77 ++++++++---------------------------------- block/blk-mq.c | 6 ++-- include/linux/blk-mq.h | 3 +- 3 files changed, 19 insertions(+), 67 deletions(-)

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index ce7f7188625eb..c8997e5c30f8a 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -217,51 +217,37 @@ struct bt_iter_data { busy_iter_fn *fn; void *data; bool reserved; - bool inflight; };

static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) { struct bt_iter_data *iter_data = data; struct blk_mq_hw_ctx *hctx = iter_data->hctx; + struct blk_mq_tags *tags = hctx->tags; bool reserved = iter_data->reserved; - struct blk_mq_tags *tags; struct request *rq;

- tags = hctx->sched_tags ? hctx->sched_tags : hctx->tags; - if (!reserved) bitnr += tags->nr_reserved_tags; + rq = tags->rqs[bitnr];

/* - * Because tags->rqs[] will not been cleaned when free driver tag - * and there is a window between get driver tag and write tags->rqs[], - * so we may see stale rq in tags->rqs[] which may have been freed. - * Using static_rqs[] is safer. - */ - rq = tags->static_rqs[bitnr]; - - /* - * There is a small window between get tag and blk_mq_rq_ctx_init, - * so rq->q and rq->mq_hctx maybe different. + * We can hit rq == NULL here, because the tagging functions + * test and set the bit before assining ->rqs[]. */ - if (rq && rq->q == hctx->queue && - (!iter_data->inflight || - blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)) + if (rq && rq->q == hctx->queue) iter_data->fn(hctx, rq, iter_data->data, reserved); return true; }

-static void bt_for_each(struct blk_mq_hw_ctx *hctx, - struct sbitmap_queue *bt, busy_iter_fn *fn, - void *data, bool reserved, bool inflight) +static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt, + busy_iter_fn *fn, void *data, bool reserved) { struct bt_iter_data iter_data = { .hctx = hctx, .fn = fn, .data = data, .reserved = reserved, - .inflight = inflight, };

sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data); @@ -359,23 +345,22 @@ void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset) } EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request);

-static void __blk_mq_queue_tag_busy_iter(struct request_queue *q, - busy_iter_fn *fn, void *priv, bool inflight) +void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, + void *priv) { struct blk_mq_hw_ctx *hctx; int i;

/* - * Get a reference of the queue unless it has been zero. We use this - * to avoid the race with the code that would modify the hctxs after - * freeze and drain the queue, including updating nr_hw_queues, io - * scheduler switching and queue clean up. + * __blk_mq_update_nr_hw_queues will update the nr_hw_queues and + * queue_hw_ctx after freeze the queue, so we use q_usage_counter + * to avoid race with it. */ if (!percpu_ref_tryget(&q->q_usage_counter)) return;

queue_for_each_hw_ctx(q, hctx, i) { - struct blk_mq_tags *tags; + struct blk_mq_tags *tags = hctx->tags;

/* * If not software queues are currently mapped to this @@ -384,45 +369,13 @@ static void __blk_mq_queue_tag_busy_iter(struct request_queue *q, if (!blk_mq_hw_queue_mapped(hctx)) continue;

- tags = hctx->sched_tags ? hctx->sched_tags : hctx->tags; - if (tags->nr_reserved_tags) - bt_for_each(hctx, &tags->breserved_tags, - fn, priv, true, inflight); - bt_for_each(hctx, &tags->bitmap_tags, - fn, priv, false, inflight); - /* - * flush_rq represents the rq with REQ_PREFLUSH and REQ_FUA - * (if FUA is not supported by device) to be issued to - * device. So we need to consider it when iterate inflight - * rqs, but needn't to count it when iterate busy tags. - */ - if (inflight && - blk_mq_rq_state(hctx->fq->flush_rq) == MQ_RQ_IN_FLIGHT) - fn(hctx, hctx->fq->flush_rq, priv, false); + bt_for_each(hctx, &tags->breserved_tags, fn, priv, true); + bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false); } blk_queue_exit(q); }

-/* - * Iterate all the busy tags including pending and in-flight ones. - */ -void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, - void *priv) -{ - __blk_mq_queue_tag_busy_iter(q, fn, priv, false); -} - -/* - * Iterate all the inflight tags. - */ -void blk_mq_queue_tag_inflight_iter(struct request_queue *q, - busy_iter_fn *fn, void *priv) -{ - __blk_mq_queue_tag_busy_iter(q, fn, priv, true); -} -EXPORT_SYMBOL(blk_mq_queue_tag_inflight_iter); - static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth, bool round_robin, int node) { diff --git a/block/blk-mq.c b/block/blk-mq.c index b37721774895b..b37d0f6156f9d 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -112,7 +112,7 @@ void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part, struct mq_inflight mi = { .part = part, .inflight = inflight, };

inflight[0] = inflight[1] = 0; - blk_mq_queue_tag_inflight_iter(q, blk_mq_check_inflight, &mi); + blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); }

static void blk_mq_check_inflight_rw(struct blk_mq_hw_ctx *hctx, @@ -131,7 +131,7 @@ void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part, struct mq_inflight mi = { .part = part, .inflight = inflight, };

inflight[0] = inflight[1] = 0; - blk_mq_queue_tag_inflight_iter(q, blk_mq_check_inflight_rw, &mi); + blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight_rw, &mi); }

void blk_freeze_queue_start(struct request_queue *q) @@ -917,7 +917,7 @@ static void blk_mq_timeout_work(struct work_struct *work) if (!percpu_ref_tryget(&q->q_usage_counter)) return;

- blk_mq_queue_tag_inflight_iter(q, blk_mq_check_expired, &next); + blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);

if (next != 0) { mod_timer(&q->timeout, next); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index f81bbcdbce0f8..3695a43eb556a 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -325,8 +325,7 @@ void blk_freeze_queue_start(struct request_queue *q); void blk_mq_freeze_queue_wait(struct request_queue *q); int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, unsigned long timeout); -void blk_mq_queue_tag_inflight_iter(struct request_queue *q, busy_iter_fn *fn, - void *priv); + int blk_mq_map_queues(struct blk_mq_tag_set *set); void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);

From: Ming Lei ming.lei@redhat.com

mainline inclusion from mainline-v5.13 commit bd63141d585bef14f4caf111f6d0e27fe2300ec6 category: bugfix bugzilla: 34280, https://gitee.com/openeuler/kernel/issues/I4AKY4 CVE: NA

-----------------------------------------------

refcount_inc_not_zero() in bt_tags_iter() still may read one freed request.

Fix the issue by the following approach:

1) hold a per-tags spinlock when reading ->rqs[tag] and calling refcount_inc_not_zero in bt_tags_iter()

2) clearing stale request referred via ->rqs[tag] before freeing request pool, the per-tags spinlock is held for clearing stale ->rq[tag]

So after we cleared stale requests, bt_tags_iter() won't observe freed request any more, also the clearing will wait for pending request reference.

The idea of clearing ->rqs[] is borrowed from John Garry's previous patch and one recent David's patch.

Tested-by: John Garry john.garry@huawei.com Reviewed-by: David Jeffery djeffery@redhat.com Reviewed-by: Bart Van Assche bvanassche@acm.org Signed-off-by: Ming Lei ming.lei@redhat.com Link: https://lore.kernel.org/r/20210511152236.763464-4-ming.lei@redhat.com Signed-off-by: Jens Axboe axboe@kernel.dk Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Hou Tao houtao1@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- block/blk-mq-tag.c | 9 +++++++-- block/blk-mq-tag.h | 6 ++++++ block/blk-mq.c | 46 +++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index dcf65c9b767ba..04986f7d87406 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -222,10 +222,14 @@ struct bt_iter_data { static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags, unsigned int bitnr) { - struct request *rq = tags->rqs[bitnr]; + struct request *rq; + unsigned long flags;

+ spin_lock_irqsave(&tags->lock, flags); + rq = tags->rqs[bitnr]; if (!rq || !refcount_inc_not_zero(&rq->ref)) - return NULL; + rq = NULL; + spin_unlock_irqrestore(&tags->lock, flags); return rq; }

@@ -439,6 +443,7 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,

tags->nr_tags = total_tags; tags->nr_reserved_tags = reserved_tags; + spin_lock_init(&tags->lock);

return blk_mq_init_bitmap_tags(tags, node, alloc_policy); } diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index 61deab0b5a5a5..1a6cfb608f8aa 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h @@ -19,6 +19,12 @@ struct blk_mq_tags { struct request **rqs; struct request **static_rqs; struct list_head page_list; + + /* + * used to clear request reference in rqs[] before freeing one + * request pool + */ + spinlock_t lock; };

diff --git a/block/blk-mq.c b/block/blk-mq.c index 303c71e81146e..a0067147dd72d 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2024,6 +2024,45 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) return cookie; }

+static size_t order_to_size(unsigned int order) +{ + return (size_t)PAGE_SIZE << order; +} + +/* called before freeing request pool in @tags */ +static void blk_mq_clear_rq_mapping(struct blk_mq_tag_set *set, + struct blk_mq_tags *tags, unsigned int hctx_idx) +{ + struct blk_mq_tags *drv_tags = set->tags[hctx_idx]; + struct page *page; + unsigned long flags; + + list_for_each_entry(page, &tags->page_list, lru) { + unsigned long start = (unsigned long)page_address(page); + unsigned long end = start + order_to_size(page->private); + int i; + + for (i = 0; i < set->queue_depth; i++) { + struct request *rq = drv_tags->rqs[i]; + unsigned long rq_addr = (unsigned long)rq; + + if (rq_addr >= start && rq_addr < end) { + WARN_ON_ONCE(refcount_read(&rq->ref) != 0); + cmpxchg(&drv_tags->rqs[i], rq, NULL); + } + } + } + + /* + * Wait until all pending iteration is done. + * + * Request reference is cleared and it is guaranteed to be observed + * after the ->lock is released. + */ + spin_lock_irqsave(&drv_tags->lock, flags); + spin_unlock_irqrestore(&drv_tags->lock, flags); +} + void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx) { @@ -2042,6 +2081,8 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, } }

+ blk_mq_clear_rq_mapping(set, tags, hctx_idx); + while (!list_empty(&tags->page_list)) { page = list_first_entry(&tags->page_list, struct page, lru); list_del_init(&page->lru); @@ -2101,11 +2142,6 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, return tags; }

-static size_t order_to_size(unsigned int order) -{ - return (size_t)PAGE_SIZE << order; -} - static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, int node) {

From: Ming Lei ming.lei@redhat.com

mainline inclusion from mainline-v5.14-rc7 commit c797b40ccc340b8a66f7a7842aecc90bf749f087 category: bugfix bugzilla: 34280, https://gitee.com/openeuler/kernel/issues/I4AKY4 CVE: NA

-----------------------------------------------

Inside blk_mq_queue_tag_busy_iter() we already grabbed request's refcount before calling ->fn(), so needn't to grab it one more time in blk_mq_check_expired().

Meantime remove extra request expire check in blk_mq_check_expired().

Cc: Keith Busch kbusch@kernel.org Signed-off-by: Ming Lei ming.lei@redhat.com Reviewed-by: Christoph Hellwig hch@lst.de Reviewed-by: John Garry john.garry@huawei.com Link: https://lore.kernel.org/r/20210811155202.629575-1-ming.lei@redhat.com Signed-off-by: Jens Axboe axboe@kernel.dk Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Hou Tao houtao1@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- block/blk-mq.c | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-)

diff --git a/block/blk-mq.c b/block/blk-mq.c index d223e771eb2e0..ed1d9bd917c23 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -868,34 +868,14 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, unsigned long *next = priv;

/* - * Just do a quick check if it is expired before locking the request in - * so we're not unnecessarilly synchronizing across CPUs. - */ - if (!blk_mq_req_expired(rq, next)) - return; - - /* - * We have reason to believe the request may be expired. Take a - * reference on the request to lock this request lifetime into its - * currently allocated context to prevent it from being reallocated in - * the event the completion by-passes this timeout handler. - * - * If the reference was already released, then the driver beat the - * timeout handler to posting a natural completion. - */ - if (!refcount_inc_not_zero(&rq->ref)) - return; - - /* - * The request is now locked and cannot be reallocated underneath the - * timeout handler's processing. Re-verify this exact request is truly - * expired; if it is not expired, then the request was completed and - * reallocated as a new request. + * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot + * be reallocated underneath the timeout handler's processing, then + * the expire check is reliable. If the request is not expired, then + * it was completed and reallocated as a new request after returning + * from blk_mq_check_expired(). */ if (blk_mq_req_expired(rq, next)) blk_mq_rq_timed_out(rq, reserved); - - blk_mq_put_rq_ref(rq, hctx); }

static void blk_mq_timeout_work(struct work_struct *work)

From: Ming Lei ming.lei@redhat.com

mainline inclusion from mainline-v5.14-rc7 commit c2da19ed50554ce52ecbad3655c98371fe58599f category: bugfix bugzilla: 34280, https://gitee.com/openeuler/kernel/issues/I4AKY4 CVE: NA

-----------------------------------------------

For fixing use-after-free during iterating over requests, we grabbed request's refcount before calling ->fn in commit 2e315dc07df0 ("blk-mq: grab rq->refcount before calling ->fn in blk_mq_tagset_busy_iter"). Turns out this way may cause kernel panic when iterating over one flush request:

1) old flush request's tag is just released, and this tag is reused by one new request, but ->rqs[] isn't updated yet

2) the flush request can be re-used for submitting one new flush command, so blk_rq_init() is called at the same time

3) meantime blk_mq_queue_tag_busy_iter() is called, and old flush request is retrieved from ->rqs[tag]; when blk_mq_put_rq_ref() is called, flush_rq->end_io may not be updated yet, so NULL pointer dereference is triggered in blk_mq_put_rq_ref().

Fix the issue by calling refcount_set(&flush_rq->ref, 1) after flush_rq->end_io is set. So far the only other caller of blk_rq_init() is scsi_ioctl_reset() in which the request doesn't enter block IO stack and the request reference count isn't used, so the change is safe.

Fixes: 2e315dc07df0 ("blk-mq: grab rq->refcount before calling ->fn in blk_mq_tagset_busy_iter") Reported-by: "Blank-Burian, Markus, Dr." blankburian@uni-muenster.de Tested-by: "Blank-Burian, Markus, Dr." blankburian@uni-muenster.de Signed-off-by: Ming Lei ming.lei@redhat.com Reviewed-by: Christoph Hellwig hch@lst.de Reviewed-by: John Garry john.garry@huawei.com Link: https://lore.kernel.org/r/20210811142624.618598-1-ming.lei@redhat.com Signed-off-by: Jens Axboe axboe@kernel.dk

conflicts: use __blk_rq_init() instead of blk_rq_init(). Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Hou Tao houtao1@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- block/blk-flush.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/block/blk-flush.c b/block/blk-flush.c index c357e5c16d89c..c87a0ffba9c74 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -340,7 +340,7 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, */ fq->flush_pending_idx ^= 1;

- blk_rq_init(q, flush_rq); + __blk_rq_init(q, flush_rq);

/* * In case of none scheduler, borrow tag from the first request @@ -371,6 +371,15 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, flush_rq->rq_disk = first_rq->rq_disk; flush_rq->end_io = flush_end_io;

+ /* + * Order WRITE ->end_io and WRITE rq->ref, and its pair is the one + * implied in refcount_inc_not_zero() called from + * blk_mq_find_and_get_req(), which orders WRITE/READ flush_rq->ref + * and READ flush_rq->end_io + */ + smp_wmb(); + refcount_set(&flush_rq->ref, 1); + return blk_flush_queue_rq(flush_rq, false); }

From: Yu Kuai yukuai3@huawei.com

hulk inclusion category: bugfix bugzilla: 34280, https://gitee.com/openeuler/kernel/issues/I4AKY4 CVE: NA

-----------------------------------------------

Signed-off-by: Yu Kuai yukuai3@huawei.com Reviewed-by: Hou Tao houtao1@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- block/blk-mq-tag.c | 16 +++++++++------- block/blk-mq-tag.h | 10 ++++++++++ block/blk-mq.c | 8 ++++---- 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 5c740716c5e5a..bee92ab06a5e3 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -225,11 +225,11 @@ static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags, struct request *rq; unsigned long flags;

- spin_lock_irqsave(&tags->lock, flags); + blk_mq_tags_lock_irqsave(tags, flags); rq = tags->rqs[bitnr]; if (!rq || !refcount_inc_not_zero(&rq->ref)) rq = NULL; - spin_unlock_irqrestore(&tags->lock, flags); + blk_mq_tags_unlock_irqrestore(tags, flags); return rq; }

@@ -422,7 +422,7 @@ static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags, free_bitmap_tags: sbitmap_queue_free(&tags->bitmap_tags); free_tags: - kfree(tags); + kfree(blk_mq_tags_to_wrapper(tags)); return NULL; }

@@ -431,19 +431,21 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, int node, int alloc_policy) { struct blk_mq_tags *tags; + struct blk_mq_tags_wrapper *tags_wrapper;

if (total_tags > BLK_MQ_TAG_MAX) { pr_err("blk-mq: tag depth too large\n"); return NULL; }

- tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node); - if (!tags) + tags_wrapper = kzalloc_node(sizeof(*tags_wrapper), GFP_KERNEL, node); + if (!tags_wrapper) return NULL;

+ tags = &tags_wrapper->tags; tags->nr_tags = total_tags; tags->nr_reserved_tags = reserved_tags; - spin_lock_init(&tags->lock); + spin_lock_init(&tags_wrapper->lock);

return blk_mq_init_bitmap_tags(tags, node, alloc_policy); } @@ -452,7 +454,7 @@ void blk_mq_free_tags(struct blk_mq_tags *tags) { sbitmap_queue_free(&tags->bitmap_tags); sbitmap_queue_free(&tags->breserved_tags); - kfree(tags); + kfree(blk_mq_tags_to_wrapper(tags)); }

int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index 1a6cfb608f8aa..bc7a4c58c76f7 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h @@ -19,6 +19,10 @@ struct blk_mq_tags { struct request **rqs; struct request **static_rqs; struct list_head page_list; +}; + +struct blk_mq_tags_wrapper { + struct blk_mq_tags tags;

/* * used to clear request reference in rqs[] before freeing one @@ -27,6 +31,12 @@ struct blk_mq_tags { spinlock_t lock; };

+#define blk_mq_tags_to_wrapper(t) \ + container_of(t, struct blk_mq_tags_wrapper, tags) +#define blk_mq_tags_lock_irqsave(tags, flags) \ + spin_lock_irqsave(&blk_mq_tags_to_wrapper(tags)->lock, flags) +#define blk_mq_tags_unlock_irqrestore(tags, flags) \ + spin_unlock_irqrestore(&blk_mq_tags_to_wrapper(tags)->lock, flags)

extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node, int alloc_policy); extern void blk_mq_free_tags(struct blk_mq_tags *tags); diff --git a/block/blk-mq.c b/block/blk-mq.c index 6da37cbb975c9..7106c94ea58fe 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2039,8 +2039,8 @@ static void blk_mq_clear_rq_mapping(struct blk_mq_tag_set *set, * Request reference is cleared and it is guaranteed to be observed * after the ->lock is released. */ - spin_lock_irqsave(&drv_tags->lock, flags); - spin_unlock_irqrestore(&drv_tags->lock, flags); + blk_mq_tags_lock_irqsave(drv_tags, flags); + blk_mq_tags_unlock_irqrestore(drv_tags, flags); }

void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, @@ -2277,8 +2277,8 @@ static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags, * Request reference is cleared and it is guaranteed to be observed * after the ->lock is released. */ - spin_lock_irqsave(&tags->lock, flags); - spin_unlock_irqrestore(&tags->lock, flags); + blk_mq_tags_lock_irqsave(tags, flags); + blk_mq_tags_unlock_irqrestore(tags, flags); }

/* hctx->ctxs will be freed in queue's release handler */