Merge tag 'for-6.6/block-2023-08-28' of git://git.kernel.dk/linux

[sfrench/cifs-2.6.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 84400157c5f4e75e046c4846b12889e60b755cae..ec922c6bccbe20737d21e03a4d4964c8dab1d66f 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -682,6 +682,21 @@ out_queue_exit:
 }
 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
 
+static void blk_mq_finish_request(struct request *rq)
+{
+       struct request_queue *q = rq->q;
+
+       if (rq->rq_flags & RQF_USE_SCHED) {
+               q->elevator->type->ops.finish_request(rq);
+               /*
+                * For postflush request that may need to be
+                * completed twice, we should clear this flag
+                * to avoid double finish_request() on the rq.
+                */
+               rq->rq_flags &= ~RQF_USE_SCHED;
+       }
+}
+
 static void __blk_mq_free_request(struct request *rq)
 {
        struct request_queue *q = rq->q;
@@ -708,9 +723,7 @@ void blk_mq_free_request(struct request *rq)
 {
        struct request_queue *q = rq->q;
 
-       if ((rq->rq_flags & RQF_USE_SCHED) &&
-           q->elevator->type->ops.finish_request)
-               q->elevator->type->ops.finish_request(rq);
+       blk_mq_finish_request(rq);
 
        if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
                laptop_io_completion(q->disk->bdi);
@@ -1021,6 +1034,8 @@ inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
        if (blk_mq_need_time_stamp(rq))
                __blk_mq_end_request_acct(rq, ktime_get_ns());
 
+       blk_mq_finish_request(rq);
+
        if (rq->end_io) {
                rq_qos_done(rq->q, rq);
                if (rq->end_io(rq, error) == RQ_END_IO_FREE)
@@ -1075,6 +1090,8 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob)
                if (iob->need_ts)
                        __blk_mq_end_request_acct(rq, now);
 
+               blk_mq_finish_request(rq);
+
                rq_qos_done(rq->q, rq);
 
                /*
@@ -2755,7 +2772,14 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 {
        struct request *rq;
 
-       if (rq_list_empty(plug->mq_list))
+       /*
+        * We may have been called recursively midway through handling
+        * plug->mq_list via a schedule() in the driver's queue_rq() callback.
+        * To avoid mq_list changing under our feet, clear rq_count early and
+        * bail out specifically if rq_count is 0 rather than checking
+        * whether the mq_list is empty.
+        */
+       if (plug->rq_count == 0)
                return;
        plug->rq_count = 0;