block: avoid extra bio reference for async O_DIRECT

[sfrench/cifs-2.6.git] / fs / block_dev.c

diff --git a/fs/block_dev.c b/fs/block_dev.c
index a80b4f0ee7c4f172d19b4df1c8c330e4c0738557..e1886cc7048fbcea35fff0591b3679aa059d8a07 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -181,7 +181,7 @@ static void blkdev_bio_end_io_simple(struct bio *bio)
        struct task_struct *waiter = bio->bi_private;
 
        WRITE_ONCE(bio->bi_private, NULL);
-       wake_up_process(waiter);
+       blk_wake_io_task(waiter);
 }
 
 static ssize_t
@@ -232,14 +232,18 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
                bio.bi_opf = dio_bio_write_op(iocb);
                task_io_account_write(ret);
        }
+       if (iocb->ki_flags & IOCB_HIPRI)
+               bio.bi_opf |= REQ_HIPRI;
 
        qc = submit_bio(&bio);
        for (;;) {
-               set_current_state(TASK_UNINTERRUPTIBLE);
+               __set_current_state(TASK_UNINTERRUPTIBLE);
+
                if (!READ_ONCE(bio.bi_private))
                        break;
+
                if (!(iocb->ki_flags & IOCB_HIPRI) ||
-                   !blk_poll(bdev_get_queue(bdev), qc))
+                   !blk_poll(bdev_get_queue(bdev), qc, true))
                        io_schedule();
        }
        __set_current_state(TASK_RUNNING);
@@ -298,12 +302,13 @@ static void blkdev_bio_end_io(struct bio *bio)
                        }
 
                        dio->iocb->ki_complete(iocb, ret, 0);
-                       bio_put(&dio->bio);
+                       if (dio->multi_bio)
+                               bio_put(&dio->bio);
                } else {
                        struct task_struct *waiter = dio->waiter;
 
                        WRITE_ONCE(dio->waiter, NULL);
-                       wake_up_process(waiter);
+                       blk_wake_io_task(waiter);
                }
        }
 
@@ -328,6 +333,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
        struct blk_plug plug;
        struct blkdev_dio *dio;
        struct bio *bio;
+       bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;
        bool is_read = (iov_iter_rw(iter) == READ), is_sync;
        loff_t pos = iocb->ki_pos;
        blk_qc_t qc = BLK_QC_T_NONE;
@@ -338,20 +344,27 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
                return -EINVAL;
 
        bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool);
-       bio_get(bio); /* extra ref for the completion handler */
 
        dio = container_of(bio, struct blkdev_dio, bio);
        dio->is_sync = is_sync = is_sync_kiocb(iocb);
-       if (dio->is_sync)
+       if (dio->is_sync) {
                dio->waiter = current;
-       else
+               bio_get(bio);
+       } else {
                dio->iocb = iocb;
+       }
 
        dio->size = 0;
        dio->multi_bio = false;
        dio->should_dirty = is_read && iter_is_iovec(iter);
 
-       blk_start_plug(&plug);
+       /*
+        * Don't plug for HIPRI/polled IO, as those should go straight
+        * to issue
+        */
+       if (!is_poll)
+               blk_start_plug(&plug);
+
        for (;;) {
                bio_set_dev(bio, bdev);
                bio->bi_iter.bi_sector = pos >> 9;
@@ -381,11 +394,21 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
 
                nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
                if (!nr_pages) {
+                       if (iocb->ki_flags & IOCB_HIPRI)
+                               bio->bi_opf |= REQ_HIPRI;
+
                        qc = submit_bio(bio);
                        break;
                }
 
                if (!dio->multi_bio) {
+                       /*
+                        * AIO needs an extra reference to ensure the dio
+                        * structure which is embedded into the first bio
+                        * stays around.
+                        */
+                       if (!is_sync)
+                               bio_get(bio);
                        dio->multi_bio = true;
                        atomic_set(&dio->ref, 2);
                } else {
@@ -395,18 +418,21 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
                submit_bio(bio);
                bio = bio_alloc(GFP_KERNEL, nr_pages);
        }
-       blk_finish_plug(&plug);
+
+       if (!is_poll)
+               blk_finish_plug(&plug);
 
        if (!is_sync)
                return -EIOCBQUEUED;
 
        for (;;) {
-               set_current_state(TASK_UNINTERRUPTIBLE);
+               __set_current_state(TASK_UNINTERRUPTIBLE);
+
                if (!READ_ONCE(dio->waiter))
                        break;
 
                if (!(iocb->ki_flags & IOCB_HIPRI) ||
-                   !blk_poll(bdev_get_queue(bdev), qc))
+                   !blk_poll(bdev_get_queue(bdev), qc, true))
                        io_schedule();
        }
        __set_current_state(TASK_RUNNING);