Merge tag 'for-linus-2019-08-17' of git://git.kernel.dk/linux-block
authorLinus Torvalds <torvalds@linux-foundation.org>
Sun, 18 Aug 2019 02:39:54 +0000 (19:39 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Sun, 18 Aug 2019 02:39:54 +0000 (19:39 -0700)
Pull block fixes from Jens Axboe:
 "A collection of fixes that should go into this series. This contains:

   - Revert of the REQ_NOWAIT_INLINE and associated dio changes. There
     were still corner cases there, and even though I had a solution for
     it, it's too involved for this stage. (me)

   - Set of NVMe fixes (via Sagi)

   - io_uring fix for fixed buffers (Anthony)

   - io_uring defer issue fix (Jackie)

   - Regression fix for queue sync at exit time (zhengbin)

   - xen blk-back memory leak fix (Wenwen)"

* tag 'for-linus-2019-08-17' of git://git.kernel.dk/linux-block:
  io_uring: fix an issue when IOSQE_IO_LINK is inserted into defer list
  block: remove REQ_NOWAIT_INLINE
  io_uring: fix manual setup of iov_iter for fixed buffers
  xen/blkback: fix memory leaks
  blk-mq: move cancel of requeue_work to the front of blk_exit_queue
  nvme-pci: Fix async probe remove race
  nvme: fix controller removal race with scan work
  nvme-rdma: fix possible use-after-free in connect error flow
  nvme: fix a possible deadlock when passthru commands sent to a multipath device
  nvme-core: Fix extra device_put() call on error path
  nvmet-file: fix nvmet_file_flush() always returning an error
  nvmet-loop: Flush nvme_delete_wq when removing the port
  nvmet: Fix use-after-free bug when a port is removed
  nvme-multipath: revalidate nvme_ns_head gendisk in nvme_validate_ns

15 files changed:
block/blk-mq.c
block/blk-sysfs.c
drivers/block/xen-blkback/xenbus.c
drivers/nvme/host/core.c
drivers/nvme/host/multipath.c
drivers/nvme/host/nvme.h
drivers/nvme/host/pci.c
drivers/nvme/host/rdma.c
drivers/nvme/target/configfs.c
drivers/nvme/target/core.c
drivers/nvme/target/loop.c
drivers/nvme/target/nvmet.h
fs/block_dev.c
fs/io_uring.c
include/linux/blk_types.h

index f78d3287dd823f6b431c87eec675580c1e987d55..0835f4d8d42e7e34c043acc040d8f5406b2e2268 100644 (file)
@@ -1958,13 +1958,9 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
        rq = blk_mq_get_request(q, bio, &data);
        if (unlikely(!rq)) {
                rq_qos_cleanup(q, bio);
-
-               cookie = BLK_QC_T_NONE;
-               if (bio->bi_opf & REQ_NOWAIT_INLINE)
-                       cookie = BLK_QC_T_EAGAIN;
-               else if (bio->bi_opf & REQ_NOWAIT)
+               if (bio->bi_opf & REQ_NOWAIT)
                        bio_wouldblock_error(bio);
-               return cookie;
+               return BLK_QC_T_NONE;
        }
 
        trace_block_getrq(q, bio, bio->bi_opf);
@@ -2666,8 +2662,6 @@ void blk_mq_release(struct request_queue *q)
        struct blk_mq_hw_ctx *hctx, *next;
        int i;
 
-       cancel_delayed_work_sync(&q->requeue_work);
-
        queue_for_each_hw_ctx(q, hctx, i)
                WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
 
index 977c659dcd1846f193ee525b36cd1c6ccecb2684..9bfa3ea4ed63038606067f1bb9d2f0c957a90279 100644 (file)
@@ -892,6 +892,9 @@ static void __blk_release_queue(struct work_struct *work)
 
        blk_free_queue_stats(q->stats);
 
+       if (queue_is_mq(q))
+               cancel_delayed_work_sync(&q->requeue_work);
+
        blk_exit_queue(q);
 
        blk_queue_free_zone_bitmaps(q);
index 3ac6a5d18071729a7747a72835223f02234e240f..b90dbcd99c03e9c4e1ae0bc5f22180b80c5903e1 100644 (file)
@@ -965,6 +965,7 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir)
                }
        }
 
+       err = -ENOMEM;
        for (i = 0; i < nr_grefs * XEN_BLKIF_REQS_PER_PAGE; i++) {
                req = kzalloc(sizeof(*req), GFP_KERNEL);
                if (!req)
@@ -987,7 +988,7 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir)
        err = xen_blkif_map(ring, ring_ref, nr_grefs, evtchn);
        if (err) {
                xenbus_dev_fatal(dev, err, "mapping ring-ref port %u", evtchn);
-               return err;
+               goto fail;
        }
 
        return 0;
@@ -1007,8 +1008,7 @@ fail:
                }
                kfree(req);
        }
-       return -ENOMEM;
-
+       return err;
 }
 
 static int connect_ring(struct backend_info *be)
index 8f3fbe5ca9379988b16c88c6e25cb45869f6a614..c258a1ce4b28aaa3d4f021f30213e80132d2a076 100644 (file)
@@ -1286,6 +1286,9 @@ static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
         */
        if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
                mutex_lock(&ctrl->scan_lock);
+               mutex_lock(&ctrl->subsys->lock);
+               nvme_mpath_start_freeze(ctrl->subsys);
+               nvme_mpath_wait_freeze(ctrl->subsys);
                nvme_start_freeze(ctrl);
                nvme_wait_freeze(ctrl);
        }
@@ -1316,6 +1319,8 @@ static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
                nvme_update_formats(ctrl);
        if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
                nvme_unfreeze(ctrl);
+               nvme_mpath_unfreeze(ctrl->subsys);
+               mutex_unlock(&ctrl->subsys->lock);
                mutex_unlock(&ctrl->scan_lock);
        }
        if (effects & NVME_CMD_EFFECTS_CCC)
@@ -1715,6 +1720,7 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
        if (ns->head->disk) {
                nvme_update_disk_info(ns->head->disk, ns, id);
                blk_queue_stack_limits(ns->head->disk->queue, ns->queue);
+               revalidate_disk(ns->head->disk);
        }
 #endif
 }
@@ -2487,6 +2493,7 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
                if (ret) {
                        dev_err(ctrl->device,
                                "failed to register subsystem device.\n");
+                       put_device(&subsys->dev);
                        goto out_unlock;
                }
                ida_init(&subsys->ns_ida);
@@ -2509,7 +2516,6 @@ out_put_subsystem:
        nvme_put_subsystem(subsys);
 out_unlock:
        mutex_unlock(&nvme_subsystems_lock);
-       put_device(&subsys->dev);
        return ret;
 }
 
@@ -3571,6 +3577,13 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
        struct nvme_ns *ns, *next;
        LIST_HEAD(ns_list);
 
+       /*
+        * make sure to requeue I/O to all namespaces as these
+        * might result from the scan itself and must complete
+        * for the scan_work to make progress
+        */
+       nvme_mpath_clear_ctrl_paths(ctrl);
+
        /* prevent racing with ns scanning */
        flush_work(&ctrl->scan_work);
 
index 4f0d0d12744e0e94dbc1e518ca0c1935648c1bac..888d4543894e7d74ea9f75a462df1ace4e3b0cec 100644 (file)
@@ -12,6 +12,36 @@ module_param(multipath, bool, 0444);
 MODULE_PARM_DESC(multipath,
        "turn on native support for multiple controllers per subsystem");
 
+void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
+{
+       struct nvme_ns_head *h;
+
+       lockdep_assert_held(&subsys->lock);
+       list_for_each_entry(h, &subsys->nsheads, entry)
+               if (h->disk)
+                       blk_mq_unfreeze_queue(h->disk->queue);
+}
+
+void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
+{
+       struct nvme_ns_head *h;
+
+       lockdep_assert_held(&subsys->lock);
+       list_for_each_entry(h, &subsys->nsheads, entry)
+               if (h->disk)
+                       blk_mq_freeze_queue_wait(h->disk->queue);
+}
+
+void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
+{
+       struct nvme_ns_head *h;
+
+       lockdep_assert_held(&subsys->lock);
+       list_for_each_entry(h, &subsys->nsheads, entry)
+               if (h->disk)
+                       blk_freeze_queue_start(h->disk->queue);
+}
+
 /*
  * If multipathing is enabled we need to always use the subsystem instance
  * number for numbering our devices to avoid conflicts between subsystems that
@@ -104,18 +134,34 @@ static const char *nvme_ana_state_names[] = {
        [NVME_ANA_CHANGE]               = "change",
 };
 
-void nvme_mpath_clear_current_path(struct nvme_ns *ns)
+bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 {
        struct nvme_ns_head *head = ns->head;
+       bool changed = false;
        int node;
 
        if (!head)
-               return;
+               goto out;
 
        for_each_node(node) {
-               if (ns == rcu_access_pointer(head->current_path[node]))
+               if (ns == rcu_access_pointer(head->current_path[node])) {
                        rcu_assign_pointer(head->current_path[node], NULL);
+                       changed = true;
+               }
        }
+out:
+       return changed;
+}
+
+void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
+{
+       struct nvme_ns *ns;
+
+       mutex_lock(&ctrl->scan_lock);
+       list_for_each_entry(ns, &ctrl->namespaces, list)
+               if (nvme_mpath_clear_current_path(ns))
+                       kblockd_schedule_work(&ns->head->requeue_work);
+       mutex_unlock(&ctrl->scan_lock);
 }
 
 static bool nvme_path_is_disabled(struct nvme_ns *ns)
@@ -226,6 +272,24 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
        return ns;
 }
 
+static bool nvme_available_path(struct nvme_ns_head *head)
+{
+       struct nvme_ns *ns;
+
+       list_for_each_entry_rcu(ns, &head->list, siblings) {
+               switch (ns->ctrl->state) {
+               case NVME_CTRL_LIVE:
+               case NVME_CTRL_RESETTING:
+               case NVME_CTRL_CONNECTING:
+                       /* fallthru */
+                       return true;
+               default:
+                       break;
+               }
+       }
+       return false;
+}
+
 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
                struct bio *bio)
 {
@@ -252,14 +316,14 @@ static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
                                      disk_devt(ns->head->disk),
                                      bio->bi_iter.bi_sector);
                ret = direct_make_request(bio);
-       } else if (!list_empty_careful(&head->list)) {
-               dev_warn_ratelimited(dev, "no path available - requeuing I/O\n");
+       } else if (nvme_available_path(head)) {
+               dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
 
                spin_lock_irq(&head->requeue_lock);
                bio_list_add(&head->requeue_list, bio);
                spin_unlock_irq(&head->requeue_lock);
        } else {
-               dev_warn_ratelimited(dev, "no path - failing I/O\n");
+               dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 
                bio->bi_status = BLK_STS_IOERR;
                bio_endio(bio);
index 26b563f9985b5a9bf4163a18646e9fee68a2ae17..778b3a0b6adb1f9866c4d7d1b0e2b5c08e277ea9 100644 (file)
@@ -490,6 +490,9 @@ static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
        return ctrl->ana_log_buf != NULL;
 }
 
+void nvme_mpath_unfreeze(struct nvme_subsystem *subsys);
+void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys);
+void nvme_mpath_start_freeze(struct nvme_subsystem *subsys);
 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
                        struct nvme_ctrl *ctrl, int *flags);
 void nvme_failover_req(struct request *req);
@@ -500,7 +503,8 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head);
 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
 void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
 void nvme_mpath_stop(struct nvme_ctrl *ctrl);
-void nvme_mpath_clear_current_path(struct nvme_ns *ns);
+bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
+void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
 struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);
 
 static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
@@ -548,7 +552,11 @@ static inline void nvme_mpath_add_disk(struct nvme_ns *ns,
 static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 {
 }
-static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
+static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
+{
+       return false;
+}
+static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 {
 }
 static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
@@ -568,6 +576,15 @@ static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 {
 }
+static inline void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
+{
+}
+static inline void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
+{
+}
+static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
+{
+}
 #endif /* CONFIG_NVME_MULTIPATH */
 
 #ifdef CONFIG_NVM
index 108e109e99f1aceddce3c535c3e25079752204bf..6bd9b10339654b3d11aa3944f7a51ee264ffb8a2 100644 (file)
@@ -2695,7 +2695,7 @@ static void nvme_async_probe(void *data, async_cookie_t cookie)
 {
        struct nvme_dev *dev = data;
 
-       nvme_reset_ctrl_sync(&dev->ctrl);
+       flush_work(&dev->ctrl.reset_work);
        flush_work(&dev->ctrl.scan_work);
        nvme_put_ctrl(&dev->ctrl);
 }
@@ -2761,6 +2761,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
        dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
 
+       nvme_reset_ctrl(&dev->ctrl);
        nvme_get_ctrl(&dev->ctrl);
        async_schedule(nvme_async_probe, dev);
 
index a249db528d543dcad37e9b5162cc4b196a2f41e1..1a6449bc547b91ab7460b817fd6aada3ca5b5acd 100644 (file)
@@ -562,13 +562,17 @@ out_destroy_cm_id:
        return ret;
 }
 
+static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
+{
+       rdma_disconnect(queue->cm_id);
+       ib_drain_qp(queue->qp);
+}
+
 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
 {
        if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
                return;
-
-       rdma_disconnect(queue->cm_id);
-       ib_drain_qp(queue->qp);
+       __nvme_rdma_stop_queue(queue);
 }
 
 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
@@ -607,11 +611,13 @@ static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
        else
                ret = nvmf_connect_admin_queue(&ctrl->ctrl);
 
-       if (!ret)
+       if (!ret) {
                set_bit(NVME_RDMA_Q_LIVE, &queue->flags);
-       else
+       } else {
+               __nvme_rdma_stop_queue(queue);
                dev_info(ctrl->ctrl.device,
                        "failed to connect queue: %d ret=%d\n", idx, ret);
+       }
        return ret;
 }
 
index cd52b9f153766602036eb9e7322f845551fff855..98613a45bd3b408e343b8e89f9605475f1cb47b9 100644 (file)
@@ -675,6 +675,7 @@ static void nvmet_port_subsys_drop_link(struct config_item *parent,
 
 found:
        list_del(&p->entry);
+       nvmet_port_del_ctrls(port, subsys);
        nvmet_port_disc_changed(port, subsys);
 
        if (list_empty(&port->subsystems))
index dad0243c7c96cfd5f84d5e996b09dcc5fa78d07b..3a67e244e5685a35f65e24e322f85b9557bba37d 100644 (file)
@@ -46,6 +46,9 @@ inline u16 errno_to_nvme_status(struct nvmet_req *req, int errno)
        u16 status;
 
        switch (errno) {
+       case 0:
+               status = NVME_SC_SUCCESS;
+               break;
        case -ENOSPC:
                req->error_loc = offsetof(struct nvme_rw_command, length);
                status = NVME_SC_CAP_EXCEEDED | NVME_SC_DNR;
@@ -280,6 +283,18 @@ void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops)
 }
 EXPORT_SYMBOL_GPL(nvmet_unregister_transport);
 
+void nvmet_port_del_ctrls(struct nvmet_port *port, struct nvmet_subsys *subsys)
+{
+       struct nvmet_ctrl *ctrl;
+
+       mutex_lock(&subsys->lock);
+       list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+               if (ctrl->port == port)
+                       ctrl->ops->delete_ctrl(ctrl);
+       }
+       mutex_unlock(&subsys->lock);
+}
+
 int nvmet_enable_port(struct nvmet_port *port)
 {
        const struct nvmet_fabrics_ops *ops;
index b16dc3981c698749a1a5f85e7066e68c946dfed4..0940c5024a3456457a732c6ae97195e346fd097d 100644 (file)
@@ -654,6 +654,14 @@ static void nvme_loop_remove_port(struct nvmet_port *port)
        mutex_lock(&nvme_loop_ports_mutex);
        list_del_init(&port->entry);
        mutex_unlock(&nvme_loop_ports_mutex);
+
+       /*
+        * Ensure any ctrls that are in the process of being
+        * deleted are in fact deleted before we return
+        * and free the port. This is to prevent active
+        * ctrls from using a port after it's freed.
+        */
+       flush_workqueue(nvme_delete_wq);
 }
 
 static const struct nvmet_fabrics_ops nvme_loop_ops = {
index 6ee66c6107391899898f812a4b4347b284d6c333..c51f8dd01dc487a1395e3a3ff5350cf260046118 100644 (file)
@@ -418,6 +418,9 @@ void nvmet_port_send_ana_event(struct nvmet_port *port);
 int nvmet_register_transport(const struct nvmet_fabrics_ops *ops);
 void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops);
 
+void nvmet_port_del_ctrls(struct nvmet_port *port,
+                         struct nvmet_subsys *subsys);
+
 int nvmet_enable_port(struct nvmet_port *port);
 void nvmet_disable_port(struct nvmet_port *port);
 
index eb657ab9406062cdd23136e66fc265e472523a80..677cb364d33f0e43e65d25349fdcdce651a6ca5e 100644 (file)
@@ -345,24 +345,15 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
        struct bio *bio;
        bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;
        bool is_read = (iov_iter_rw(iter) == READ), is_sync;
-       bool nowait = (iocb->ki_flags & IOCB_NOWAIT) != 0;
        loff_t pos = iocb->ki_pos;
        blk_qc_t qc = BLK_QC_T_NONE;
-       gfp_t gfp;
-       int ret;
+       int ret = 0;
 
        if ((pos | iov_iter_alignment(iter)) &
            (bdev_logical_block_size(bdev) - 1))
                return -EINVAL;
 
-       if (nowait)
-               gfp = GFP_NOWAIT;
-       else
-               gfp = GFP_KERNEL;
-
-       bio = bio_alloc_bioset(gfp, nr_pages, &blkdev_dio_pool);
-       if (!bio)
-               return -EAGAIN;
+       bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool);
 
        dio = container_of(bio, struct blkdev_dio, bio);
        dio->is_sync = is_sync = is_sync_kiocb(iocb);
@@ -384,7 +375,6 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
        if (!is_poll)
                blk_start_plug(&plug);
 
-       ret = 0;
        for (;;) {
                bio_set_dev(bio, bdev);
                bio->bi_iter.bi_sector = pos >> 9;
@@ -409,14 +399,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
                        task_io_account_write(bio->bi_iter.bi_size);
                }
 
-               /*
-                * Tell underlying layer to not block for resource shortage.
-                * And if we would have blocked, return error inline instead
-                * of through the bio->bi_end_io() callback.
-                */
-               if (nowait)
-                       bio->bi_opf |= (REQ_NOWAIT | REQ_NOWAIT_INLINE);
-
+               dio->size += bio->bi_iter.bi_size;
                pos += bio->bi_iter.bi_size;
 
                nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
@@ -428,13 +411,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
                                polled = true;
                        }
 
-                       dio->size += bio->bi_iter.bi_size;
                        qc = submit_bio(bio);
-                       if (qc == BLK_QC_T_EAGAIN) {
-                               dio->size -= bio->bi_iter.bi_size;
-                               ret = -EAGAIN;
-                               goto error;
-                       }
 
                        if (polled)
                                WRITE_ONCE(iocb->ki_cookie, qc);
@@ -455,19 +432,8 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
                        atomic_inc(&dio->ref);
                }
 
-               dio->size += bio->bi_iter.bi_size;
-               qc = submit_bio(bio);
-               if (qc == BLK_QC_T_EAGAIN) {
-                       dio->size -= bio->bi_iter.bi_size;
-                       ret = -EAGAIN;
-                       goto error;
-               }
-
-               bio = bio_alloc(gfp, nr_pages);
-               if (!bio) {
-                       ret = -EAGAIN;
-                       goto error;
-               }
+               submit_bio(bio);
+               bio = bio_alloc(GFP_KERNEL, nr_pages);
        }
 
        if (!is_poll)
@@ -487,7 +453,6 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
        }
        __set_current_state(TASK_RUNNING);
 
-out:
        if (!ret)
                ret = blk_status_to_errno(dio->bio.bi_status);
        if (likely(!ret))
@@ -495,10 +460,6 @@ out:
 
        bio_put(&dio->bio);
        return ret;
-error:
-       if (!is_poll)
-               blk_finish_plug(&plug);
-       goto out;
 }
 
 static ssize_t
index d542f1cf4428ed79af62c76273d32885490f72ff..24bbe3cb7ad45e72b1cfca18c3084795d0be6a8c 100644 (file)
@@ -1097,10 +1097,8 @@ static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
 
                        iter->bvec = bvec + seg_skip;
                        iter->nr_segs -= seg_skip;
-                       iter->count -= (seg_skip << PAGE_SHIFT);
+                       iter->count -= bvec->bv_len + offset;
                        iter->iov_offset = offset & ~PAGE_MASK;
-                       if (iter->iov_offset)
-                               iter->count -= iter->iov_offset;
                }
        }
 
@@ -2025,6 +2023,15 @@ static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 {
        int ret;
 
+       ret = io_req_defer(ctx, req, s->sqe);
+       if (ret) {
+               if (ret != -EIOCBQUEUED) {
+                       io_free_req(req);
+                       io_cqring_add_event(ctx, s->sqe->user_data, ret);
+               }
+               return 0;
+       }
+
        ret = __io_submit_sqe(ctx, req, s, true);
        if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
                struct io_uring_sqe *sqe_copy;
@@ -2097,13 +2104,6 @@ err:
                return;
        }
 
-       ret = io_req_defer(ctx, req, s->sqe);
-       if (ret) {
-               if (ret != -EIOCBQUEUED)
-                       goto err_req;
-               return;
-       }
-
        /*
         * If we already have a head request, queue this one for async
         * submittal once the head completes. If we don't have a head but
index 1b1fa1557e68aefe5cf5df252485d398b8a38e35..feff3fe4467ec97ce92aa7e6b5b13ef05012bb47 100644 (file)
@@ -311,7 +311,6 @@ enum req_flag_bits {
        __REQ_RAHEAD,           /* read ahead, can fail anytime */
        __REQ_BACKGROUND,       /* background IO */
        __REQ_NOWAIT,           /* Don't wait if request will block */
-       __REQ_NOWAIT_INLINE,    /* Return would-block error inline */
        /*
         * When a shared kthread needs to issue a bio for a cgroup, doing
         * so synchronously can lead to priority inversions as the kthread
@@ -346,7 +345,6 @@ enum req_flag_bits {
 #define REQ_RAHEAD             (1ULL << __REQ_RAHEAD)
 #define REQ_BACKGROUND         (1ULL << __REQ_BACKGROUND)
 #define REQ_NOWAIT             (1ULL << __REQ_NOWAIT)
-#define REQ_NOWAIT_INLINE      (1ULL << __REQ_NOWAIT_INLINE)
 #define REQ_CGROUP_PUNT                (1ULL << __REQ_CGROUP_PUNT)
 
 #define REQ_NOUNMAP            (1ULL << __REQ_NOUNMAP)
@@ -420,13 +418,12 @@ static inline int op_stat_group(unsigned int op)
 
 typedef unsigned int blk_qc_t;
 #define BLK_QC_T_NONE          -1U
-#define BLK_QC_T_EAGAIN                -2U
 #define BLK_QC_T_SHIFT         16
 #define BLK_QC_T_INTERNAL      (1U << 31)
 
 static inline bool blk_qc_t_valid(blk_qc_t cookie)
 {
-       return cookie != BLK_QC_T_NONE && cookie != BLK_QC_T_EAGAIN;
+       return cookie != BLK_QC_T_NONE;
 }
 
 static inline unsigned int blk_qc_t_to_queue_num(blk_qc_t cookie)