io_uring: remove req init from io_get_req()

io_get_req() do two different things: io_kiocb allocation and
initialisation. Move init part out of it and rename into
io_alloc_req(). It's simpler this way and also have better data
locality.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 845c173d92820d01ce8bbe53a51c2c089d6876ca..7f9bf8b6e6afe4d58faef9b32062467ecb1775ca 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1293,8 +1293,8 @@ static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
        return NULL;
 }
 
-static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
-                                  struct io_submit_state *state)
+static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx,
+                                    struct io_submit_state *state)
 {
        gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
        struct io_kiocb *req;
@@ -1327,22 +1327,9 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
                req = state->reqs[state->free_reqs];
        }
 
-got_it:
-       req->io = NULL;
-       req->file = NULL;
-       req->ctx = ctx;
-       req->flags = 0;
-       /* one is dropped after submission, the other at completion */
-       refcount_set(&req->refs, 2);
-       req->task = NULL;
-       req->result = 0;
-       INIT_IO_WORK(&req->work, io_wq_submit_work);
        return req;
 fallback:
-       req = io_get_fallback_req(ctx);
-       if (req)
-               goto got_it;
-       return NULL;
+       return io_get_fallback_req(ctx);
 }
 
 static inline void io_put_file(struct io_kiocb *req, struct file *file,
@@ -5804,6 +5791,28 @@ static inline void io_consume_sqe(struct io_ring_ctx *ctx)
        ctx->cached_sq_head++;
 }
 
+static void io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
+                       const struct io_uring_sqe *sqe)
+{
+       /*
+        * All io need record the previous position, if LINK vs DARIN,
+        * it can be used to mark the position of the first IO in the
+        * link list.
+        */
+       req->sequence = ctx->cached_sq_head;
+       req->opcode = READ_ONCE(sqe->opcode);
+       req->user_data = READ_ONCE(sqe->user_data);
+       req->io = NULL;
+       req->file = NULL;
+       req->ctx = ctx;
+       req->flags = 0;
+       /* one is dropped after submission, the other at completion */
+       refcount_set(&req->refs, 2);
+       req->task = NULL;
+       req->result = 0;
+       INIT_IO_WORK(&req->work, io_wq_submit_work);
+}
+
 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
                          struct file *ring_file, int ring_fd,
                          struct mm_struct **mm, bool async)
@@ -5844,23 +5853,15 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
                        io_consume_sqe(ctx);
                        break;
                }
-               req = io_get_req(ctx, statep);
+               req = io_alloc_req(ctx, statep);
                if (unlikely(!req)) {
                        if (!submitted)
                                submitted = -EAGAIN;
                        break;
                }
 
-               /*
-                * All io need record the previous position, if LINK vs DARIN,
-                * it can be used to mark the position of the first IO in the
-                * link list.
-                */
-               req->sequence = ctx->cached_sq_head;
-               req->opcode = READ_ONCE(sqe->opcode);
-               req->user_data = READ_ONCE(sqe->user_data);
+               io_init_req(ctx, req, sqe);
                io_consume_sqe(ctx);
-
                /* will complete beyond this point, count as submitted */
                submitted++;