aio: bad AIO race in aio_complete() leads to process hang

[sfrench/cifs-2.6.git] / fs / aio.c

diff --git a/fs/aio.c b/fs/aio.c
index ea2e198203818a4703f8dc56a6d8473cf6ada21a..6af92194062266d842adfd57cf8abff425bd6ec5 100644 (file)
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -303,7 +303,7 @@ static void wait_for_all_aios(struct kioctx *ctx)
        set_task_state(tsk, TASK_UNINTERRUPTIBLE);
        while (ctx->reqs_active) {
                spin_unlock_irq(&ctx->ctx_lock);
-               schedule();
+               io_schedule();
                set_task_state(tsk, TASK_UNINTERRUPTIBLE);
                spin_lock_irq(&ctx->ctx_lock);
        }
@@ -317,13 +317,13 @@ out:
 /* wait_on_sync_kiocb:
  *     Waits on the given sync kiocb to complete.
  */
-ssize_t fastcall wait_on_sync_kiocb(struct kiocb *iocb)
+ssize_t wait_on_sync_kiocb(struct kiocb *iocb)
 {
        while (iocb->ki_users) {
                set_current_state(TASK_UNINTERRUPTIBLE);
                if (!iocb->ki_users)
                        break;
-               schedule();
+               io_schedule();
        }
        __set_current_state(TASK_RUNNING);
        return iocb->ki_user_data;
@@ -336,7 +336,7 @@ ssize_t fastcall wait_on_sync_kiocb(struct kiocb *iocb)
  * go away, they will call put_ioctx and release any pinned memory
  * associated with the request (held via struct page * references).
  */
-void fastcall exit_aio(struct mm_struct *mm)
+void exit_aio(struct mm_struct *mm)
 {
        struct kioctx *ctx = mm->ioctx_list;
        mm->ioctx_list = NULL;
@@ -365,7 +365,7 @@ void fastcall exit_aio(struct mm_struct *mm)
  *     Called when the last user of an aio context has gone away,
  *     and the struct needs to be freed.
  */
-void fastcall __put_ioctx(struct kioctx *ctx)
+void __put_ioctx(struct kioctx *ctx)
 {
        unsigned nr_events = ctx->max_reqs;
 
@@ -397,8 +397,7 @@ void fastcall __put_ioctx(struct kioctx *ctx)
  * This prevents races between the aio code path referencing the
  * req (after submitting it) and aio_complete() freeing the req.
  */
-static struct kiocb *FASTCALL(__aio_get_req(struct kioctx *ctx));
-static struct kiocb fastcall *__aio_get_req(struct kioctx *ctx)
+static struct kiocb *__aio_get_req(struct kioctx *ctx)
 {
        struct kiocb *req = NULL;
        struct aio_ring *ring;
@@ -533,7 +532,7 @@ static int __aio_put_req(struct kioctx *ctx, struct kiocb *req)
  *     Returns true if this put was the last user of the kiocb,
  *     false if the request is still in use.
  */
-int fastcall aio_put_req(struct kiocb *req)
+int aio_put_req(struct kiocb *req)
 {
        struct kioctx *ctx = req->ki_ctx;
        int ret;
@@ -710,18 +709,9 @@ static ssize_t aio_run_iocb(struct kiocb *iocb)
 
        /*
         * Now we are all set to call the retry method in async
-        * context. By setting this thread's io_wait context
-        * to point to the wait queue entry inside the currently
-        * running iocb for the duration of the retry, we ensure
-        * that async notification wakeups are queued by the
-        * operation instead of blocking waits, and when notified,
-        * cause the iocb to be kicked for continuation (through
-        * the aio_wake_function callback).
+        * context.
         */
-       BUG_ON(current->io_wait != NULL);
-       current->io_wait = &iocb->ki_wait;
        ret = retry(iocb);
-       current->io_wait = NULL;
 
        if (ret != -EIOCBRETRY && ret != -EIOCBQUEUED) {
                BUG_ON(!list_empty(&iocb->ki_wait.task_list));
@@ -902,7 +892,7 @@ static void try_queue_kicked_iocb(struct kiocb *iocb)
  *      The retry is usually executed by aio workqueue
  *      threads (See aio_kick_handler).
  */
-void fastcall kick_iocb(struct kiocb *iocb)
+void kick_iocb(struct kiocb *iocb)
 {
        /* sync iocbs are easy: they can only ever be executing from a 
         * single context. */
@@ -921,7 +911,7 @@ EXPORT_SYMBOL(kick_iocb);
  *     Returns true if this is the last user of the request.  The 
  *     only other user of the request can be the cancellation code.
  */
-int fastcall aio_complete(struct kiocb *iocb, long res, long res2)
+int aio_complete(struct kiocb *iocb, long res, long res2)
 {
        struct kioctx   *ctx = iocb->ki_ctx;
        struct aio_ring_info    *info;
@@ -1006,6 +996,14 @@ put_rq:
        /* everything turned out well, dispose of the aiocb. */
        ret = __aio_put_req(ctx, iocb);
 
+       /*
+        * We have to order our ring_info tail store above and test
+        * of the wait list below outside the wait lock.  This is
+        * like in wake_up_bit() where clearing a bit has to be
+        * ordered with the unlocked test.
+        */
+       smp_mb();
+
        if (waitqueue_active(&ctx->wait))
                wake_up(&ctx->wait);
 
@@ -1170,7 +1168,12 @@ retry:
                        ret = 0;
                        if (to.timed_out)       /* Only check after read evt */
                                break;
-                       schedule();
+                       /* Try to only show up in io wait if there are ops
+                        *  in flight */
+                       if (ctx->reqs_active)
+                               io_schedule();
+                       else
+                               schedule();
                        if (signal_pending(tsk)) {
                                ret = -EINTR;
                                break;
@@ -1334,6 +1337,10 @@ static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
                opcode = IOCB_CMD_PWRITEV;
        }
 
+       /* This matches the pread()/pwrite() logic */
+       if (iocb->ki_pos < 0)
+               return -EINVAL;
+
        do {
                ret = rw_op(iocb, &iocb->ki_iovec[iocb->ki_cur_seg],
                            iocb->ki_nr_segs - iocb->ki_cur_seg,
@@ -1352,6 +1359,13 @@ static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
        if ((ret == 0) || (iocb->ki_left == 0))
                ret = iocb->ki_nbytes - iocb->ki_left;
 
+       /* If we managed to write some out we return that, rather than
+        * the eventual error. */
+       if (opcode == IOCB_CMD_PWRITEV
+           && ret < 0 && ret != -EIOCBQUEUED && ret != -EIOCBRETRY
+           && iocb->ki_nbytes - iocb->ki_left)
+               ret = iocb->ki_nbytes - iocb->ki_left;
+
        return ret;
 }
 
@@ -1508,10 +1522,7 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb)
  *     Simply triggers a retry of the operation via kick_iocb.
  *
  *     This callback is specified in the wait queue entry in
- *     a kiocb (current->io_wait points to this wait queue
- *     entry when an aio operation executes; it is used
- *     instead of a synchronous wait when an i/o blocking
- *     condition is encountered during aio).
+ *     a kiocb.
  *
  * Note:
  * This routine is executed with the wait queue lock held.
@@ -1530,7 +1541,7 @@ static int aio_wake_function(wait_queue_t *wait, unsigned mode,
        return 1;
 }
 
-int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
+int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
                         struct iocb *iocb)
 {
        struct kiocb *req;