block: deactivate blk_stat timer in wbt_disable_default()

[sfrench/cifs-2.6.git] / block / blk-wbt.c

diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index 8ac93fcbaa2eaaf680cebac8f3d8da8f9a25805d..463e4eb80287df806c2a09005a8a9aac19e44aa0 100644 (file)
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -489,31 +489,21 @@ static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
 }
 
 struct wbt_wait_data {
-       struct wait_queue_entry wq;
-       struct task_struct *task;
        struct rq_wb *rwb;
-       struct rq_wait *rqw;
+       enum wbt_flags wb_acct;
        unsigned long rw;
-       bool got_token;
 };
 
-static int wbt_wake_function(struct wait_queue_entry *curr, unsigned int mode,
-                            int wake_flags, void *key)
+static bool wbt_inflight_cb(struct rq_wait *rqw, void *private_data)
 {
-       struct wbt_wait_data *data = container_of(curr, struct wbt_wait_data,
-                                                       wq);
-
-       /*
-        * If we fail to get a budget, return -1 to interrupt the wake up
-        * loop in __wake_up_common.
-        */
-       if (!rq_wait_inc_below(data->rqw, get_limit(data->rwb, data->rw)))
-               return -1;
+       struct wbt_wait_data *data = private_data;
+       return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw));
+}
 
-       data->got_token = true;
-       list_del_init(&curr->entry);
-       wake_up_process(data->task);
-       return 1;
+static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data)
+{
+       struct wbt_wait_data *data = private_data;
+       wbt_rqw_done(data->rwb, rqw, data->wb_acct);
 }
 
 /*
@@ -521,57 +511,16 @@ static int wbt_wake_function(struct wait_queue_entry *curr, unsigned int mode,
  * the timer to kick off queuing again.
  */
 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
-                      unsigned long rw, spinlock_t *lock)
-       __releases(lock)
-       __acquires(lock)
+                      unsigned long rw)
 {
        struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
        struct wbt_wait_data data = {
-               .wq = {
-                       .func   = wbt_wake_function,
-                       .entry  = LIST_HEAD_INIT(data.wq.entry),
-               },
-               .task = current,
                .rwb = rwb,
-               .rqw = rqw,
+               .wb_acct = wb_acct,
                .rw = rw,
        };
-       bool has_sleeper;
-
-       has_sleeper = wq_has_sleeper(&rqw->wait);
-       if (!has_sleeper && rq_wait_inc_below(rqw, get_limit(rwb, rw)))
-               return;
 
-       prepare_to_wait_exclusive(&rqw->wait, &data.wq, TASK_UNINTERRUPTIBLE);
-       do {
-               if (data.got_token)
-                       break;
-
-               if (!has_sleeper &&
-                   rq_wait_inc_below(rqw, get_limit(rwb, rw))) {
-                       finish_wait(&rqw->wait, &data.wq);
-
-                       /*
-                        * We raced with wbt_wake_function() getting a token,
-                        * which means we now have two. Put our local token
-                        * and wake anyone else potentially waiting for one.
-                        */
-                       if (data.got_token)
-                               wbt_rqw_done(rwb, rqw, wb_acct);
-                       break;
-               }
-
-               if (lock) {
-                       spin_unlock_irq(lock);
-                       io_schedule();
-                       spin_lock_irq(lock);
-               } else
-                       io_schedule();
-
-               has_sleeper = false;
-       } while (1);
-
-       finish_wait(&rqw->wait, &data.wq);
+       rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);
 }
 
 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
@@ -624,7 +573,7 @@ static void wbt_cleanup(struct rq_qos *rqos, struct bio *bio)
  * in an irq held spinlock, if it holds one when calling this function.
  * If we do sleep, we'll release and re-grab it.
  */
-static void wbt_wait(struct rq_qos *rqos, struct bio *bio, spinlock_t *lock)
+static void wbt_wait(struct rq_qos *rqos, struct bio *bio)
 {
        struct rq_wb *rwb = RQWB(rqos);
        enum wbt_flags flags;
@@ -636,7 +585,7 @@ static void wbt_wait(struct rq_qos *rqos, struct bio *bio, spinlock_t *lock)
                return;
        }
 
-       __wbt_wait(rwb, flags, bio->bi_opf, lock);
+       __wbt_wait(rwb, flags, bio->bi_opf);
 
        if (!blk_stat_is_active(rwb->cb))
                rwb_arm_timer(rwb);
@@ -709,8 +658,7 @@ void wbt_enable_default(struct request_queue *q)
        if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
                return;
 
-       if ((q->mq_ops && IS_ENABLED(CONFIG_BLK_WBT_MQ)) ||
-           (q->request_fn && IS_ENABLED(CONFIG_BLK_WBT_SQ)))
+       if (queue_is_mq(q) && IS_ENABLED(CONFIG_BLK_WBT_MQ))
                wbt_init(q);
 }
 EXPORT_SYMBOL_GPL(wbt_enable_default);
@@ -760,12 +708,13 @@ void wbt_disable_default(struct request_queue *q)
        if (!rqos)
                return;
        rwb = RQWB(rqos);
-       if (rwb->enable_state == WBT_STATE_ON_DEFAULT)
+       if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {
+               blk_stat_deactivate(rwb->cb);
                rwb->wb_normal = 0;
+       }
 }
 EXPORT_SYMBOL_GPL(wbt_disable_default);
 
-
 static struct rq_qos_ops wbt_rqos_ops = {
        .throttle = wbt_wait,
        .issue = wbt_issue,