blkcg: handle dying request_queue when associating a blkg

[sfrench/cifs-2.6.git] / block / blk-cgroup.c

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index b421a9457e058be6c7a69af07cb8b37e7aab2969..c30661ddc873e04c5864d002cd3474a74180f1c1 100644 (file)
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -81,6 +81,37 @@ static void blkg_free(struct blkcg_gq *blkg)
        kfree(blkg);
 }
 
+static void __blkg_release(struct rcu_head *rcu)
+{
+       struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
+
+       percpu_ref_exit(&blkg->refcnt);
+
+       /* release the blkcg and parent blkg refs this blkg has been holding */
+       css_put(&blkg->blkcg->css);
+       if (blkg->parent)
+               blkg_put(blkg->parent);
+
+       wb_congested_put(blkg->wb_congested);
+
+       blkg_free(blkg);
+}
+
+/*
+ * A group is RCU protected, but having an rcu lock does not mean that one
+ * can access all the fields of blkg and assume these are valid.  For
+ * example, don't try to follow throtl_data and request queue links.
+ *
+ * Having a reference to blkg under an rcu allows accesses to only values
+ * local to groups like group stats and group rate limits.
+ */
+static void blkg_release(struct percpu_ref *ref)
+{
+       struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
+
+       call_rcu(&blkg->rcu_head, __blkg_release);
+}
+
 /**
  * blkg_alloc - allocate a blkg
  * @blkcg: block cgroup the new blkg is associated with
@@ -107,7 +138,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
        blkg->q = q;
        INIT_LIST_HEAD(&blkg->q_node);
        blkg->blkcg = blkcg;
-       atomic_set(&blkg->refcnt, 1);
 
        for (i = 0; i < BLKCG_MAX_POLS; i++) {
                struct blkcg_policy *pol = blkcg_policy[i];
@@ -172,6 +202,12 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
        WARN_ON_ONCE(!rcu_read_lock_held());
        lockdep_assert_held(&q->queue_lock);
 
+       /* request_queue is dying, do not create/recreate a blkg */
+       if (blk_queue_dying(q)) {
+               ret = -ENODEV;
+               goto err_free_blkg;
+       }
+
        /* blkg holds a reference to blkcg */
        if (!css_tryget_online(&blkcg->css)) {
                ret = -ENODEV;
@@ -207,6 +243,11 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
                blkg_get(blkg->parent);
        }
 
+       ret = percpu_ref_init(&blkg->refcnt, blkg_release, 0,
+                             GFP_NOWAIT | __GFP_NOWARN);
+       if (ret)
+               goto err_cancel_ref;
+
        /* invoke per-policy init */
        for (i = 0; i < BLKCG_MAX_POLS; i++) {
                struct blkcg_policy *pol = blkcg_policy[i];
@@ -239,6 +280,8 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
        blkg_put(blkg);
        return ERR_PTR(ret);
 
+err_cancel_ref:
+       percpu_ref_exit(&blkg->refcnt);
 err_put_congested:
        wb_congested_put(wb_congested);
 err_put_css:
@@ -258,9 +301,8 @@ err_free_blkg:
  * that all non-root blkg's have access to the parent blkg.  This function
  * should be called under RCU read lock and @q->queue_lock.
  *
- * Returns pointer to the looked up or created blkg on success, ERR_PTR()
- * value on error.  If @q is dead, returns ERR_PTR(-EINVAL).  If @q is not
- * dead and bypassing, returns ERR_PTR(-EBUSY).
+ * Returns the blkg or the closest blkg if blkg_create() fails as it walks
+ * down from root.
  */
 struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
                                      struct request_queue *q)
@@ -276,19 +318,29 @@ struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
 
        /*
         * Create blkgs walking down from blkcg_root to @blkcg, so that all
-        * non-root blkgs have access to their parents.
+        * non-root blkgs have access to their parents.  Returns the closest
+        * blkg to the intended blkg should blkg_create() fail.
         */
        while (true) {
                struct blkcg *pos = blkcg;
                struct blkcg *parent = blkcg_parent(blkcg);
-
-               while (parent && !__blkg_lookup(parent, q, false)) {
+               struct blkcg_gq *ret_blkg = q->root_blkg;
+
+               while (parent) {
+                       blkg = __blkg_lookup(parent, q, false);
+                       if (blkg) {
+                               /* remember closest blkg */
+                               ret_blkg = blkg;
+                               break;
+                       }
                        pos = parent;
                        parent = blkcg_parent(parent);
                }
 
                blkg = blkg_create(pos, q, NULL);
-               if (pos == blkcg || IS_ERR(blkg))
+               if (IS_ERR(blkg))
+                       return ret_blkg;
+               if (pos == blkcg)
                        return blkg;
        }
 }
@@ -358,7 +410,7 @@ static void blkg_destroy(struct blkcg_gq *blkg)
         * Put the reference taken at the time of creation so that when all
         * queues are gone, group can be destroyed.
         */
-       blkg_put(blkg);
+       percpu_ref_kill(&blkg->refcnt);
 }
 
 /**
@@ -1690,8 +1742,7 @@ void blkcg_maybe_throttle_current(void)
        blkg = blkg_lookup(blkcg, q);
        if (!blkg)
                goto out;
-       blkg = blkg_try_get(blkg);
-       if (!blkg)
+       if (!blkg_tryget(blkg))
                goto out;
        rcu_read_unlock();