sched/fair: Optimize and simplify rq leaf_cfs_rq_list

[sfrench/cifs-2.6.git] / kernel / sched / fair.c

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 1b2cac76b35d1277bbe5c4303c7302d9d71ba4e6..7d8ef01669a531d0ac244f7568d0379c5f05aae2 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3179,6 +3179,8 @@ void reweight_task(struct task_struct *p, int prio)
        load->inv_weight = sched_prio_to_wmult[prio];
 }
 
+static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
+
 #ifdef CONFIG_FAIR_GROUP_SCHED
 #ifdef CONFIG_SMP
 /*
@@ -3289,8 +3291,6 @@ static long calc_group_shares(struct cfs_rq *cfs_rq)
 }
 #endif /* CONFIG_SMP */
 
-static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
-
 /*
  * Recomputes the group entity based on the current state of its group
  * runqueue.
@@ -4403,16 +4403,11 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
                __enqueue_entity(cfs_rq, se);
        se->on_rq = 1;
 
-       /*
-        * When bandwidth control is enabled, cfs might have been removed
-        * because of a parent been throttled but cfs->nr_running > 1. Try to
-        * add it unconditionally.
-        */
-       if (cfs_rq->nr_running == 1 || cfs_bandwidth_used())
-               list_add_leaf_cfs_rq(cfs_rq);
-
-       if (cfs_rq->nr_running == 1)
+       if (cfs_rq->nr_running == 1) {
                check_enqueue_throttle(cfs_rq);
+               if (!throttled_hierarchy(cfs_rq))
+                       list_add_leaf_cfs_rq(cfs_rq);
+       }
 }
 
 static void __clear_buddies_last(struct sched_entity *se)
@@ -5027,11 +5022,18 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
        /* update hierarchical throttle state */
        walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
 
-       /* Nothing to run but something to decay (on_list)? Complete the branch */
        if (!cfs_rq->load.weight) {
-               if (cfs_rq->on_list)
-                       goto unthrottle_throttle;
-               return;
+               if (!cfs_rq->on_list)
+                       return;
+               /*
+                * Nothing to run but something to decay (on_list)?
+                * Complete the branch.
+                */
+               for_each_sched_entity(se) {
+                       if (list_add_leaf_cfs_rq(cfs_rq_of(se)))
+                               break;
+               }
+               goto unthrottle_throttle;
        }
 
        task_delta = cfs_rq->h_nr_running;
@@ -5069,31 +5071,12 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
                /* end evaluation on encountering a throttled cfs_rq */
                if (cfs_rq_throttled(qcfs_rq))
                        goto unthrottle_throttle;
-
-               /*
-                * One parent has been throttled and cfs_rq removed from the
-                * list. Add it back to not break the leaf list.
-                */
-               if (throttled_hierarchy(qcfs_rq))
-                       list_add_leaf_cfs_rq(qcfs_rq);
        }
 
        /* At this point se is NULL and we are at root level*/
        add_nr_running(rq, task_delta);
 
 unthrottle_throttle:
-       /*
-        * The cfs_rq_throttled() breaks in the above iteration can result in
-        * incomplete leaf list maintenance, resulting in triggering the
-        * assertion below.
-        */
-       for_each_sched_entity(se) {
-               struct cfs_rq *qcfs_rq = cfs_rq_of(se);
-
-               if (list_add_leaf_cfs_rq(qcfs_rq))
-                       break;
-       }
-
        assert_list_leaf_cfs_rq(rq);
 
        /* Determine whether we need to wake up potentially idle CPU: */
@@ -5748,13 +5731,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
                /* end evaluation on encountering a throttled cfs_rq */
                if (cfs_rq_throttled(cfs_rq))
                        goto enqueue_throttle;
-
-               /*
-                * One parent has been throttled and cfs_rq removed from the
-                * list. Add it back to not break the leaf list.
-                */
-               if (throttled_hierarchy(cfs_rq))
-                       list_add_leaf_cfs_rq(cfs_rq);
        }
 
        /* At this point se is NULL and we are at root level*/
@@ -5778,21 +5754,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
                update_overutilized_status(rq);
 
 enqueue_throttle:
-       if (cfs_bandwidth_used()) {
-               /*
-                * When bandwidth control is enabled; the cfs_rq_throttled()
-                * breaks in the above iteration can result in incomplete
-                * leaf list maintenance, resulting in triggering the assertion
-                * below.
-                */
-               for_each_sched_entity(se) {
-                       cfs_rq = cfs_rq_of(se);
-
-                       if (list_add_leaf_cfs_rq(cfs_rq))
-                               break;
-               }
-       }
-
        assert_list_leaf_cfs_rq(rq);
 
        hrtick_update(rq);
@@ -11316,9 +11277,13 @@ static inline bool vruntime_normalized(struct task_struct *p)
  */
 static void propagate_entity_cfs_rq(struct sched_entity *se)
 {
-       struct cfs_rq *cfs_rq;
+       struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+       if (cfs_rq_throttled(cfs_rq))
+               return;
 
-       list_add_leaf_cfs_rq(cfs_rq_of(se));
+       if (!throttled_hierarchy(cfs_rq))
+               list_add_leaf_cfs_rq(cfs_rq);
 
        /* Start to propagate at parent */
        se = se->parent;
@@ -11326,14 +11291,13 @@ static void propagate_entity_cfs_rq(struct sched_entity *se)
        for_each_sched_entity(se) {
                cfs_rq = cfs_rq_of(se);
 
-               if (!cfs_rq_throttled(cfs_rq)){
-                       update_load_avg(cfs_rq, se, UPDATE_TG);
-                       list_add_leaf_cfs_rq(cfs_rq);
-                       continue;
-               }
+               update_load_avg(cfs_rq, se, UPDATE_TG);
 
-               if (list_add_leaf_cfs_rq(cfs_rq))
+               if (cfs_rq_throttled(cfs_rq))
                        break;
+
+               if (!throttled_hierarchy(cfs_rq))
+                       list_add_leaf_cfs_rq(cfs_rq);
        }
 }
 #else