sched/rt: Make update_curr_rt() more accurate

[sfrench/cifs-2.6.git] / kernel / sched / rt.c

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 862a513adca3d0285e7fa45ac878520ea2b70cf8..aad49451584e6766d1b9a2f657397da345146c23 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -951,11 +951,13 @@ static void update_curr_rt(struct rq *rq)
        struct task_struct *curr = rq->curr;
        struct sched_rt_entity *rt_se = &curr->rt;
        u64 delta_exec;
+       u64 now;
 
        if (curr->sched_class != &rt_sched_class)
                return;
 
-       delta_exec = rq_clock_task(rq) - curr->se.exec_start;
+       now = rq_clock_task(rq);
+       delta_exec = now - curr->se.exec_start;
        if (unlikely((s64)delta_exec <= 0))
                return;
 
@@ -968,7 +970,7 @@ static void update_curr_rt(struct rq *rq)
        curr->se.sum_exec_runtime += delta_exec;
        account_group_exec_runtime(curr, delta_exec);
 
-       curr->se.exec_start = rq_clock_task(rq);
+       curr->se.exec_start = now;
        cgroup_account_cputime(curr, delta_exec);
 
        sched_rt_avg_update(rq, delta_exec);
@@ -1907,9 +1909,8 @@ static void push_rt_tasks(struct rq *rq)
  * the rt_loop_next will cause the iterator to perform another scan.
  *
  */
-static int rto_next_cpu(struct rq *rq)
+static int rto_next_cpu(struct root_domain *rd)
 {
-       struct root_domain *rd = rq->rd;
        int next;
        int cpu;
 
@@ -1985,19 +1986,24 @@ static void tell_cpu_to_push(struct rq *rq)
         * Otherwise it is finishing up and an ipi needs to be sent.
         */
        if (rq->rd->rto_cpu < 0)
-               cpu = rto_next_cpu(rq);
+               cpu = rto_next_cpu(rq->rd);
 
        raw_spin_unlock(&rq->rd->rto_lock);
 
        rto_start_unlock(&rq->rd->rto_loop_start);
 
-       if (cpu >= 0)
+       if (cpu >= 0) {
+               /* Make sure the rd does not get freed while pushing */
+               sched_get_rd(rq->rd);
                irq_work_queue_on(&rq->rd->rto_push_work, cpu);
+       }
 }
 
 /* Called from hardirq context */
 void rto_push_irq_work_func(struct irq_work *work)
 {
+       struct root_domain *rd =
+               container_of(work, struct root_domain, rto_push_work);
        struct rq *rq;
        int cpu;
 
@@ -2013,18 +2019,20 @@ void rto_push_irq_work_func(struct irq_work *work)
                raw_spin_unlock(&rq->lock);
        }
 
-       raw_spin_lock(&rq->rd->rto_lock);
+       raw_spin_lock(&rd->rto_lock);
 
        /* Pass the IPI to the next rt overloaded queue */
-       cpu = rto_next_cpu(rq);
+       cpu = rto_next_cpu(rd);
 
-       raw_spin_unlock(&rq->rd->rto_lock);
+       raw_spin_unlock(&rd->rto_lock);
 
-       if (cpu < 0)
+       if (cpu < 0) {
+               sched_put_rd(rd);
                return;
+       }
 
        /* Try the next RT overloaded CPU */
-       irq_work_queue_on(&rq->rd->rto_push_work, cpu);
+       irq_work_queue_on(&rd->rto_push_work, cpu);
 }
 #endif /* HAVE_RT_PUSH_IPI */