sched: wakeup-buddy tasks are cache-hot

[sfrench/cifs-2.6.git] / kernel / sched.c

diff --git a/kernel/sched.c b/kernel/sched.c
index 9df9ba73cb7a941ea7a8feb18a58b35e726b26a4..3f7c5eb254e205c24bc95bf5ac264e28d918d9b8 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -301,7 +301,7 @@ struct cfs_rq {
        /* 'curr' points to currently running entity on this cfs_rq.
         * It is set to NULL otherwise (i.e when none are currently running).
         */
-       struct sched_entity *curr;
+       struct sched_entity *curr, *next;
 
        unsigned long nr_spread_over;
 
@@ -1084,7 +1084,7 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
        u64 tmp;
 
        if (unlikely(!lw->inv_weight))
-               lw->inv_weight = (WMULT_CONST - lw->weight/2) / lw->weight + 1;
+               lw->inv_weight = (WMULT_CONST-lw->weight/2) / (lw->weight+1);
 
        tmp = (u64)delta_exec * weight;
        /*
@@ -1108,11 +1108,13 @@ calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
 static inline void update_load_add(struct load_weight *lw, unsigned long inc)
 {
        lw->weight += inc;
+       lw->inv_weight = 0;
 }
 
 static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
 {
        lw->weight -= dec;
+       lw->inv_weight = 0;
 }
 
 /*
@@ -1394,6 +1396,12 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
 {
        s64 delta;
 
+       /*
+        * Buddy candidates are cache hot:
+        */
+       if (&p->se == cfs_rq_of(&p->se)->next)
+               return 1;
+
        if (p->sched_class != &fair_sched_class)
                return 0;
 
@@ -1853,10 +1861,11 @@ out_activate:
                schedstat_inc(p, se.nr_wakeups_remote);
        update_rq_clock(rq);
        activate_task(rq, p, 1);
-       check_preempt_curr(rq, p);
        success = 1;
 
 out_running:
+       check_preempt_curr(rq, p);
+
        p->state = TASK_RUNNING;
 #ifdef CONFIG_SMP
        if (p->sched_class->task_wake_up)
@@ -1890,6 +1899,8 @@ static void __sched_fork(struct task_struct *p)
        p->se.exec_start                = 0;
        p->se.sum_exec_runtime          = 0;
        p->se.prev_sum_exec_runtime     = 0;
+       p->se.last_wakeup               = 0;
+       p->se.avg_overlap               = 0;
 
 #ifdef CONFIG_SCHEDSTATS
        p->se.wait_start                = 0;