sched: Fix smp_call_function_single_async() usage for ILB

[sfrench/cifs-2.6.git] / kernel / sched / fair.c

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index dda9b194d225ff2f5bfb1c9c7559b4f86c254d54..2890bd54a088a13a4f0d9e06e9bc38d387f2808d 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -10024,6 +10024,10 @@ static void kick_ilb(unsigned int flags)
        if (ilb_cpu >= nr_cpu_ids)
                return;
 
+       /*
+        * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
+        * the first flag owns it; cleared by nohz_csd_func().
+        */
        flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu));
        if (flags & NOHZ_KICK_MASK)
                return;
@@ -10371,20 +10375,14 @@ abort:
  */
 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
 {
-       int this_cpu = this_rq->cpu;
-       unsigned int flags;
+       unsigned int flags = this_rq->nohz_idle_balance;
 
-       if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK))
+       if (!flags)
                return false;
 
-       if (idle != CPU_IDLE) {
-               atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
-               return false;
-       }
+       this_rq->nohz_idle_balance = 0;
 
-       /* could be _relaxed() */
-       flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
-       if (!(flags & NOHZ_KICK_MASK))
+       if (idle != CPU_IDLE)
                return false;
 
        _nohz_idle_balance(this_rq, flags, idle);