NOHZ: reevaluate idle sleep length after add_timer_on()

add_timer_on() can add a timer on a CPU which is currently in a long
idle sleep, but the timer wheel is not reevaluated by the nohz code on
that CPU. So a timer can be delayed for quite a long time. This
triggered a false positive in the clocksource watchdog code.

To avoid this we need to wake up the idle CPU and enforce the
reevaluation of the timer wheel for the next timer event.

Add a function, which checks a given CPU for idle state, marks the
idle task with NEED_RESCHED and sends a reschedule IPI to notify the
other CPU of the change in the timer wheel.

Call this function from add_timer_on().

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Ingo Molnar <mingo@elte.hu>
Cc: stable@kernel.org
--
 include/linux/sched.h |    6 ++++++
 kernel/sched.c        |   43 +++++++++++++++++++++++++++++++++++++++++++
 kernel/timer.c        |   10 +++++++++-
 3 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index fed07d03364e869df06d7f1405d23d75850fe2b7..6a1e7afb099bc981f91a23ac9698e7244bab4aa0 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1541,6 +1541,12 @@ static inline void idle_task_exit(void) {}
 
 extern void sched_idle_next(void);
 
 
 extern void sched_idle_next(void);
 

+#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
+extern void wake_up_idle_cpu(int cpu);
+#else
+static inline void wake_up_idle_cpu(int cpu) { }
+#endif
+

 #ifdef CONFIG_SCHED_DEBUG
 extern unsigned int sysctl_sched_latency;
 extern unsigned int sysctl_sched_min_granularity;
 #ifdef CONFIG_SCHED_DEBUG
 extern unsigned int sysctl_sched_latency;
 extern unsigned int sysctl_sched_min_granularity;

diff --git a/kernel/sched.c b/kernel/sched.c
index 28c73f07efb2f7506f70be5ceb7a44c134770afa..8dcdec6fe0fe0983f4a90e8d51e501871974d383 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -1052,6 +1052,49 @@ static void resched_cpu(int cpu)
        resched_task(cpu_curr(cpu));
        spin_unlock_irqrestore(&rq->lock, flags);
 }
        resched_task(cpu_curr(cpu));
        spin_unlock_irqrestore(&rq->lock, flags);
 }

+
+#ifdef CONFIG_NO_HZ
+/*
+ * When add_timer_on() enqueues a timer into the timer wheel of an
+ * idle CPU then this timer might expire before the next timer event
+ * which is scheduled to wake up that CPU. In case of a completely
+ * idle system the next event might even be infinite time into the
+ * future. wake_up_idle_cpu() ensures that the CPU is woken up and
+ * leaves the inner idle loop so the newly added timer is taken into
+ * account when the CPU goes back to idle and evaluates the timer
+ * wheel for the next timer event.
+ */
+void wake_up_idle_cpu(int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+
+       if (cpu == smp_processor_id())
+               return;
+
+       /*
+        * This is safe, as this function is called with the timer
+        * wheel base lock of (cpu) held. When the CPU is on the way
+        * to idle and has not yet set rq->curr to idle then it will
+        * be serialized on the timer wheel base lock and take the new
+        * timer into account automatically.
+        */
+       if (rq->curr != rq->idle)
+               return;
+
+       /*
+        * We can set TIF_RESCHED on the idle task of the other CPU
+        * lockless. The worst case is that the other CPU runs the
+        * idle task through an additional NOOP schedule()
+        */
+       set_tsk_thread_flag(rq->idle, TIF_NEED_RESCHED);
+
+       /* NEED_RESCHED must be visible before we test polling */
+       smp_mb();
+       if (!tsk_is_polling(rq->idle))
+               smp_send_reschedule(cpu);
+}
+#endif
+

 #else
 static void __resched_task(struct task_struct *p, int tif_bit)
 {
 #else
 static void __resched_task(struct task_struct *p, int tif_bit)
 {

diff --git a/kernel/timer.c b/kernel/timer.c
index 99b00a25f88b3efc4c0558c78ece429d3d3300c3..b024106daa70163646b31a031085b4d732b4f096 100644 (file)
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -451,10 +451,18 @@ void add_timer_on(struct timer_list *timer, int cpu)
        spin_lock_irqsave(&base->lock, flags);
        timer_set_base(timer, base);
        internal_add_timer(base, timer);
        spin_lock_irqsave(&base->lock, flags);
        timer_set_base(timer, base);
        internal_add_timer(base, timer);

+       /*
+        * Check whether the other CPU is idle and needs to be
+        * triggered to reevaluate the timer wheel when nohz is
+        * active. We are protected against the other CPU fiddling
+        * with the timer by holding the timer base lock. This also
+        * makes sure that a CPU on the way to idle can not evaluate
+        * the timer wheel.
+        */
+       wake_up_idle_cpu(cpu);

        spin_unlock_irqrestore(&base->lock, flags);
 }
 
        spin_unlock_irqrestore(&base->lock, flags);
 }
 

-

 /**
  * mod_timer - modify a timer's timeout
  * @timer: the timer to be modified
 /**
  * mod_timer - modify a timer's timeout
  * @timer: the timer to be modified