time: tick-sched: Reorganize idle tick management code

Prepare the scheduler tick code for reworking the idle loop to
avoid stopping the tick in some cases.

The idea is to split the nohz idle entry call to decouple the idle
time stats accounting and preparatory work from the actual tick stop
code, in order to later be able to delay the tick stop once we reach
more power-knowledgeable callers.

Move away the tick_nohz_start_idle() invocation from
__tick_nohz_idle_enter(), rename the latter to
__tick_nohz_idle_stop_tick() and define tick_nohz_idle_stop_tick()
as a wrapper around it for calling it from the outside.

Make tick_nohz_idle_enter() only call tick_nohz_start_idle() instead
of calling the entire __tick_nohz_idle_enter(), add another wrapper
disabling and enabling interrupts around tick_nohz_idle_stop_tick()
and make the current callers of tick_nohz_idle_enter() call it too
to retain their current functionality.

Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>

diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c
index c0c756c76afebf08d282fb35c2fc49b641120bb4..2e20ae2fa2d6c3b865f2c745ad9896a752954907 100644 (file)
--- a/arch/x86/xen/smp_pv.c
+++ b/arch/x86/xen/smp_pv.c
@@ -425,6 +425,7 @@ static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
         * data back is to call:
         */
        tick_nohz_idle_enter();
+       tick_nohz_idle_stop_tick_protected();
 
        cpuhp_online_idle(CPUHP_AP_ONLINE_IDLE);
 }

diff --git a/include/linux/tick.h b/include/linux/tick.h
index 7f8c9a127f5a0c964a63581dfc954abf6d56ff4f..1d253df9ea3c57c76aeee546663d175bf9f02caf 100644 (file)
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -115,6 +115,7 @@ enum tick_dep_bits {
 extern bool tick_nohz_enabled;
 extern bool tick_nohz_tick_stopped(void);
 extern bool tick_nohz_tick_stopped_cpu(int cpu);
+extern void tick_nohz_idle_stop_tick(void);
 extern void tick_nohz_idle_enter(void);
 extern void tick_nohz_idle_exit(void);
 extern void tick_nohz_irq_exit(void);
@@ -123,10 +124,19 @@ extern unsigned long tick_nohz_get_idle_calls(void);
 extern unsigned long tick_nohz_get_idle_calls_cpu(int cpu);
 extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
 extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
+
+static inline void tick_nohz_idle_stop_tick_protected(void)
+{
+       local_irq_disable();
+       tick_nohz_idle_stop_tick();
+       local_irq_enable();
+}
+
 #else /* !CONFIG_NO_HZ_COMMON */
 #define tick_nohz_enabled (0)
 static inline int tick_nohz_tick_stopped(void) { return 0; }
 static inline int tick_nohz_tick_stopped_cpu(int cpu) { return 0; }
+static inline void tick_nohz_idle_stop_tick(void) { }
 static inline void tick_nohz_idle_enter(void) { }
 static inline void tick_nohz_idle_exit(void) { }
 
@@ -136,6 +146,8 @@ static inline ktime_t tick_nohz_get_sleep_length(void)
 }
 static inline u64 get_cpu_idle_time_us(int cpu, u64 *unused) { return -1; }
 static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1; }
+
+static inline void tick_nohz_idle_stop_tick_protected(void) { }
 #endif /* !CONFIG_NO_HZ_COMMON */
 
 #ifdef CONFIG_NO_HZ_FULL

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 2975f195e1c40427acc33f938a3c9c0174ab5db2..c0bc111878e6e017b499897c0aaec31f155b569f 100644 (file)
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -216,6 +216,7 @@ static void do_idle(void)
 
        __current_set_polling();
        tick_nohz_idle_enter();
+       tick_nohz_idle_stop_tick_protected();
 
        while (!need_resched()) {
                check_pgt_cache();

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 5d4a0342f934ddd0915e0e889cdaebb2ab9a8088..678349aec483f13e1fce1185d6aa2f74f8b6d397 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -528,14 +528,11 @@ static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
        sched_clock_idle_wakeup_event();
 }
 
-static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
+static void tick_nohz_start_idle(struct tick_sched *ts)
 {
-       ktime_t now = ktime_get();
-
-       ts->idle_entrytime = now;
+       ts->idle_entrytime = ktime_get();
        ts->idle_active = 1;
        sched_clock_idle_sleep_event();
-       return now;
 }
 
 /**
@@ -894,19 +891,21 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
        return true;
 }
 
-static void __tick_nohz_idle_enter(struct tick_sched *ts)
+static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
 {
-       ktime_t now, expires;
+       ktime_t expires;
        int cpu = smp_processor_id();
 
-       now = tick_nohz_start_idle(ts);
-
        if (can_stop_idle_tick(cpu, ts)) {
                int was_stopped = ts->tick_stopped;
 
                ts->idle_calls++;
 
-               expires = tick_nohz_stop_sched_tick(ts, now, cpu);
+               /*
+                * The idle entry time should be a sufficient approximation of
+                * the current time at this point.
+                */
+               expires = tick_nohz_stop_sched_tick(ts, ts->idle_entrytime, cpu);
                if (expires > 0LL) {
                        ts->idle_sleeps++;
                        ts->idle_expires = expires;
@@ -920,16 +919,19 @@ static void __tick_nohz_idle_enter(struct tick_sched *ts)
 }
 
 /**
- * tick_nohz_idle_enter - stop the idle tick from the idle task
+ * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
  *
  * When the next event is more than a tick into the future, stop the idle tick
- * Called when we start the idle loop.
- *
- * The arch is responsible of calling:
+ */
+void tick_nohz_idle_stop_tick(void)
+{
+       __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
+}
+
+/**
+ * tick_nohz_idle_enter - prepare for entering idle on the current CPU
  *
- * - rcu_idle_enter() after its last use of RCU before the CPU is put
- *  to sleep.
- * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
+ * Called when we start the idle loop.
  */
 void tick_nohz_idle_enter(void)
 {
@@ -941,7 +943,7 @@ void tick_nohz_idle_enter(void)
 
        ts = this_cpu_ptr(&tick_cpu_sched);
        ts->inidle = 1;
-       __tick_nohz_idle_enter(ts);
+       tick_nohz_start_idle(ts);
 
        local_irq_enable();
 }
@@ -958,10 +960,12 @@ void tick_nohz_irq_exit(void)
 {
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
-       if (ts->inidle)
-               __tick_nohz_idle_enter(ts);
-       else
+       if (ts->inidle) {
+               tick_nohz_start_idle(ts);
+               __tick_nohz_idle_stop_tick(ts);
+       } else {
                tick_nohz_full_update_tick(ts);
+       }
 }
 
 /**