Merge tag 'for-linus-4.8-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git...
[sfrench/cifs-2.6.git] / kernel / sched / cputime.c
index 8c4c6dcc052c867fe480b7541575dc66a2a7cb04..1934f658c03604272e5809f32fee1a6a3c928990 100644 (file)
@@ -49,15 +49,12 @@ DEFINE_PER_CPU(seqcount_t, irq_time_seq);
  */
 void irqtime_account_irq(struct task_struct *curr)
 {
-       unsigned long flags;
        s64 delta;
        int cpu;
 
        if (!sched_clock_irqtime)
                return;
 
-       local_irq_save(flags);
-
        cpu = smp_processor_id();
        delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
        __this_cpu_add(irq_start_time, delta);
@@ -75,44 +72,53 @@ void irqtime_account_irq(struct task_struct *curr)
                __this_cpu_add(cpu_softirq_time, delta);
 
        irq_time_write_end();
-       local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
 
-static int irqtime_account_hi_update(void)
+static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 {
        u64 *cpustat = kcpustat_this_cpu->cpustat;
        unsigned long flags;
-       u64 latest_ns;
-       int ret = 0;
+       cputime_t irq_cputime;
 
        local_irq_save(flags);
-       latest_ns = this_cpu_read(cpu_hardirq_time);
-       if (nsecs_to_cputime64(latest_ns) > cpustat[CPUTIME_IRQ])
-               ret = 1;
+       irq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_hardirq_time)) -
+                     cpustat[CPUTIME_IRQ];
+       irq_cputime = min(irq_cputime, maxtime);
+       cpustat[CPUTIME_IRQ] += irq_cputime;
        local_irq_restore(flags);
-       return ret;
+       return irq_cputime;
 }
 
-static int irqtime_account_si_update(void)
+static cputime_t irqtime_account_si_update(cputime_t maxtime)
 {
        u64 *cpustat = kcpustat_this_cpu->cpustat;
        unsigned long flags;
-       u64 latest_ns;
-       int ret = 0;
+       cputime_t softirq_cputime;
 
        local_irq_save(flags);
-       latest_ns = this_cpu_read(cpu_softirq_time);
-       if (nsecs_to_cputime64(latest_ns) > cpustat[CPUTIME_SOFTIRQ])
-               ret = 1;
+       softirq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_softirq_time)) -
+                         cpustat[CPUTIME_SOFTIRQ];
+       softirq_cputime = min(softirq_cputime, maxtime);
+       cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
        local_irq_restore(flags);
-       return ret;
+       return softirq_cputime;
 }
 
 #else /* CONFIG_IRQ_TIME_ACCOUNTING */
 
 #define sched_clock_irqtime    (0)
 
+static cputime_t irqtime_account_hi_update(cputime_t dummy)
+{
+       return 0;
+}
+
+static cputime_t irqtime_account_si_update(cputime_t dummy)
+{
+       return 0;
+}
+
 #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
 
 static inline void task_group_account_field(struct task_struct *p, int index,
@@ -257,29 +263,42 @@ void account_idle_time(cputime_t cputime)
                cpustat[CPUTIME_IDLE] += (__force u64) cputime;
 }
 
-static __always_inline bool steal_account_process_tick(void)
+static __always_inline cputime_t steal_account_process_time(cputime_t maxtime)
 {
 #ifdef CONFIG_PARAVIRT
        if (static_key_false(&paravirt_steal_enabled)) {
+               cputime_t steal_cputime;
                u64 steal;
-               unsigned long steal_jiffies;
 
                steal = paravirt_steal_clock(smp_processor_id());
                steal -= this_rq()->prev_steal_time;
 
-               /*
-                * steal is in nsecs but our caller is expecting steal
-                * time in jiffies. Lets cast the result to jiffies
-                * granularity and account the rest on the next rounds.
-                */
-               steal_jiffies = nsecs_to_jiffies(steal);
-               this_rq()->prev_steal_time += jiffies_to_nsecs(steal_jiffies);
+               steal_cputime = min(nsecs_to_cputime(steal), maxtime);
+               account_steal_time(steal_cputime);
+               this_rq()->prev_steal_time += cputime_to_nsecs(steal_cputime);
 
-               account_steal_time(jiffies_to_cputime(steal_jiffies));
-               return steal_jiffies;
+               return steal_cputime;
        }
 #endif
-       return false;
+       return 0;
+}
+
+/*
+ * Account how much elapsed time was spent in steal, irq, or softirq time.
+ */
+static inline cputime_t account_other_time(cputime_t max)
+{
+       cputime_t accounted;
+
+       accounted = steal_account_process_time(max);
+
+       if (accounted < max)
+               accounted += irqtime_account_hi_update(max - accounted);
+
+       if (accounted < max)
+               accounted += irqtime_account_si_update(max - accounted);
+
+       return accounted;
 }
 
 /*
@@ -342,21 +361,23 @@ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
 static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
                                         struct rq *rq, int ticks)
 {
-       cputime_t scaled = cputime_to_scaled(cputime_one_jiffy);
-       u64 cputime = (__force u64) cputime_one_jiffy;
-       u64 *cpustat = kcpustat_this_cpu->cpustat;
+       u64 cputime = (__force u64) cputime_one_jiffy * ticks;
+       cputime_t scaled, other;
 
-       if (steal_account_process_tick())
+       /*
+        * When returning from idle, many ticks can get accounted at
+        * once, including some ticks of steal, irq, and softirq time.
+        * Subtract those ticks from the amount of time accounted to
+        * idle, or potentially user or system time. Due to rounding,
+        * other time can exceed ticks occasionally.
+        */
+       other = account_other_time(cputime);
+       if (other >= cputime)
                return;
+       cputime -= other;
+       scaled = cputime_to_scaled(cputime);
 
-       cputime *= ticks;
-       scaled *= ticks;
-
-       if (irqtime_account_hi_update()) {
-               cpustat[CPUTIME_IRQ] += cputime;
-       } else if (irqtime_account_si_update()) {
-               cpustat[CPUTIME_SOFTIRQ] += cputime;
-       } else if (this_cpu_ksoftirqd() == p) {
+       if (this_cpu_ksoftirqd() == p) {
                /*
                 * ksoftirqd time do not get accounted in cpu_softirq_time.
                 * So, we have to handle it separately here.
@@ -406,6 +427,10 @@ void vtime_common_task_switch(struct task_struct *prev)
 }
 #endif
 
+#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
+
+
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 /*
  * Archs that account the whole time spent in the idle task
  * (outside irq) as idle time can rely on this and just implement
@@ -415,33 +440,16 @@ void vtime_common_task_switch(struct task_struct *prev)
  * vtime_account().
  */
 #ifndef __ARCH_HAS_VTIME_ACCOUNT
-void vtime_common_account_irq_enter(struct task_struct *tsk)
+void vtime_account_irq_enter(struct task_struct *tsk)
 {
-       if (!in_interrupt()) {
-               /*
-                * If we interrupted user, context_tracking_in_user()
-                * is 1 because the context tracking don't hook
-                * on irq entry/exit. This way we know if
-                * we need to flush user time on kernel entry.
-                */
-               if (context_tracking_in_user()) {
-                       vtime_account_user(tsk);
-                       return;
-               }
-
-               if (is_idle_task(tsk)) {
-                       vtime_account_idle(tsk);
-                       return;
-               }
-       }
-       vtime_account_system(tsk);
+       if (!in_interrupt() && is_idle_task(tsk))
+               vtime_account_idle(tsk);
+       else
+               vtime_account_system(tsk);
 }
-EXPORT_SYMBOL_GPL(vtime_common_account_irq_enter);
+EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
 #endif /* __ARCH_HAS_VTIME_ACCOUNT */
-#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
-
 
-#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
        *ut = p->utime;
@@ -466,7 +474,7 @@ void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime
  */
 void account_process_tick(struct task_struct *p, int user_tick)
 {
-       cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
+       cputime_t cputime, scaled, steal;
        struct rq *rq = this_rq();
 
        if (vtime_accounting_cpu_enabled())
@@ -477,16 +485,21 @@ void account_process_tick(struct task_struct *p, int user_tick)
                return;
        }
 
-       if (steal_account_process_tick())
+       cputime = cputime_one_jiffy;
+       steal = steal_account_process_time(cputime);
+
+       if (steal >= cputime)
                return;
 
+       cputime -= steal;
+       scaled = cputime_to_scaled(cputime);
+
        if (user_tick)
-               account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
+               account_user_time(p, cputimescaled);
        else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
-               account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
-                                   one_jiffy_scaled);
+               account_system_time(p, HARDIRQ_OFFSET, cputime, scaled);
        else
-               account_idle_time(cputime_one_jiffy);
+               account_idle_time(cputime);
 }
 
 /*
@@ -671,12 +684,14 @@ static cputime_t vtime_delta(struct task_struct *tsk)
 static cputime_t get_vtime_delta(struct task_struct *tsk)
 {
        unsigned long now = READ_ONCE(jiffies);
-       unsigned long delta = now - tsk->vtime_snap;
+       cputime_t delta, other;
 
+       delta = jiffies_to_cputime(now - tsk->vtime_snap);
+       other = account_other_time(delta);
        WARN_ON_ONCE(tsk->vtime_snap_whence == VTIME_INACTIVE);
        tsk->vtime_snap = now;
 
-       return jiffies_to_cputime(delta);
+       return delta - other;
 }
 
 static void __vtime_account_system(struct task_struct *tsk)
@@ -696,16 +711,6 @@ void vtime_account_system(struct task_struct *tsk)
        write_seqcount_end(&tsk->vtime_seqcount);
 }
 
-void vtime_gen_account_irq_exit(struct task_struct *tsk)
-{
-       write_seqcount_begin(&tsk->vtime_seqcount);
-       if (vtime_delta(tsk))
-               __vtime_account_system(tsk);
-       if (context_tracking_in_user())
-               tsk->vtime_snap_whence = VTIME_USER;
-       write_seqcount_end(&tsk->vtime_seqcount);
-}
-
 void vtime_account_user(struct task_struct *tsk)
 {
        cputime_t delta_cpu;