Merge tag 'rcu.2022.05.19a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck...

[sfrench/cifs-2.6.git] / kernel / sched / core.c

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 068c088e958440808f6f33856fcbf061e47c0b36..2a05096559a263e3559fd60e5f4cbc02181f1a34 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5752,6 +5752,8 @@ static inline struct task_struct *pick_task(struct rq *rq)
 
 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
 
+static void queue_core_balance(struct rq *rq);
+
 static struct task_struct *
 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 {
@@ -5801,7 +5803,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
                }
 
                rq->core_pick = NULL;
-               return next;
+               goto out;
        }
 
        put_prev_task_balance(rq, prev, rf);
@@ -5851,7 +5853,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
                         */
                        WARN_ON_ONCE(fi_before);
                        task_vruntime_update(rq, next, false);
-                       goto done;
+                       goto out_set_next;
                }
        }
 
@@ -5970,8 +5972,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
                resched_curr(rq_i);
        }
 
-done:
+out_set_next:
        set_next_task(rq, next);
+out:
+       if (rq->core->core_forceidle_count && next == rq->idle)
+               queue_core_balance(rq);
+
        return next;
 }
 
@@ -6000,7 +6006,7 @@ static bool try_steal_cookie(int this, int that)
                if (p == src->core_pick || p == src->curr)
                        goto next;
 
-               if (!cpumask_test_cpu(this, &p->cpus_mask))
+               if (!is_cpu_allowed(p, this))
                        goto next;
 
                if (p->core_occupation > dst->idle->core_occupation)
@@ -6066,7 +6072,7 @@ static void sched_core_balance(struct rq *rq)
 
 static DEFINE_PER_CPU(struct callback_head, core_balance_head);
 
-void queue_core_balance(struct rq *rq)
+static void queue_core_balance(struct rq *rq)
 {
        if (!sched_core_enabled(rq))
                return;
@@ -6376,7 +6382,7 @@ static void __sched notrace __schedule(unsigned int sched_mode)
                migrate_disable_switch(rq, prev);
                psi_sched_switch(prev, next, !task_on_rq_queued(prev));
 
-               trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev_state, prev, next);
+               trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);
 
                /* Also unlocks the rq: */
                rq = context_switch(rq, prev, next, &rf);