Merge branch 'for-4.15' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup

[sfrench/cifs-2.6.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 14db76cd496ffb68641c3cc68a9be72205524e1b..b19552a212de379f8a06589249fd0d78af4482dc 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 
 #include <linux/sched.h>
 #include <linux/sched/autogroup.h>
@@ -29,6 +30,7 @@
 #include <linux/irq_work.h>
 #include <linux/tick.h>
 #include <linux/slab.h>
+#include <linux/cgroup.h>
 
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
@@ -36,7 +38,6 @@
 
 #include "cpupri.h"
 #include "cpudeadline.h"
-#include "cpuacct.h"
 
 #ifdef CONFIG_SCHED_DEBUG
 # define SCHED_WARN_ON(x)      WARN_ONCE(x, #x)
@@ -226,7 +227,7 @@ struct dl_bw {
 static inline void __dl_update(struct dl_bw *dl_b, s64 bw);
 
 static inline
-void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw, int cpus)
+void __dl_sub(struct dl_bw *dl_b, u64 tsk_bw, int cpus)
 {
        dl_b->total_bw -= tsk_bw;
        __dl_update(dl_b, (s32)tsk_bw / cpus);
@@ -255,7 +256,6 @@ extern int sched_dl_overflow(struct task_struct *p, int policy,
 extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr);
 extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr);
 extern bool __checkparam_dl(const struct sched_attr *attr);
-extern void __dl_clear_params(struct task_struct *p);
 extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);
 extern int dl_task_can_attach(struct task_struct *p,
                              const struct cpumask *cs_cpus_allowed);
@@ -418,6 +418,7 @@ struct cfs_bandwidth { };
 /* CFS-related fields in a runqueue */
 struct cfs_rq {
        struct load_weight load;
+       unsigned long runnable_weight;
        unsigned int nr_running, h_nr_running;
 
        u64 exec_clock;
@@ -443,18 +444,22 @@ struct cfs_rq {
         * CFS load tracking
         */
        struct sched_avg avg;
-       u64 runnable_load_sum;
-       unsigned long runnable_load_avg;
-#ifdef CONFIG_FAIR_GROUP_SCHED
-       unsigned long tg_load_avg_contrib;
-       unsigned long propagate_avg;
-#endif
-       atomic_long_t removed_load_avg, removed_util_avg;
 #ifndef CONFIG_64BIT
        u64 load_last_update_time_copy;
 #endif
+       struct {
+               raw_spinlock_t  lock ____cacheline_aligned;
+               int             nr;
+               unsigned long   load_avg;
+               unsigned long   util_avg;
+               unsigned long   runnable_sum;
+       } removed;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
+       unsigned long tg_load_avg_contrib;
+       long propagate;
+       long prop_runnable_sum;
+
        /*
         *   h_load = weight * f(tg)
         *
@@ -501,7 +506,7 @@ static inline int rt_bandwidth_enabled(void)
 }
 
 /* RT IPI pull logic requires IRQ_WORK */
-#ifdef CONFIG_IRQ_WORK
+#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_SMP)
 # define HAVE_RT_PUSH_IPI
 #endif
 
@@ -523,12 +528,6 @@ struct rt_rq {
        unsigned long rt_nr_total;
        int overloaded;
        struct plist_head pushable_tasks;
-#ifdef HAVE_RT_PUSH_IPI
-       int push_flags;
-       int push_cpu;
-       struct irq_work push_work;
-       raw_spinlock_t push_lock;
-#endif
 #endif /* CONFIG_SMP */
        int rt_queued;
 
@@ -637,6 +636,19 @@ struct root_domain {
        struct dl_bw dl_bw;
        struct cpudl cpudl;
 
+#ifdef HAVE_RT_PUSH_IPI
+       /*
+        * For IPI pull requests, loop across the rto_mask.
+        */
+       struct irq_work rto_push_work;
+       raw_spinlock_t rto_lock;
+       /* These are only updated and read within rto_lock */
+       int rto_loop;
+       int rto_cpu;
+       /* These atomics are updated outside of a lock */
+       atomic_t rto_loop_next;
+       atomic_t rto_loop_start;
+#endif
        /*
         * The "RT overload" flag: it gets set if a CPU has more than
         * one runnable RT task.
@@ -654,6 +666,9 @@ extern void init_defrootdomain(void);
 extern int sched_init_domains(const struct cpumask *cpu_map);
 extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
 
+#ifdef HAVE_RT_PUSH_IPI
+extern void rto_push_irq_work_func(struct irq_work *work);
+#endif
 #endif /* CONFIG_SMP */
 
 /*
@@ -1218,8 +1233,6 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 # define const_debug const
 #endif
 
-extern const_debug unsigned int sysctl_sched_features;
-
 #define SCHED_FEAT(name, enabled)      \
        __SCHED_FEAT_##name ,
 
@@ -1231,6 +1244,13 @@ enum {
 #undef SCHED_FEAT
 
 #if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL)
+
+/*
+ * To support run-time toggling of sched features, all the translation units
+ * (but core.c) reference the sysctl_sched_features defined in core.c.
+ */
+extern const_debug unsigned int sysctl_sched_features;
+
 #define SCHED_FEAT(name, enabled)                                      \
 static __always_inline bool static_branch_##name(struct static_key *key) \
 {                                                                      \
@@ -1238,13 +1258,27 @@ static __always_inline bool static_branch_##name(struct static_key *key) \
 }
 
 #include "features.h"
-
 #undef SCHED_FEAT
 
 extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
 #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
+
 #else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */
+
+/*
+ * Each translation unit has its own copy of sysctl_sched_features to allow
+ * constants propagation at compile time and compiler optimization based on
+ * features default.
+ */
+#define SCHED_FEAT(name, enabled)      \
+       (1UL << __SCHED_FEAT_##name) * enabled |
+static const_debug __maybe_unused unsigned int sysctl_sched_features =
+#include "features.h"
+       0;
+#undef SCHED_FEAT
+
 #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
+
 #endif /* SCHED_DEBUG && HAVE_JUMP_LABEL */
 
 extern struct static_key_false sched_numa_balancing;
@@ -1529,6 +1563,8 @@ extern void init_sched_dl_class(void);
 extern void init_sched_rt_class(void);
 extern void init_sched_fair_class(void);
 
+extern void reweight_task(struct task_struct *p, int prio);
+
 extern void resched_curr(struct rq *rq);
 extern void resched_cpu(int cpu);