Merge branch 'sched/urgent' into sched/core, to pick up fixes

[linux.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4a2e8cae63c41111672a898d3955008cd345c69b..632804fa0b125e83fd62e9e4056386c052f697ac 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -715,8 +715,12 @@ struct root_domain {
        cpumask_var_t           span;
        cpumask_var_t           online;
 
-       /* Indicate more than one runnable task for any CPU */
-       bool                    overload;
+       /*
+        * Indicate pullable load on at least one CPU, e.g:
+        * - More than one runnable task
+        * - Running task is misfit
+        */
+       int                     overload;
 
        /*
         * The bit corresponding to a CPU gets set here if such CPU has more
@@ -783,6 +787,7 @@ struct rq {
 #ifdef CONFIG_NUMA_BALANCING
        unsigned int            nr_numa_running;
        unsigned int            nr_preferred_running;
+       unsigned int            numa_migrate_on;
 #endif
        #define CPU_LOAD_IDX_MAX 5
        unsigned long           cpu_load[CPU_LOAD_IDX_MAX];
@@ -842,6 +847,8 @@ struct rq {
 
        unsigned char           idle_balance;
 
+       unsigned long           misfit_task_load;
+
        /* For active balancing */
        int                     active_balance;
        int                     push_cpu;
@@ -1185,6 +1192,7 @@ DECLARE_PER_CPU(int, sd_llc_id);
 DECLARE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
 DECLARE_PER_CPU(struct sched_domain *, sd_numa);
 DECLARE_PER_CPU(struct sched_domain *, sd_asym);
+extern struct static_key_false sched_asym_cpucapacity;
 
 struct sched_group_capacity {
        atomic_t                ref;
@@ -1194,6 +1202,7 @@ struct sched_group_capacity {
         */
        unsigned long           capacity;
        unsigned long           min_capacity;           /* Min per-CPU capacity in group */
+       unsigned long           max_capacity;           /* Max per-CPU capacity in group */
        unsigned long           next_update;
        int                     imbalance;              /* XXX unrelated to capacity but shared group state */
 
@@ -1393,7 +1402,7 @@ static const_debug __maybe_unused unsigned int sysctl_sched_features =
        0;
 #undef SCHED_FEAT
 
-#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
+#define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
 
 #endif /* SCHED_DEBUG && HAVE_JUMP_LABEL */
 
@@ -1523,7 +1532,7 @@ struct sched_class {
 
 #ifdef CONFIG_SMP
        int  (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
-       void (*migrate_task_rq)(struct task_struct *p);
+       void (*migrate_task_rq)(struct task_struct *p, int new_cpu);
 
        void (*task_woken)(struct rq *this_rq, struct task_struct *task);
 
@@ -1693,8 +1702,8 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
 
        if (prev_nr < 2 && rq->nr_running >= 2) {
 #ifdef CONFIG_SMP
-               if (!rq->rd->overload)
-                       rq->rd->overload = true;
+               if (!READ_ONCE(rq->rd->overload))
+                       WRITE_ONCE(rq->rd->overload, 1);
 #endif
        }