Merge branch 'sched/urgent' into sched/core, to pick up fixes

[linux.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index c7742dcc136c6aa3ba95f0874c02c96e1eaccde8..632804fa0b125e83fd62e9e4056386c052f697ac 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -594,6 +594,7 @@ struct rt_rq {
        unsigned long           rt_nr_total;
        int                     overloaded;
        struct plist_head       pushable_tasks;
+
 #endif /* CONFIG_SMP */
        int                     rt_queued;
 
@@ -673,7 +674,26 @@ struct dl_rq {
        u64                     bw_ratio;
 };
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
+/* An entity is a task if it doesn't "own" a runqueue */
+#define entity_is_task(se)     (!se->my_q)
+#else
+#define entity_is_task(se)     1
+#endif
+
 #ifdef CONFIG_SMP
+/*
+ * XXX we want to get rid of these helpers and use the full load resolution.
+ */
+static inline long se_weight(struct sched_entity *se)
+{
+       return scale_load_down(se->load.weight);
+}
+
+static inline long se_runnable(struct sched_entity *se)
+{
+       return scale_load_down(se->runnable_weight);
+}
 
 static inline bool sched_asym_prefer(int a, int b)
 {
@@ -695,8 +715,12 @@ struct root_domain {
        cpumask_var_t           span;
        cpumask_var_t           online;
 
-       /* Indicate more than one runnable task for any CPU */
-       bool                    overload;
+       /*
+        * Indicate pullable load on at least one CPU, e.g:
+        * - More than one runnable task
+        * - Running task is misfit
+        */
+       int                     overload;
 
        /*
         * The bit corresponding to a CPU gets set here if such CPU has more
@@ -763,6 +787,7 @@ struct rq {
 #ifdef CONFIG_NUMA_BALANCING
        unsigned int            nr_numa_running;
        unsigned int            nr_preferred_running;
+       unsigned int            numa_migrate_on;
 #endif
        #define CPU_LOAD_IDX_MAX 5
        unsigned long           cpu_load[CPU_LOAD_IDX_MAX];
@@ -822,6 +847,8 @@ struct rq {
 
        unsigned char           idle_balance;
 
+       unsigned long           misfit_task_load;
+
        /* For active balancing */
        int                     active_balance;
        int                     push_cpu;
@@ -833,8 +860,12 @@ struct rq {
 
        struct list_head cfs_tasks;
 
-       u64                     rt_avg;
-       u64                     age_stamp;
+       struct sched_avg        avg_rt;
+       struct sched_avg        avg_dl;
+#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
+#define HAVE_SCHED_AVG_IRQ
+       struct sched_avg        avg_irq;
+#endif
        u64                     idle_stamp;
        u64                     avg_idle;
 
@@ -1075,7 +1106,8 @@ enum numa_faults_stats {
 };
 extern void sched_setnuma(struct task_struct *p, int node);
 extern int migrate_task_to(struct task_struct *p, int cpu);
-extern int migrate_swap(struct task_struct *, struct task_struct *);
+extern int migrate_swap(struct task_struct *p, struct task_struct *t,
+                       int cpu, int scpu);
 extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p);
 #else
 static inline void
@@ -1160,6 +1192,7 @@ DECLARE_PER_CPU(int, sd_llc_id);
 DECLARE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
 DECLARE_PER_CPU(struct sched_domain *, sd_numa);
 DECLARE_PER_CPU(struct sched_domain *, sd_asym);
+extern struct static_key_false sched_asym_cpucapacity;
 
 struct sched_group_capacity {
        atomic_t                ref;
@@ -1169,6 +1202,7 @@ struct sched_group_capacity {
         */
        unsigned long           capacity;
        unsigned long           min_capacity;           /* Min per-CPU capacity in group */
+       unsigned long           max_capacity;           /* Max per-CPU capacity in group */
        unsigned long           next_update;
        int                     imbalance;              /* XXX unrelated to capacity but shared group state */
 
@@ -1368,7 +1402,7 @@ static const_debug __maybe_unused unsigned int sysctl_sched_features =
        0;
 #undef SCHED_FEAT
 
-#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
+#define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
 
 #endif /* SCHED_DEBUG && HAVE_JUMP_LABEL */
 
@@ -1498,7 +1532,7 @@ struct sched_class {
 
 #ifdef CONFIG_SMP
        int  (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
-       void (*migrate_task_rq)(struct task_struct *p);
+       void (*migrate_task_rq)(struct task_struct *p, int new_cpu);
 
        void (*task_woken)(struct rq *this_rq, struct task_struct *task);
 
@@ -1668,8 +1702,8 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
 
        if (prev_nr < 2 && rq->nr_running >= 2) {
 #ifdef CONFIG_SMP
-               if (!rq->rd->overload)
-                       rq->rd->overload = true;
+               if (!READ_ONCE(rq->rd->overload))
+                       WRITE_ONCE(rq->rd->overload, 1);
 #endif
        }
 
@@ -1690,15 +1724,9 @@ extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags);
 
 extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
 
-extern const_debug unsigned int sysctl_sched_time_avg;
 extern const_debug unsigned int sysctl_sched_nr_migrate;
 extern const_debug unsigned int sysctl_sched_migration_cost;
 
-static inline u64 sched_avg_period(void)
-{
-       return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
-}
-
 #ifdef CONFIG_SCHED_HRTICK
 
 /*
@@ -1735,8 +1763,6 @@ unsigned long arch_scale_freq_capacity(int cpu)
 #endif
 
 #ifdef CONFIG_SMP
-extern void sched_avg_update(struct rq *rq);
-
 #ifndef arch_scale_cpu_capacity
 static __always_inline
 unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu)
@@ -1747,12 +1773,6 @@ unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu)
        return SCHED_CAPACITY_SCALE;
 }
 #endif
-
-static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
-{
-       rq->rt_avg += rt_delta * arch_scale_freq_capacity(cpu_of(rq));
-       sched_avg_update(rq);
-}
 #else
 #ifndef arch_scale_cpu_capacity
 static __always_inline
@@ -1761,8 +1781,6 @@ unsigned long arch_scale_cpu_capacity(void __always_unused *sd, int cpu)
        return SCHED_CAPACITY_SCALE;
 }
 #endif
-static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { }
-static inline void sched_avg_update(struct rq *rq) { }
 #endif
 
 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
@@ -2177,11 +2195,16 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
 #endif
 
 #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
-static inline unsigned long cpu_util_dl(struct rq *rq)
+static inline unsigned long cpu_bw_dl(struct rq *rq)
 {
        return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT;
 }
 
+static inline unsigned long cpu_util_dl(struct rq *rq)
+{
+       return READ_ONCE(rq->avg_dl.util_avg);
+}
+
 static inline unsigned long cpu_util_cfs(struct rq *rq)
 {
        unsigned long util = READ_ONCE(rq->cfs.avg.util_avg);
@@ -2193,4 +2216,37 @@ static inline unsigned long cpu_util_cfs(struct rq *rq)
 
        return util;
 }
+
+static inline unsigned long cpu_util_rt(struct rq *rq)
+{
+       return READ_ONCE(rq->avg_rt.util_avg);
+}
+#endif
+
+#ifdef HAVE_SCHED_AVG_IRQ
+static inline unsigned long cpu_util_irq(struct rq *rq)
+{
+       return rq->avg_irq.util_avg;
+}
+
+static inline
+unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)
+{
+       util *= (max - irq);
+       util /= max;
+
+       return util;
+
+}
+#else
+static inline unsigned long cpu_util_irq(struct rq *rq)
+{
+       return 0;
+}
+
+static inline
+unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)
+{
+       return util;
+}
 #endif