cpusets: Rebuild root domain deadline accounting information

When the topology of root domains is modified by CPUset or CPUhotplug
operations information about the current deadline bandwidth held in the
root domain is lost.

This patch addresses the issue by recalculating the lost deadline
bandwidth information by circling through the deadline tasks held in
CPUsets and adding their current load to the root domain they are
associated with.

Tested-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Signed-off-by: Mathieu Poirier <mathieu.poirier@linaro.org>
Signed-off-by: Juri Lelli <juri.lelli@redhat.com>
[ Various additional modifications. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: bristot@redhat.com
Cc: claudio@evidence.eu.com
Cc: lizefan@huawei.com
Cc: longman@redhat.com
Cc: luca.abeni@santannapisa.it
Cc: rostedt@goodmis.org
Cc: tj@kernel.org
Cc: tommaso.cucinotta@santannapisa.it
Link: https://lkml.kernel.org/r/20190719140000.31694-4-juri.lelli@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index f6b048902d6c5337a05b0ef96a955f768483808e..3ba3e6da13a6fb350d93917f57066bc552c38c22 100644 (file)
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -150,6 +150,7 @@ struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
 struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
                                        struct cgroup_subsys_state **dst_cssp);
 
+void cgroup_enable_task_cg_lists(void);
 void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,
                         struct css_task_iter *it);
 struct task_struct *css_task_iter_next(struct css_task_iter *it);

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 9f51932bd543f68e0d18d93dfe7c1bdaa047e616..b94ad92dfbe6a11853d5aa1c12db709838e311a3 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -295,6 +295,11 @@ enum uclamp_id {
        UCLAMP_CNT
 };
 
+#ifdef CONFIG_SMP
+extern struct root_domain def_root_domain;
+extern struct mutex sched_domains_mutex;
+#endif
+
 struct sched_info {
 #ifdef CONFIG_SCHED_INFO
        /* Cumulative counters: */

diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h
index 0cb034331cbb80e592da0d52eec8d9ae63d2e6ba..1aff00b65f3cb92c145d764e27eb81329cddeb50 100644 (file)
--- a/include/linux/sched/deadline.h
+++ b/include/linux/sched/deadline.h
@@ -24,3 +24,11 @@ static inline bool dl_time_before(u64 a, u64 b)
 {
        return (s64)(a - b) < 0;
 }
+
+#ifdef CONFIG_SMP
+
+struct root_domain;
+extern void dl_add_task_root_domain(struct task_struct *p);
+extern void dl_clear_root_domain(struct root_domain *rd);
+
+#endif /* CONFIG_SMP */

diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 753afbca549fdaeba9c133c192c1927d1d9c6f3c..4b5bc452176ca83f807fb5055ecff7ea15b01c64 100644 (file)
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -1891,7 +1891,7 @@ static int cgroup_reconfigure(struct fs_context *fc)
  */
 static bool use_task_css_set_links __read_mostly;
 
-static void cgroup_enable_task_cg_lists(void)
+void cgroup_enable_task_cg_lists(void)
 {
        struct task_struct *p, *g;
 

diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 5aa37531ce76fd3a02896dafa6550d0233c19822..846cbdb68566583ccc95d4d22170d40cfb1fa8b7 100644 (file)
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -45,6 +45,7 @@
 #include <linux/proc_fs.h>
 #include <linux/rcupdate.h>
 #include <linux/sched.h>
+#include <linux/sched/deadline.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/task.h>
 #include <linux/seq_file.h>
@@ -894,6 +895,67 @@ static int generate_sched_domains(cpumask_var_t **domains,
        return ndoms;
 }
 
+static void update_tasks_root_domain(struct cpuset *cs)
+{
+       struct css_task_iter it;
+       struct task_struct *task;
+
+       css_task_iter_start(&cs->css, 0, &it);
+
+       while ((task = css_task_iter_next(&it)))
+               dl_add_task_root_domain(task);
+
+       css_task_iter_end(&it);
+}
+
+static void rebuild_root_domains(void)
+{
+       struct cpuset *cs = NULL;
+       struct cgroup_subsys_state *pos_css;
+
+       lockdep_assert_held(&cpuset_mutex);
+       lockdep_assert_cpus_held();
+       lockdep_assert_held(&sched_domains_mutex);
+
+       cgroup_enable_task_cg_lists();
+
+       rcu_read_lock();
+
+       /*
+        * Clear default root domain DL accounting, it will be computed again
+        * if a task belongs to it.
+        */
+       dl_clear_root_domain(&def_root_domain);
+
+       cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
+
+               if (cpumask_empty(cs->effective_cpus)) {
+                       pos_css = css_rightmost_descendant(pos_css);
+                       continue;
+               }
+
+               css_get(&cs->css);
+
+               rcu_read_unlock();
+
+               update_tasks_root_domain(cs);
+
+               rcu_read_lock();
+               css_put(&cs->css);
+       }
+       rcu_read_unlock();
+}
+
+static void
+partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
+                                   struct sched_domain_attr *dattr_new)
+{
+       mutex_lock(&sched_domains_mutex);
+       partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
+       rebuild_root_domains();
+       mutex_unlock(&sched_domains_mutex);
+}
+
 /*
  * Rebuild scheduler domains.
  *
@@ -931,7 +993,7 @@ static void rebuild_sched_domains_locked(void)
        ndoms = generate_sched_domains(&doms, &attr);
 
        /* Have scheduler rebuild the domains */
-       partition_sched_domains(ndoms, doms, attr);
+       partition_and_rebuild_sched_domains(ndoms, doms, attr);
 out:
        put_online_cpus();
 }

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index ef5b9f6b1d421bbd3e88e5300fa8f0ceeb448cbd..0f9d2180be237bd48ae5f808a476f77afc81f5e5 100644 (file)
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -2283,6 +2283,36 @@ void __init init_sched_dl_class(void)
                                        GFP_KERNEL, cpu_to_node(i));
 }
 
+void dl_add_task_root_domain(struct task_struct *p)
+{
+       struct rq_flags rf;
+       struct rq *rq;
+       struct dl_bw *dl_b;
+
+       rq = task_rq_lock(p, &rf);
+       if (!dl_task(p))
+               goto unlock;
+
+       dl_b = &rq->rd->dl_bw;
+       raw_spin_lock(&dl_b->lock);
+
+       __dl_add(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span));
+
+       raw_spin_unlock(&dl_b->lock);
+
+unlock:
+       task_rq_unlock(rq, p, &rf);
+}
+
+void dl_clear_root_domain(struct root_domain *rd)
+{
+       unsigned long flags;
+
+       raw_spin_lock_irqsave(&rd->dl_bw.lock, flags);
+       rd->dl_bw.total_bw = 0;
+       raw_spin_unlock_irqrestore(&rd->dl_bw.lock, flags);
+}
+
 #endif /* CONFIG_SMP */
 
 static void switched_from_dl(struct rq *rq, struct task_struct *p)

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 16126efd14eda8a2182347615a305a24cd3e2551..7583faddba335a438da60617c11650fcb6293a67 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -778,9 +778,6 @@ struct root_domain {
        struct perf_domain __rcu *pd;
 };
 
-extern struct root_domain def_root_domain;
-extern struct mutex sched_domains_mutex;
-
 extern void init_defrootdomain(void);
 extern int sched_init_domains(const struct cpumask *cpu_map);
 extern void rq_attach_root(struct rq *rq, struct root_domain *rd);

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 5a174ae6ecf3bb8d7787b3ea84a0c8fec6a1bfa6..8f83e8e3ea9abe75eeb72040a5569d1bf074f76e 100644 (file)
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -2203,8 +2203,19 @@ void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[],
        for (i = 0; i < ndoms_cur; i++) {
                for (j = 0; j < n && !new_topology; j++) {
                        if (cpumask_equal(doms_cur[i], doms_new[j]) &&
-                           dattrs_equal(dattr_cur, i, dattr_new, j))
+                           dattrs_equal(dattr_cur, i, dattr_new, j)) {
+                               struct root_domain *rd;
+
+                               /*
+                                * This domain won't be destroyed and as such
+                                * its dl_bw->total_bw needs to be cleared.  It
+                                * will be recomputed in function
+                                * update_tasks_root_domain().
+                                */
+                               rd = cpu_rq(cpumask_any(doms_cur[i]))->rd;
+                               dl_clear_root_domain(rd);
                                goto match1;
+                       }
                }
                /* No match - a current sched domain not in new doms_new[] */
                detach_destroy_domains(doms_cur[i]);