sched/cpufreq, sched/uclamp: Add clamps for FAIR and RT tasks

Each time a frequency update is required via schedutil, a frequency is
selected to (possibly) satisfy the utilization reported by each
scheduling class and irqs. However, when utilization clamping is in use,
the frequency selection should consider userspace utilization clamping
hints.  This will allow, for example, to:

 - boost tasks which are directly affecting the user experience
   by running them at least at a minimum "requested" frequency

 - cap low priority tasks not directly affecting the user experience
   by running them only up to a maximum "allowed" frequency

These constraints are meant to support a per-task based tuning of the
frequency selection thus supporting a fine grained definition of
performance boosting vs energy saving strategies in kernel space.

Add support to clamp the utilization of RUNNABLE FAIR and RT tasks
within the boundaries defined by their aggregated utilization clamp
constraints.

Do that by considering the max(min_util, max_util) to give boosted tasks
the performance they need even when they happen to be co-scheduled with
other capped tasks.

Signed-off-by: Patrick Bellasi <patrick.bellasi@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alessio Balsini <balsini@android.com>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Joel Fernandes <joelaf@google.com>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Paul Turner <pjt@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Quentin Perret <quentin.perret@arm.com>
Cc: Rafael J . Wysocki <rafael.j.wysocki@intel.com>
Cc: Steve Muckle <smuckle@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Todd Kjos <tkjos@google.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Viresh Kumar <viresh.kumar@linaro.org>
Link: https://lkml.kernel.org/r/20190621084217.8167-10-patrick.bellasi@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 7c4ce69067c4000320dd1059577efa39e897ff64..d84e036a7536bf4fd74abdd0cc409a7f3bbf203a 100644 (file)
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -202,8 +202,10 @@ unsigned long schedutil_freq_util(int cpu, unsigned long util_cfs,
        unsigned long dl_util, util, irq;
        struct rq *rq = cpu_rq(cpu);
 
-       if (type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt))
+       if (!IS_BUILTIN(CONFIG_UCLAMP_TASK) &&
+           type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) {
                return max;
+       }
 
        /*
         * Early check to see if IRQ/steal time saturates the CPU, can be
@@ -219,9 +221,16 @@ unsigned long schedutil_freq_util(int cpu, unsigned long util_cfs,
         * CFS tasks and we use the same metric to track the effective
         * utilization (PELT windows are synchronized) we can directly add them
         * to obtain the CPU's actual utilization.
+        *
+        * CFS and RT utilization can be boosted or capped, depending on
+        * utilization clamp constraints requested by currently RUNNABLE
+        * tasks.
+        * When there are no CFS RUNNABLE tasks, clamps are released and
+        * frequency will be gracefully reduced with the utilization decay.
         */
-       util = util_cfs;
-       util += cpu_util_rt(rq);
+       util = util_cfs + cpu_util_rt(rq);
+       if (type == FREQUENCY_UTIL)
+               util = uclamp_util(rq, util);
 
        dl_util = cpu_util_dl(rq);
 

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3bdcd3c718bcf5739174ad7716d340c36b118dac..28db7ce5c3a6915924f1bc0b8b2b8e9e15348331 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -10393,6 +10393,10 @@ const struct sched_class fair_sched_class = {
 #ifdef CONFIG_FAIR_GROUP_SCHED
        .task_change_group      = task_change_group_fair,
 #endif
+
+#ifdef CONFIG_UCLAMP_TASK
+       .uclamp_enabled         = 1,
+#endif
 };
 
 #ifdef CONFIG_SCHED_DEBUG

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 63ad7c90822c1d03fe748c6fb5ea010f61b9d2fd..a532558a51768de0ec30f9492764fca7292c5058 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -2400,6 +2400,10 @@ const struct sched_class rt_sched_class = {
        .switched_to            = switched_to_rt,
 
        .update_curr            = update_curr_rt,
+
+#ifdef CONFIG_UCLAMP_TASK
+       .uclamp_enabled         = 1,
+#endif
 };
 
 #ifdef CONFIG_RT_GROUP_SCHED

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 0d2ba8bb2cb3a54e85d2bdb40a31886af5ea3de4..9b0c77a99346984d05b84dc0d697095dfb09fb7f 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2265,6 +2265,29 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
 #endif /* CONFIG_CPU_FREQ */
 
+#ifdef CONFIG_UCLAMP_TASK
+static inline unsigned int uclamp_util(struct rq *rq, unsigned int util)
+{
+       unsigned int min_util = READ_ONCE(rq->uclamp[UCLAMP_MIN].value);
+       unsigned int max_util = READ_ONCE(rq->uclamp[UCLAMP_MAX].value);
+
+       /*
+        * Since CPU's {min,max}_util clamps are MAX aggregated considering
+        * RUNNABLE tasks with _different_ clamps, we can end up with an
+        * inversion. Fix it now when the clamps are applied.
+        */
+       if (unlikely(min_util >= max_util))
+               return min_util;
+
+       return clamp(util, min_util, max_util);
+}
+#else /* CONFIG_UCLAMP_TASK */
+static inline unsigned int uclamp_util(struct rq *rq, unsigned int util)
+{
+       return util;
+}
+#endif /* CONFIG_UCLAMP_TASK */
+
 #ifdef arch_scale_freq_capacity
 # ifndef arch_scale_freq_invariant
 #  define arch_scale_freq_invariant()  true