cpufreq: intel_pstate: Add HWP boost utility and sched util hooks

[linux.git] / drivers / cpufreq / intel_pstate.c

diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 08960a55eb27a43e3b99bc2c7d70518166d84a22..3949e3861f55cf10b5359a920ba52edc1e3a096f 100644 (file)
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -221,6 +221,9 @@ struct global_params {
  *                     preference/bias
  * @epp_saved:         Saved EPP/EPB during system suspend or CPU offline
  *                     operation
+ * @hwp_req_cached:    Cached value of the last HWP Request MSR
+ * @hwp_cap_cached:    Cached value of the last HWP Capabilities MSR
+ * @hwp_boost_min:     Last HWP boosted min performance
  *
  * This structure stores per CPU instance data for all CPUs.
  */
@@ -253,6 +256,9 @@ struct cpudata {
        s16 epp_policy;
        s16 epp_default;
        s16 epp_saved;
+       u64 hwp_req_cached;
+       u64 hwp_cap_cached;
+       u32 hwp_boost_min;
 };
 
 static struct cpudata **all_cpu_data;
@@ -285,6 +291,7 @@ static struct pstate_funcs pstate_funcs __read_mostly;
 
 static int hwp_active __read_mostly;
 static bool per_cpu_limits __read_mostly;
+static bool hwp_boost __read_mostly;
 
 static struct cpufreq_driver *intel_pstate_driver __read_mostly;
 
@@ -689,6 +696,7 @@ static void intel_pstate_get_hwp_max(unsigned int cpu, int *phy_max,
        u64 cap;
 
        rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
+       WRITE_ONCE(all_cpu_data[cpu]->hwp_cap_cached, cap);
        if (global.no_turbo)
                *current_max = HWP_GUARANTEED_PERF(cap);
        else
@@ -763,6 +771,7 @@ static void intel_pstate_hwp_set(unsigned int cpu)
                intel_pstate_set_epb(cpu, epp);
        }
 skip_epp:
+       WRITE_ONCE(cpu_data->hwp_req_cached, value);
        wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
 }
 
@@ -1381,6 +1390,81 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
        intel_pstate_set_min_pstate(cpu);
 }
 
+/*
+ * Long hold time will keep high perf limits for long time,
+ * which negatively impacts perf/watt for some workloads,
+ * like specpower. 3ms is based on experiements on some
+ * workoads.
+ */
+static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC;
+
+static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)
+{
+       u64 hwp_req = READ_ONCE(cpu->hwp_req_cached);
+       u32 max_limit = (hwp_req & 0xff00) >> 8;
+       u32 min_limit = (hwp_req & 0xff);
+       u32 boost_level1;
+
+       /*
+        * Cases to consider (User changes via sysfs or boot time):
+        * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
+        *      No boost, return.
+        * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
+        *     Should result in one level boost only for P0.
+        * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
+        *     Should result in two level boost:
+        *         (min + p1)/2 and P1.
+        * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
+        *     Should result in three level boost:
+        *        (min + p1)/2, P1 and P0.
+        */
+
+       /* If max and min are equal or already at max, nothing to boost */
+       if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit)
+               return;
+
+       if (!cpu->hwp_boost_min)
+               cpu->hwp_boost_min = min_limit;
+
+       /* level at half way mark between min and guranteed */
+       boost_level1 = (HWP_GUARANTEED_PERF(cpu->hwp_cap_cached) + min_limit) >> 1;
+
+       if (cpu->hwp_boost_min < boost_level1)
+               cpu->hwp_boost_min = boost_level1;
+       else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(cpu->hwp_cap_cached))
+               cpu->hwp_boost_min = HWP_GUARANTEED_PERF(cpu->hwp_cap_cached);
+       else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(cpu->hwp_cap_cached) &&
+                max_limit != HWP_GUARANTEED_PERF(cpu->hwp_cap_cached))
+               cpu->hwp_boost_min = max_limit;
+       else
+               return;
+
+       hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min;
+       wrmsrl(MSR_HWP_REQUEST, hwp_req);
+       cpu->last_update = cpu->sample.time;
+}
+
+static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)
+{
+       if (cpu->hwp_boost_min) {
+               bool expired;
+
+               /* Check if we are idle for hold time to boost down */
+               expired = time_after64(cpu->sample.time, cpu->last_update +
+                                      hwp_boost_hold_time_ns);
+               if (expired) {
+                       wrmsrl(MSR_HWP_REQUEST, cpu->hwp_req_cached);
+                       cpu->hwp_boost_min = 0;
+               }
+       }
+       cpu->last_update = cpu->sample.time;
+}
+
+static inline void intel_pstate_update_util_hwp(struct update_util_data *data,
+                                               u64 time, unsigned int flags)
+{
+}
+
 static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
 {
        struct sample *sample = &cpu->sample;
@@ -1684,7 +1768,7 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
 {
        struct cpudata *cpu = all_cpu_data[cpu_num];
 
-       if (hwp_active)
+       if (hwp_active && !hwp_boost)
                return;
 
        if (cpu->update_util_set)
@@ -1693,7 +1777,9 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
        /* Prevent intel_pstate_update_util() from using stale data. */
        cpu->sample.time = 0;
        cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
-                                    intel_pstate_update_util);
+                                    (hwp_active ?
+                                     intel_pstate_update_util_hwp :
+                                     intel_pstate_update_util));
        cpu->update_util_set = true;
 }
 
@@ -1805,8 +1891,16 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
                intel_pstate_set_update_util_hook(policy->cpu);
        }
 
-       if (hwp_active)
+       if (hwp_active) {
+               /*
+                * When hwp_boost was active before and dynamically it
+                * was turned off, in that case we need to clear the
+                * update util hook.
+                */
+               if (!hwp_boost)
+                       intel_pstate_clear_update_util_hook(policy->cpu);
                intel_pstate_hwp_set(policy->cpu);
+       }
 
        mutex_unlock(&intel_pstate_limits_lock);