Merge branches 'pm-core', 'pm-qos', 'pm-domains' and 'pm-opp'

[linux.git] / kernel / sched / clock.c

diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c
index b3466d4e0cc2d63ea607014610db6750057ace09..ad64efe41722bef0e3a056386f677de8ecd3eda9 100644 (file)
--- a/kernel/sched/clock.c
+++ b/kernel/sched/clock.c
@@ -83,8 +83,39 @@ void sched_clock_init(void)
 }
 
 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
+/*
+ * We must start with !__sched_clock_stable because the unstable -> stable
+ * transition is accurate, while the stable -> unstable transition is not.
+ *
+ * Similarly we start with __sched_clock_stable_early, thereby assuming we
+ * will become stable, such that there's only a single 1 -> 0 transition.
+ */
 static DEFINE_STATIC_KEY_FALSE(__sched_clock_stable);
-static int __sched_clock_stable_early;
+static int __sched_clock_stable_early = 1;
+
+/*
+ * We want: ktime_get_ns() + gtod_offset == sched_clock() + raw_offset
+ */
+static __read_mostly u64 raw_offset;
+static __read_mostly u64 gtod_offset;
+
+struct sched_clock_data {
+       u64                     tick_raw;
+       u64                     tick_gtod;
+       u64                     clock;
+};
+
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
+
+static inline struct sched_clock_data *this_scd(void)
+{
+       return this_cpu_ptr(&sched_clock_data);
+}
+
+static inline struct sched_clock_data *cpu_sdc(int cpu)
+{
+       return &per_cpu(sched_clock_data, cpu);
+}
 
 int sched_clock_stable(void)
 {
@@ -93,31 +124,40 @@ int sched_clock_stable(void)
 
 static void __set_sched_clock_stable(void)
 {
+       struct sched_clock_data *scd = this_scd();
+
+       /*
+        * Attempt to make the (initial) unstable->stable transition continuous.
+        */
+       raw_offset = (scd->tick_gtod + gtod_offset) - (scd->tick_raw);
+
+       printk(KERN_INFO "sched_clock: Marking stable (%lld, %lld)->(%lld, %lld)\n",
+                       scd->tick_gtod, gtod_offset,
+                       scd->tick_raw,  raw_offset);
+
        static_branch_enable(&__sched_clock_stable);
        tick_dep_clear(TICK_DEP_BIT_CLOCK_UNSTABLE);
 }
 
-void set_sched_clock_stable(void)
+static void __clear_sched_clock_stable(struct work_struct *work)
 {
-       __sched_clock_stable_early = 1;
-
-       smp_mb(); /* matches sched_clock_init_late() */
+       struct sched_clock_data *scd = this_scd();
 
        /*
-        * This really should only be called early (before
-        * sched_clock_init_late()) when guestimating our sched_clock() is
-        * solid.
+        * Attempt to make the stable->unstable transition continuous.
         *
-        * After that we test stability and we can negate our guess using
-        * clear_sched_clock_stable, possibly from a watchdog.
+        * Trouble is, this is typically called from the TSC watchdog
+        * timer, which is late per definition. This means the tick
+        * values can already be screwy.
+        *
+        * Still do what we can.
         */
-       if (WARN_ON_ONCE(sched_clock_running == 2))
-               __set_sched_clock_stable();
-}
+       gtod_offset = (scd->tick_raw + raw_offset) - (scd->tick_gtod);
+
+       printk(KERN_INFO "sched_clock: Marking unstable (%lld, %lld)<-(%lld, %lld)\n",
+                       scd->tick_gtod, gtod_offset,
+                       scd->tick_raw,  raw_offset);
 
-static void __clear_sched_clock_stable(struct work_struct *work)
-{
-       /* XXX worry about clock continuity */
        static_branch_disable(&__sched_clock_stable);
        tick_dep_set(TICK_DEP_BIT_CLOCK_UNSTABLE);
 }
@@ -134,28 +174,9 @@ void clear_sched_clock_stable(void)
                schedule_work(&sched_clock_work);
 }
 
-struct sched_clock_data {
-       u64                     tick_raw;
-       u64                     tick_gtod;
-       u64                     clock;
-};
-
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
-
-static inline struct sched_clock_data *this_scd(void)
-{
-       return this_cpu_ptr(&sched_clock_data);
-}
-
-static inline struct sched_clock_data *cpu_sdc(int cpu)
-{
-       return &per_cpu(sched_clock_data, cpu);
-}
-
 void sched_clock_init_late(void)
 {
        sched_clock_running = 2;
-
        /*
         * Ensure that it is impossible to not do a static_key update.
         *
@@ -167,8 +188,6 @@ void sched_clock_init_late(void)
 
        if (__sched_clock_stable_early)
                __set_sched_clock_stable();
-       else
-               __clear_sched_clock_stable(NULL);
 }
 
 /*
@@ -210,7 +229,7 @@ static u64 sched_clock_local(struct sched_clock_data *scd)
         *                    scd->tick_gtod + TICK_NSEC);
         */
 
-       clock = scd->tick_gtod + delta;
+       clock = scd->tick_gtod + gtod_offset + delta;
        min_clock = wrap_max(scd->tick_gtod, old_clock);
        max_clock = wrap_max(old_clock, scd->tick_gtod + TICK_NSEC);
 
@@ -296,7 +315,7 @@ u64 sched_clock_cpu(int cpu)
        u64 clock;
 
        if (sched_clock_stable())
-               return sched_clock();
+               return sched_clock() + raw_offset;
 
        if (unlikely(!sched_clock_running))
                return 0ull;
@@ -317,23 +336,22 @@ EXPORT_SYMBOL_GPL(sched_clock_cpu);
 void sched_clock_tick(void)
 {
        struct sched_clock_data *scd;
-       u64 now, now_gtod;
-
-       if (sched_clock_stable())
-               return;
-
-       if (unlikely(!sched_clock_running))
-               return;
 
        WARN_ON_ONCE(!irqs_disabled());
 
+       /*
+        * Update these values even if sched_clock_stable(), because it can
+        * become unstable at any point in time at which point we need some
+        * values to fall back on.
+        *
+        * XXX arguably we can skip this if we expose tsc_clocksource_reliable
+        */
        scd = this_scd();
-       now_gtod = ktime_to_ns(ktime_get());
-       now = sched_clock();
+       scd->tick_raw  = sched_clock();
+       scd->tick_gtod = ktime_get_ns();
 
-       scd->tick_raw = now;
-       scd->tick_gtod = now_gtod;
-       sched_clock_local(scd);
+       if (!sched_clock_stable() && likely(sched_clock_running))
+               sched_clock_local(scd);
 }
 
 /*