kernel/time/tick-broadcast.c

   1 /*
   2  * linux/kernel/time/tick-broadcast.c
   3  *
   4  * This file contains functions which emulate a local clock-event
   5  * device via a broadcast event source.
   6  *
   7  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   8  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   9  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10  *
  11  * This code is licenced under the GPL version 2. For details see
  12  * kernel-base/COPYING.
  13  */
  14 #include <linux/cpu.h>
  15 #include <linux/err.h>
  16 #include <linux/hrtimer.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/percpu.h>
  19 #include <linux/profile.h>
  20 #include <linux/sched.h>
  21 #include <linux/smp.h>
  22 #include <linux/module.h>
  23
  24 #include "tick-internal.h"
  25
  26 /*
  27  * Broadcast support for broken x86 hardware, where the local apic
  28  * timer stops in C3 state.
  29  */
  30
  31 static struct tick_device tick_broadcast_device;
  32 static cpumask_var_t tick_broadcast_mask;
  33 static cpumask_var_t tick_broadcast_on;
  34 static cpumask_var_t tmpmask;
  35 static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  36 static int tick_broadcast_forced;
  37
  38 #ifdef CONFIG_TICK_ONESHOT
  39 static void tick_broadcast_clear_oneshot(int cpu);
  40 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
  41 #else
  42 static inline void tick_broadcast_clear_oneshot(int cpu) { }
  43 static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
  44 #endif
  45
  46 /*
  47  * Debugging: see timer_list.c
  48  */
  49 struct tick_device *tick_get_broadcast_device(void)
  50 {
  51         return &tick_broadcast_device;
  52 }
  53
  54 struct cpumask *tick_get_broadcast_mask(void)
  55 {
  56         return tick_broadcast_mask;
  57 }
  58
  59 /*
  60  * Start the device in periodic mode
  61  */
  62 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  63 {
  64         if (bc)
  65                 tick_setup_periodic(bc, 1);
  66 }
  67
  68 /*
  69  * Check, if the device can be utilized as broadcast device:
  70  */
  71 static bool tick_check_broadcast_device(struct clock_event_device *curdev,
  72                                         struct clock_event_device *newdev)
  73 {
  74         if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
  75             (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
  76             (newdev->features & CLOCK_EVT_FEAT_C3STOP))
  77                 return false;
  78
  79         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
  80             !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
  81                 return false;
  82
  83         return !curdev || newdev->rating > curdev->rating;
  84 }
  85
  86 /*
  87  * Conditionally install/replace broadcast device
  88  */
  89 void tick_install_broadcast_device(struct clock_event_device *dev)
  90 {
  91         struct clock_event_device *cur = tick_broadcast_device.evtdev;
  92
  93         if (!tick_check_broadcast_device(cur, dev))
  94                 return;
  95
  96         if (!try_module_get(dev->owner))
  97                 return;
  98
  99         clockevents_exchange_device(cur, dev);
 100         if (cur)
 101                 cur->event_handler = clockevents_handle_noop;
 102         tick_broadcast_device.evtdev = dev;
 103         if (!cpumask_empty(tick_broadcast_mask))
 104                 tick_broadcast_start_periodic(dev);
 105         /*
 106          * Inform all cpus about this. We might be in a situation
 107          * where we did not switch to oneshot mode because the per cpu
 108          * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
 109          * of a oneshot capable broadcast device. Without that
 110          * notification the systems stays stuck in periodic mode
 111          * forever.
 112          */
 113         if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
 114                 tick_clock_notify();
 115 }
 116
 117 /*
 118  * Check, if the device is the broadcast device
 119  */
 120 int tick_is_broadcast_device(struct clock_event_device *dev)
 121 {
 122         return (dev && tick_broadcast_device.evtdev == dev);
 123 }
 124
 125 int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
 126 {
 127         int ret = -ENODEV;
 128
 129         if (tick_is_broadcast_device(dev)) {
 130                 raw_spin_lock(&tick_broadcast_lock);
 131                 ret = __clockevents_update_freq(dev, freq);
 132                 raw_spin_unlock(&tick_broadcast_lock);
 133         }
 134         return ret;
 135 }
 136
 137
 138 static void err_broadcast(const struct cpumask *mask)
 139 {
 140         pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
 141 }
 142
 143 static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
 144 {
 145         if (!dev->broadcast)
 146                 dev->broadcast = tick_broadcast;
 147         if (!dev->broadcast) {
 148                 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
 149                              dev->name);
 150                 dev->broadcast = err_broadcast;
 151         }
 152 }
 153
 154 /*
 155  * Check, if the device is disfunctional and a place holder, which
 156  * needs to be handled by the broadcast device.
 157  */
 158 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
 159 {
 160         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 161         unsigned long flags;
 162         int ret = 0;
 163
 164         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 165
 166         /*
 167          * Devices might be registered with both periodic and oneshot
 168          * mode disabled. This signals, that the device needs to be
 169          * operated from the broadcast device and is a placeholder for
 170          * the cpu local device.
 171          */
 172         if (!tick_device_is_functional(dev)) {
 173                 dev->event_handler = tick_handle_periodic;
 174                 tick_device_setup_broadcast_func(dev);
 175                 cpumask_set_cpu(cpu, tick_broadcast_mask);
 176                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 177                         tick_broadcast_start_periodic(bc);
 178                 else
 179                         tick_broadcast_setup_oneshot(bc);
 180                 ret = 1;
 181         } else {
 182                 /*
 183                  * Clear the broadcast bit for this cpu if the
 184                  * device is not power state affected.
 185                  */
 186                 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 187                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 188                 else
 189                         tick_device_setup_broadcast_func(dev);
 190
 191                 /*
 192                  * Clear the broadcast bit if the CPU is not in
 193                  * periodic broadcast on state.
 194                  */
 195                 if (!cpumask_test_cpu(cpu, tick_broadcast_on))
 196                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 197
 198                 switch (tick_broadcast_device.mode) {
 199                 case TICKDEV_MODE_ONESHOT:
 200                         /*
 201                          * If the system is in oneshot mode we can
 202                          * unconditionally clear the oneshot mask bit,
 203                          * because the CPU is running and therefore
 204                          * not in an idle state which causes the power
 205                          * state affected device to stop. Let the
 206                          * caller initialize the device.
 207                          */
 208                         tick_broadcast_clear_oneshot(cpu);
 209                         ret = 0;
 210                         break;
 211
 212                 case TICKDEV_MODE_PERIODIC:
 213                         /*
 214                          * If the system is in periodic mode, check
 215                          * whether the broadcast device can be
 216                          * switched off now.
 217                          */
 218                         if (cpumask_empty(tick_broadcast_mask) && bc)
 219                                 clockevents_shutdown(bc);
 220                         /*
 221                          * If we kept the cpu in the broadcast mask,
 222                          * tell the caller to leave the per cpu device
 223                          * in shutdown state. The periodic interrupt
 224                          * is delivered by the broadcast device, if
 225                          * the broadcast device exists and is not
 226                          * hrtimer based.
 227                          */
 228                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 229                                 ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
 230                         break;
 231                 default:
 232                         break;
 233                 }
 234         }
 235         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 236         return ret;
 237 }
 238
 239 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 240 int tick_receive_broadcast(void)
 241 {
 242         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 243         struct clock_event_device *evt = td->evtdev;
 244
 245         if (!evt)
 246                 return -ENODEV;
 247
 248         if (!evt->event_handler)
 249                 return -EINVAL;
 250
 251         evt->event_handler(evt);
 252         return 0;
 253 }
 254 #endif
 255
 256 /*
 257  * Broadcast the event to the cpus, which are set in the mask (mangled).
 258  */
 259 static bool tick_do_broadcast(struct cpumask *mask)
 260 {
 261         int cpu = smp_processor_id();
 262         struct tick_device *td;
 263         bool local = false;
 264
 265         /*
 266          * Check, if the current cpu is in the mask
 267          */
 268         if (cpumask_test_cpu(cpu, mask)) {
 269                 struct clock_event_device *bc = tick_broadcast_device.evtdev;
 270
 271                 cpumask_clear_cpu(cpu, mask);
 272                 /*
 273                  * We only run the local handler, if the broadcast
 274                  * device is not hrtimer based. Otherwise we run into
 275                  * a hrtimer recursion.
 276                  *
 277                  * local timer_interrupt()
 278                  *   local_handler()
 279                  *     expire_hrtimers()
 280                  *       bc_handler()
 281                  *         local_handler()
 282                  *           expire_hrtimers()
 283                  */
 284                 local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
 285         }
 286
 287         if (!cpumask_empty(mask)) {
 288                 /*
 289                  * It might be necessary to actually check whether the devices
 290                  * have different broadcast functions. For now, just use the
 291                  * one of the first device. This works as long as we have this
 292                  * misfeature only on x86 (lapic)
 293                  */
 294                 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
 295                 td->evtdev->broadcast(mask);
 296         }
 297         return local;
 298 }
 299
 300 /*
 301  * Periodic broadcast:
 302  * - invoke the broadcast handlers
 303  */
 304 static bool tick_do_periodic_broadcast(void)
 305 {
 306         cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
 307         return tick_do_broadcast(tmpmask);
 308 }
 309
 310 /*
 311  * Event handler for periodic broadcast ticks
 312  */
 313 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 314 {
 315         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 316         bool bc_local;
 317
 318         raw_spin_lock(&tick_broadcast_lock);
 319
 320         /* Handle spurious interrupts gracefully */
 321         if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
 322                 raw_spin_unlock(&tick_broadcast_lock);
 323                 return;
 324         }
 325
 326         bc_local = tick_do_periodic_broadcast();
 327
 328         if (clockevent_state_oneshot(dev)) {
 329                 ktime_t next = ktime_add(dev->next_event, tick_period);
 330
 331                 clockevents_program_event(dev, next, true);
 332         }
 333         raw_spin_unlock(&tick_broadcast_lock);
 334
 335         /*
 336          * We run the handler of the local cpu after dropping
 337          * tick_broadcast_lock because the handler might deadlock when
 338          * trying to switch to oneshot mode.
 339          */
 340         if (bc_local)
 341                 td->evtdev->event_handler(td->evtdev);
 342 }
 343
 344 /**
 345  * tick_broadcast_control - Enable/disable or force broadcast mode
 346  * @mode:       The selected broadcast mode
 347  *
 348  * Called when the system enters a state where affected tick devices
 349  * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
 350  */
 351 void tick_broadcast_control(enum tick_broadcast_mode mode)
 352 {
 353         struct clock_event_device *bc, *dev;
 354         struct tick_device *td;
 355         int cpu, bc_stopped;
 356         unsigned long flags;
 357
 358         /* Protects also the local clockevent device. */
 359         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 360         td = this_cpu_ptr(&tick_cpu_device);
 361         dev = td->evtdev;
 362
 363         /*
 364          * Is the device not affected by the powerstate ?
 365          */
 366         if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 367                 goto out;
 368
 369         if (!tick_device_is_functional(dev))
 370                 goto out;
 371
 372         cpu = smp_processor_id();
 373         bc = tick_broadcast_device.evtdev;
 374         bc_stopped = cpumask_empty(tick_broadcast_mask);
 375
 376         switch (mode) {
 377         case TICK_BROADCAST_FORCE:
 378                 tick_broadcast_forced = 1;
 379         case TICK_BROADCAST_ON:
 380                 cpumask_set_cpu(cpu, tick_broadcast_on);
 381                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
 382                         /*
 383                          * Only shutdown the cpu local device, if:
 384                          *
 385                          * - the broadcast device exists
 386                          * - the broadcast device is not a hrtimer based one
 387                          * - the broadcast device is in periodic mode to
 388                          *   avoid a hickup during switch to oneshot mode
 389                          */
 390                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
 391                             tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 392                                 clockevents_shutdown(dev);
 393                 }
 394                 break;
 395
 396         case TICK_BROADCAST_OFF:
 397                 if (tick_broadcast_forced)
 398                         break;
 399                 cpumask_clear_cpu(cpu, tick_broadcast_on);
 400                 if (!tick_device_is_functional(dev))
 401                         break;
 402                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
 403                         if (tick_broadcast_device.mode ==
 404                             TICKDEV_MODE_PERIODIC)
 405                                 tick_setup_periodic(dev, 0);
 406                 }
 407                 break;
 408         }
 409
 410         if (bc) {
 411                 if (cpumask_empty(tick_broadcast_mask)) {
 412                         if (!bc_stopped)
 413                                 clockevents_shutdown(bc);
 414                 } else if (bc_stopped) {
 415                         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 416                                 tick_broadcast_start_periodic(bc);
 417                         else
 418                                 tick_broadcast_setup_oneshot(bc);
 419                 }
 420         }
 421 out:
 422         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 423 }
 424 EXPORT_SYMBOL_GPL(tick_broadcast_control);
 425
 426 /*
 427  * Set the periodic handler depending on broadcast on/off
 428  */
 429 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 430 {
 431         if (!broadcast)
 432                 dev->event_handler = tick_handle_periodic;
 433         else
 434                 dev->event_handler = tick_handle_periodic_broadcast;
 435 }
 436
 437 #ifdef CONFIG_HOTPLUG_CPU
 438 /*
 439  * Remove a CPU from broadcasting
 440  */
 441 void tick_shutdown_broadcast(unsigned int cpu)
 442 {
 443         struct clock_event_device *bc;
 444         unsigned long flags;
 445
 446         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 447
 448         bc = tick_broadcast_device.evtdev;
 449         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 450         cpumask_clear_cpu(cpu, tick_broadcast_on);
 451
 452         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 453                 if (bc && cpumask_empty(tick_broadcast_mask))
 454                         clockevents_shutdown(bc);
 455         }
 456
 457         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 458 }
 459 #endif
 460
 461 void tick_suspend_broadcast(void)
 462 {
 463         struct clock_event_device *bc;
 464         unsigned long flags;
 465
 466         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 467
 468         bc = tick_broadcast_device.evtdev;
 469         if (bc)
 470                 clockevents_shutdown(bc);
 471
 472         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 473 }
 474
 475 /*
 476  * This is called from tick_resume_local() on a resuming CPU. That's
 477  * called from the core resume function, tick_unfreeze() and the magic XEN
 478  * resume hackery.
 479  *
 480  * In none of these cases the broadcast device mode can change and the
 481  * bit of the resuming CPU in the broadcast mask is safe as well.
 482  */
 483 bool tick_resume_check_broadcast(void)
 484 {
 485         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
 486                 return false;
 487         else
 488                 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
 489 }
 490
 491 void tick_resume_broadcast(void)
 492 {
 493         struct clock_event_device *bc;
 494         unsigned long flags;
 495
 496         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 497
 498         bc = tick_broadcast_device.evtdev;
 499
 500         if (bc) {
 501                 clockevents_tick_resume(bc);
 502
 503                 switch (tick_broadcast_device.mode) {
 504                 case TICKDEV_MODE_PERIODIC:
 505                         if (!cpumask_empty(tick_broadcast_mask))
 506                                 tick_broadcast_start_periodic(bc);
 507                         break;
 508                 case TICKDEV_MODE_ONESHOT:
 509                         if (!cpumask_empty(tick_broadcast_mask))
 510                                 tick_resume_broadcast_oneshot(bc);
 511                         break;
 512                 }
 513         }
 514         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 515 }
 516
 517 #ifdef CONFIG_TICK_ONESHOT
 518
 519 static cpumask_var_t tick_broadcast_oneshot_mask;
 520 static cpumask_var_t tick_broadcast_pending_mask;
 521 static cpumask_var_t tick_broadcast_force_mask;
 522
 523 /*
 524  * Exposed for debugging: see timer_list.c
 525  */
 526 struct cpumask *tick_get_broadcast_oneshot_mask(void)
 527 {
 528         return tick_broadcast_oneshot_mask;
 529 }
 530
 531 /*
 532  * Called before going idle with interrupts disabled. Checks whether a
 533  * broadcast event from the other core is about to happen. We detected
 534  * that in tick_broadcast_oneshot_control(). The callsite can use this
 535  * to avoid a deep idle transition as we are about to get the
 536  * broadcast IPI right away.
 537  */
 538 int tick_check_broadcast_expired(void)
 539 {
 540         return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
 541 }
 542
 543 /*
 544  * Set broadcast interrupt affinity
 545  */
 546 static void tick_broadcast_set_affinity(struct clock_event_device *bc,
 547                                         const struct cpumask *cpumask)
 548 {
 549         if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
 550                 return;
 551
 552         if (cpumask_equal(bc->cpumask, cpumask))
 553                 return;
 554
 555         bc->cpumask = cpumask;
 556         irq_set_affinity(bc->irq, bc->cpumask);
 557 }
 558
 559 static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
 560                                      ktime_t expires)
 561 {
 562         if (!clockevent_state_oneshot(bc))
 563                 clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 564
 565         clockevents_program_event(bc, expires, 1);
 566         tick_broadcast_set_affinity(bc, cpumask_of(cpu));
 567 }
 568
 569 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
 570 {
 571         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 572 }
 573
 574 /*
 575  * Called from irq_enter() when idle was interrupted to reenable the
 576  * per cpu device.
 577  */
 578 void tick_check_oneshot_broadcast_this_cpu(void)
 579 {
 580         if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
 581                 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 582
 583                 /*
 584                  * We might be in the middle of switching over from
 585                  * periodic to oneshot. If the CPU has not yet
 586                  * switched over, leave the device alone.
 587                  */
 588                 if (td->mode == TICKDEV_MODE_ONESHOT) {
 589                         clockevents_switch_state(td->evtdev,
 590                                               CLOCK_EVT_STATE_ONESHOT);
 591                 }
 592         }
 593 }
 594
 595 /*
 596  * Handle oneshot mode broadcasting
 597  */
 598 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 599 {
 600         struct tick_device *td;
 601         ktime_t now, next_event;
 602         int cpu, next_cpu = 0;
 603         bool bc_local;
 604
 605         raw_spin_lock(&tick_broadcast_lock);
 606         dev->next_event = KTIME_MAX;
 607         next_event = KTIME_MAX;
 608         cpumask_clear(tmpmask);
 609         now = ktime_get();
 610         /* Find all expired events */
 611         for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
 612                 td = &per_cpu(tick_cpu_device, cpu);
 613                 if (td->evtdev->next_event <= now) {
 614                         cpumask_set_cpu(cpu, tmpmask);
 615                         /*
 616                          * Mark the remote cpu in the pending mask, so
 617                          * it can avoid reprogramming the cpu local
 618                          * timer in tick_broadcast_oneshot_control().
 619                          */
 620                         cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
 621                 } else if (td->evtdev->next_event < next_event) {
 622                         next_event = td->evtdev->next_event;
 623                         next_cpu = cpu;
 624                 }
 625         }
 626
 627         /*
 628          * Remove the current cpu from the pending mask. The event is
 629          * delivered immediately in tick_do_broadcast() !
 630          */
 631         cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
 632
 633         /* Take care of enforced broadcast requests */
 634         cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
 635         cpumask_clear(tick_broadcast_force_mask);
 636
 637         /*
 638          * Sanity check. Catch the case where we try to broadcast to
 639          * offline cpus.
 640          */
 641         if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
 642                 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
 643
 644         /*
 645          * Wakeup the cpus which have an expired event.
 646          */
 647         bc_local = tick_do_broadcast(tmpmask);
 648
 649         /*
 650          * Two reasons for reprogram:
 651          *
 652          * - The global event did not expire any CPU local
 653          * events. This happens in dyntick mode, as the maximum PIT
 654          * delta is quite small.
 655          *
 656          * - There are pending events on sleeping CPUs which were not
 657          * in the event mask
 658          */
 659         if (next_event != KTIME_MAX)
 660                 tick_broadcast_set_event(dev, next_cpu, next_event);
 661
 662         raw_spin_unlock(&tick_broadcast_lock);
 663
 664         if (bc_local) {
 665                 td = this_cpu_ptr(&tick_cpu_device);
 666                 td->evtdev->event_handler(td->evtdev);
 667         }
 668 }
 669
 670 static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
 671 {
 672         if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 673                 return 0;
 674         if (bc->next_event == KTIME_MAX)
 675                 return 0;
 676         return bc->bound_on == cpu ? -EBUSY : 0;
 677 }
 678
 679 static void broadcast_shutdown_local(struct clock_event_device *bc,
 680                                      struct clock_event_device *dev)
 681 {
 682         /*
 683          * For hrtimer based broadcasting we cannot shutdown the cpu
 684          * local device if our own event is the first one to expire or
 685          * if we own the broadcast timer.
 686          */
 687         if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
 688                 if (broadcast_needs_cpu(bc, smp_processor_id()))
 689                         return;
 690                 if (dev->next_event < bc->next_event)
 691                         return;
 692         }
 693         clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
 694 }
 695
 696 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 697 {
 698         struct clock_event_device *bc, *dev;
 699         int cpu, ret = 0;
 700         ktime_t now;
 701
 702         /*
 703          * If there is no broadcast device, tell the caller not to go
 704          * into deep idle.
 705          */
 706         if (!tick_broadcast_device.evtdev)
 707                 return -EBUSY;
 708
 709         dev = this_cpu_ptr(&tick_cpu_device)->evtdev;
 710
 711         raw_spin_lock(&tick_broadcast_lock);
 712         bc = tick_broadcast_device.evtdev;
 713         cpu = smp_processor_id();
 714
 715         if (state == TICK_BROADCAST_ENTER) {
 716                 /*
 717                  * If the current CPU owns the hrtimer broadcast
 718                  * mechanism, it cannot go deep idle and we do not add
 719                  * the CPU to the broadcast mask. We don't have to go
 720                  * through the EXIT path as the local timer is not
 721                  * shutdown.
 722                  */
 723                 ret = broadcast_needs_cpu(bc, cpu);
 724                 if (ret)
 725                         goto out;
 726
 727                 /*
 728                  * If the broadcast device is in periodic mode, we
 729                  * return.
 730                  */
 731                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 732                         /* If it is a hrtimer based broadcast, return busy */
 733                         if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
 734                                 ret = -EBUSY;
 735                         goto out;
 736                 }
 737
 738                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
 739                         WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
 740
 741                         /* Conditionally shut down the local timer. */
 742                         broadcast_shutdown_local(bc, dev);
 743
 744                         /*
 745                          * We only reprogram the broadcast timer if we
 746                          * did not mark ourself in the force mask and
 747                          * if the cpu local event is earlier than the
 748                          * broadcast event. If the current CPU is in
 749                          * the force mask, then we are going to be
 750                          * woken by the IPI right away; we return
 751                          * busy, so the CPU does not try to go deep
 752                          * idle.
 753                          */
 754                         if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
 755                                 ret = -EBUSY;
 756                         } else if (dev->next_event < bc->next_event) {
 757                                 tick_broadcast_set_event(bc, cpu, dev->next_event);
 758                                 /*
 759                                  * In case of hrtimer broadcasts the
 760                                  * programming might have moved the
 761                                  * timer to this cpu. If yes, remove
 762                                  * us from the broadcast mask and
 763                                  * return busy.
 764                                  */
 765                                 ret = broadcast_needs_cpu(bc, cpu);
 766                                 if (ret) {
 767                                         cpumask_clear_cpu(cpu,
 768                                                 tick_broadcast_oneshot_mask);
 769                                 }
 770                         }
 771                 }
 772         } else {
 773                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
 774                         clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
 775                         /*
 776                          * The cpu which was handling the broadcast
 777                          * timer marked this cpu in the broadcast
 778                          * pending mask and fired the broadcast
 779                          * IPI. So we are going to handle the expired
 780                          * event anyway via the broadcast IPI
 781                          * handler. No need to reprogram the timer
 782                          * with an already expired event.
 783                          */
 784                         if (cpumask_test_and_clear_cpu(cpu,
 785                                        tick_broadcast_pending_mask))
 786                                 goto out;
 787
 788                         /*
 789                          * Bail out if there is no next event.
 790                          */
 791                         if (dev->next_event == KTIME_MAX)
 792                                 goto out;
 793                         /*
 794                          * If the pending bit is not set, then we are
 795                          * either the CPU handling the broadcast
 796                          * interrupt or we got woken by something else.
 797                          *
 798                          * We are not longer in the broadcast mask, so
 799                          * if the cpu local expiry time is already
 800                          * reached, we would reprogram the cpu local
 801                          * timer with an already expired event.
 802                          *
 803                          * This can lead to a ping-pong when we return
 804                          * to idle and therefor rearm the broadcast
 805                          * timer before the cpu local timer was able
 806                          * to fire. This happens because the forced
 807                          * reprogramming makes sure that the event
 808                          * will happen in the future and depending on
 809                          * the min_delta setting this might be far
 810                          * enough out that the ping-pong starts.
 811                          *
 812                          * If the cpu local next_event has expired
 813                          * then we know that the broadcast timer
 814                          * next_event has expired as well and
 815                          * broadcast is about to be handled. So we
 816                          * avoid reprogramming and enforce that the
 817                          * broadcast handler, which did not run yet,
 818                          * will invoke the cpu local handler.
 819                          *
 820                          * We cannot call the handler directly from
 821                          * here, because we might be in a NOHZ phase
 822                          * and we did not go through the irq_enter()
 823                          * nohz fixups.
 824                          */
 825                         now = ktime_get();
 826                         if (dev->next_event <= now) {
 827                                 cpumask_set_cpu(cpu, tick_broadcast_force_mask);
 828                                 goto out;
 829                         }
 830                         /*
 831                          * We got woken by something else. Reprogram
 832                          * the cpu local timer device.
 833                          */
 834                         tick_program_event(dev->next_event, 1);
 835                 }
 836         }
 837 out:
 838         raw_spin_unlock(&tick_broadcast_lock);
 839         return ret;
 840 }
 841
 842 /*
 843  * Reset the one shot broadcast for a cpu
 844  *
 845  * Called with tick_broadcast_lock held
 846  */
 847 static void tick_broadcast_clear_oneshot(int cpu)
 848 {
 849         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 850         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 851 }
 852
 853 static void tick_broadcast_init_next_event(struct cpumask *mask,
 854                                            ktime_t expires)
 855 {
 856         struct tick_device *td;
 857         int cpu;
 858
 859         for_each_cpu(cpu, mask) {
 860                 td = &per_cpu(tick_cpu_device, cpu);
 861                 if (td->evtdev)
 862                         td->evtdev->next_event = expires;
 863         }
 864 }
 865
 866 /**
 867  * tick_broadcast_setup_oneshot - setup the broadcast device
 868  */
 869 void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 870 {
 871         int cpu = smp_processor_id();
 872
 873         if (!bc)
 874                 return;
 875
 876         /* Set it up only once ! */
 877         if (bc->event_handler != tick_handle_oneshot_broadcast) {
 878                 int was_periodic = clockevent_state_periodic(bc);
 879
 880                 bc->event_handler = tick_handle_oneshot_broadcast;
 881
 882                 /*
 883                  * We must be careful here. There might be other CPUs
 884                  * waiting for periodic broadcast. We need to set the
 885                  * oneshot_mask bits for those and program the
 886                  * broadcast device to fire.
 887                  */
 888                 cpumask_copy(tmpmask, tick_broadcast_mask);
 889                 cpumask_clear_cpu(cpu, tmpmask);
 890                 cpumask_or(tick_broadcast_oneshot_mask,
 891                            tick_broadcast_oneshot_mask, tmpmask);
 892
 893                 if (was_periodic && !cpumask_empty(tmpmask)) {
 894                         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 895                         tick_broadcast_init_next_event(tmpmask,
 896                                                        tick_next_period);
 897                         tick_broadcast_set_event(bc, cpu, tick_next_period);
 898                 } else
 899                         bc->next_event = KTIME_MAX;
 900         } else {
 901                 /*
 902                  * The first cpu which switches to oneshot mode sets
 903                  * the bit for all other cpus which are in the general
 904                  * (periodic) broadcast mask. So the bit is set and
 905                  * would prevent the first broadcast enter after this
 906                  * to program the bc device.
 907                  */
 908                 tick_broadcast_clear_oneshot(cpu);
 909         }
 910 }
 911
 912 /*
 913  * Select oneshot operating mode for the broadcast device
 914  */
 915 void tick_broadcast_switch_to_oneshot(void)
 916 {
 917         struct clock_event_device *bc;
 918         unsigned long flags;
 919
 920         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 921
 922         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 923         bc = tick_broadcast_device.evtdev;
 924         if (bc)
 925                 tick_broadcast_setup_oneshot(bc);
 926
 927         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 928 }
 929
 930 #ifdef CONFIG_HOTPLUG_CPU
 931 void hotplug_cpu__broadcast_tick_pull(int deadcpu)
 932 {
 933         struct clock_event_device *bc;
 934         unsigned long flags;
 935
 936         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 937         bc = tick_broadcast_device.evtdev;
 938
 939         if (bc && broadcast_needs_cpu(bc, deadcpu)) {
 940                 /* This moves the broadcast assignment to this CPU: */
 941                 clockevents_program_event(bc, bc->next_event, 1);
 942         }
 943         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 944 }
 945
 946 /*
 947  * Remove a dead CPU from broadcasting
 948  */
 949 void tick_shutdown_broadcast_oneshot(unsigned int cpu)
 950 {
 951         unsigned long flags;
 952
 953         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 954
 955         /*
 956          * Clear the broadcast masks for the dead cpu, but do not stop
 957          * the broadcast device!
 958          */
 959         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 960         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 961         cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
 962
 963         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 964 }
 965 #endif
 966
 967 /*
 968  * Check, whether the broadcast device is in one shot mode
 969  */
 970 int tick_broadcast_oneshot_active(void)
 971 {
 972         return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
 973 }
 974
 975 /*
 976  * Check whether the broadcast device supports oneshot.
 977  */
 978 bool tick_broadcast_oneshot_available(void)
 979 {
 980         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 981
 982         return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
 983 }
 984
 985 #else
 986 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 987 {
 988         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 989
 990         if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
 991                 return -EBUSY;
 992
 993         return 0;
 994 }
 995 #endif
 996
 997 void __init tick_broadcast_init(void)
 998 {
 999         zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
1000         zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
1001         zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
1002 #ifdef CONFIG_TICK_ONESHOT
1003         zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
1004         zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
1005         zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
1006 #endif
1007 }