kernel/time/tick-broadcast.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains functions which emulate a local clock-event
   4  * device via a broadcast event source.
   5  *
   6  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   7  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   8  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
   9  */
  10 #include <linux/cpu.h>
  11 #include <linux/err.h>
  12 #include <linux/hrtimer.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/percpu.h>
  15 #include <linux/profile.h>
  16 #include <linux/sched.h>
  17 #include <linux/smp.h>
  18 #include <linux/module.h>
  19
  20 #include "tick-internal.h"
  21
  22 /*
  23  * Broadcast support for broken x86 hardware, where the local apic
  24  * timer stops in C3 state.
  25  */
  26
  27 static struct tick_device tick_broadcast_device;
  28 static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
  29 static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
  30 static cpumask_var_t tmpmask __cpumask_var_read_mostly;
  31 static int tick_broadcast_forced;
  32
  33 static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  34
  35 #ifdef CONFIG_TICK_ONESHOT
  36 static void tick_broadcast_setup_oneshot(struct clock_event_device *bc);
  37 static void tick_broadcast_clear_oneshot(int cpu);
  38 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
  39 #else
  40 static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { BUG(); }
  41 static inline void tick_broadcast_clear_oneshot(int cpu) { }
  42 static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
  43 #endif
  44
  45 /*
  46  * Debugging: see timer_list.c
  47  */
  48 struct tick_device *tick_get_broadcast_device(void)
  49 {
  50         return &tick_broadcast_device;
  51 }
  52
  53 struct cpumask *tick_get_broadcast_mask(void)
  54 {
  55         return tick_broadcast_mask;
  56 }
  57
  58 /*
  59  * Start the device in periodic mode
  60  */
  61 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  62 {
  63         if (bc)
  64                 tick_setup_periodic(bc, 1);
  65 }
  66
  67 /*
  68  * Check, if the device can be utilized as broadcast device:
  69  */
  70 static bool tick_check_broadcast_device(struct clock_event_device *curdev,
  71                                         struct clock_event_device *newdev)
  72 {
  73         if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
  74             (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
  75             (newdev->features & CLOCK_EVT_FEAT_C3STOP))
  76                 return false;
  77
  78         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
  79             !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
  80                 return false;
  81
  82         return !curdev || newdev->rating > curdev->rating;
  83 }
  84
  85 /*
  86  * Conditionally install/replace broadcast device
  87  */
  88 void tick_install_broadcast_device(struct clock_event_device *dev)
  89 {
  90         struct clock_event_device *cur = tick_broadcast_device.evtdev;
  91
  92         if (!tick_check_broadcast_device(cur, dev))
  93                 return;
  94
  95         if (!try_module_get(dev->owner))
  96                 return;
  97
  98         clockevents_exchange_device(cur, dev);
  99         if (cur)
 100                 cur->event_handler = clockevents_handle_noop;
 101         tick_broadcast_device.evtdev = dev;
 102         if (!cpumask_empty(tick_broadcast_mask))
 103                 tick_broadcast_start_periodic(dev);
 104         /*
 105          * Inform all cpus about this. We might be in a situation
 106          * where we did not switch to oneshot mode because the per cpu
 107          * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
 108          * of a oneshot capable broadcast device. Without that
 109          * notification the systems stays stuck in periodic mode
 110          * forever.
 111          */
 112         if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
 113                 tick_clock_notify();
 114 }
 115
 116 /*
 117  * Check, if the device is the broadcast device
 118  */
 119 int tick_is_broadcast_device(struct clock_event_device *dev)
 120 {
 121         return (dev && tick_broadcast_device.evtdev == dev);
 122 }
 123
 124 int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
 125 {
 126         int ret = -ENODEV;
 127
 128         if (tick_is_broadcast_device(dev)) {
 129                 raw_spin_lock(&tick_broadcast_lock);
 130                 ret = __clockevents_update_freq(dev, freq);
 131                 raw_spin_unlock(&tick_broadcast_lock);
 132         }
 133         return ret;
 134 }
 135
 136
 137 static void err_broadcast(const struct cpumask *mask)
 138 {
 139         pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
 140 }
 141
 142 static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
 143 {
 144         if (!dev->broadcast)
 145                 dev->broadcast = tick_broadcast;
 146         if (!dev->broadcast) {
 147                 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
 148                              dev->name);
 149                 dev->broadcast = err_broadcast;
 150         }
 151 }
 152
 153 /*
 154  * Check, if the device is disfunctional and a place holder, which
 155  * needs to be handled by the broadcast device.
 156  */
 157 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
 158 {
 159         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 160         unsigned long flags;
 161         int ret = 0;
 162
 163         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 164
 165         /*
 166          * Devices might be registered with both periodic and oneshot
 167          * mode disabled. This signals, that the device needs to be
 168          * operated from the broadcast device and is a placeholder for
 169          * the cpu local device.
 170          */
 171         if (!tick_device_is_functional(dev)) {
 172                 dev->event_handler = tick_handle_periodic;
 173                 tick_device_setup_broadcast_func(dev);
 174                 cpumask_set_cpu(cpu, tick_broadcast_mask);
 175                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 176                         tick_broadcast_start_periodic(bc);
 177                 else
 178                         tick_broadcast_setup_oneshot(bc);
 179                 ret = 1;
 180         } else {
 181                 /*
 182                  * Clear the broadcast bit for this cpu if the
 183                  * device is not power state affected.
 184                  */
 185                 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 186                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 187                 else
 188                         tick_device_setup_broadcast_func(dev);
 189
 190                 /*
 191                  * Clear the broadcast bit if the CPU is not in
 192                  * periodic broadcast on state.
 193                  */
 194                 if (!cpumask_test_cpu(cpu, tick_broadcast_on))
 195                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 196
 197                 switch (tick_broadcast_device.mode) {
 198                 case TICKDEV_MODE_ONESHOT:
 199                         /*
 200                          * If the system is in oneshot mode we can
 201                          * unconditionally clear the oneshot mask bit,
 202                          * because the CPU is running and therefore
 203                          * not in an idle state which causes the power
 204                          * state affected device to stop. Let the
 205                          * caller initialize the device.
 206                          */
 207                         tick_broadcast_clear_oneshot(cpu);
 208                         ret = 0;
 209                         break;
 210
 211                 case TICKDEV_MODE_PERIODIC:
 212                         /*
 213                          * If the system is in periodic mode, check
 214                          * whether the broadcast device can be
 215                          * switched off now.
 216                          */
 217                         if (cpumask_empty(tick_broadcast_mask) && bc)
 218                                 clockevents_shutdown(bc);
 219                         /*
 220                          * If we kept the cpu in the broadcast mask,
 221                          * tell the caller to leave the per cpu device
 222                          * in shutdown state. The periodic interrupt
 223                          * is delivered by the broadcast device, if
 224                          * the broadcast device exists and is not
 225                          * hrtimer based.
 226                          */
 227                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 228                                 ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
 229                         break;
 230                 default:
 231                         break;
 232                 }
 233         }
 234         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 235         return ret;
 236 }
 237
 238 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 239 int tick_receive_broadcast(void)
 240 {
 241         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 242         struct clock_event_device *evt = td->evtdev;
 243
 244         if (!evt)
 245                 return -ENODEV;
 246
 247         if (!evt->event_handler)
 248                 return -EINVAL;
 249
 250         evt->event_handler(evt);
 251         return 0;
 252 }
 253 #endif
 254
 255 /*
 256  * Broadcast the event to the cpus, which are set in the mask (mangled).
 257  */
 258 static bool tick_do_broadcast(struct cpumask *mask)
 259 {
 260         int cpu = smp_processor_id();
 261         struct tick_device *td;
 262         bool local = false;
 263
 264         /*
 265          * Check, if the current cpu is in the mask
 266          */
 267         if (cpumask_test_cpu(cpu, mask)) {
 268                 struct clock_event_device *bc = tick_broadcast_device.evtdev;
 269
 270                 cpumask_clear_cpu(cpu, mask);
 271                 /*
 272                  * We only run the local handler, if the broadcast
 273                  * device is not hrtimer based. Otherwise we run into
 274                  * a hrtimer recursion.
 275                  *
 276                  * local timer_interrupt()
 277                  *   local_handler()
 278                  *     expire_hrtimers()
 279                  *       bc_handler()
 280                  *         local_handler()
 281                  *           expire_hrtimers()
 282                  */
 283                 local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
 284         }
 285
 286         if (!cpumask_empty(mask)) {
 287                 /*
 288                  * It might be necessary to actually check whether the devices
 289                  * have different broadcast functions. For now, just use the
 290                  * one of the first device. This works as long as we have this
 291                  * misfeature only on x86 (lapic)
 292                  */
 293                 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
 294                 td->evtdev->broadcast(mask);
 295         }
 296         return local;
 297 }
 298
 299 /*
 300  * Periodic broadcast:
 301  * - invoke the broadcast handlers
 302  */
 303 static bool tick_do_periodic_broadcast(void)
 304 {
 305         cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
 306         return tick_do_broadcast(tmpmask);
 307 }
 308
 309 /*
 310  * Event handler for periodic broadcast ticks
 311  */
 312 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 313 {
 314         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 315         bool bc_local;
 316
 317         raw_spin_lock(&tick_broadcast_lock);
 318
 319         /* Handle spurious interrupts gracefully */
 320         if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
 321                 raw_spin_unlock(&tick_broadcast_lock);
 322                 return;
 323         }
 324
 325         bc_local = tick_do_periodic_broadcast();
 326
 327         if (clockevent_state_oneshot(dev)) {
 328                 ktime_t next = ktime_add(dev->next_event, tick_period);
 329
 330                 clockevents_program_event(dev, next, true);
 331         }
 332         raw_spin_unlock(&tick_broadcast_lock);
 333
 334         /*
 335          * We run the handler of the local cpu after dropping
 336          * tick_broadcast_lock because the handler might deadlock when
 337          * trying to switch to oneshot mode.
 338          */
 339         if (bc_local)
 340                 td->evtdev->event_handler(td->evtdev);
 341 }
 342
 343 /**
 344  * tick_broadcast_control - Enable/disable or force broadcast mode
 345  * @mode:       The selected broadcast mode
 346  *
 347  * Called when the system enters a state where affected tick devices
 348  * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
 349  */
 350 void tick_broadcast_control(enum tick_broadcast_mode mode)
 351 {
 352         struct clock_event_device *bc, *dev;
 353         struct tick_device *td;
 354         int cpu, bc_stopped;
 355         unsigned long flags;
 356
 357         /* Protects also the local clockevent device. */
 358         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 359         td = this_cpu_ptr(&tick_cpu_device);
 360         dev = td->evtdev;
 361
 362         /*
 363          * Is the device not affected by the powerstate ?
 364          */
 365         if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 366                 goto out;
 367
 368         if (!tick_device_is_functional(dev))
 369                 goto out;
 370
 371         cpu = smp_processor_id();
 372         bc = tick_broadcast_device.evtdev;
 373         bc_stopped = cpumask_empty(tick_broadcast_mask);
 374
 375         switch (mode) {
 376         case TICK_BROADCAST_FORCE:
 377                 tick_broadcast_forced = 1;
 378                 /* fall through */
 379         case TICK_BROADCAST_ON:
 380                 cpumask_set_cpu(cpu, tick_broadcast_on);
 381                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
 382                         /*
 383                          * Only shutdown the cpu local device, if:
 384                          *
 385                          * - the broadcast device exists
 386                          * - the broadcast device is not a hrtimer based one
 387                          * - the broadcast device is in periodic mode to
 388                          *   avoid a hickup during switch to oneshot mode
 389                          */
 390                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
 391                             tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 392                                 clockevents_shutdown(dev);
 393                 }
 394                 break;
 395
 396         case TICK_BROADCAST_OFF:
 397                 if (tick_broadcast_forced)
 398                         break;
 399                 cpumask_clear_cpu(cpu, tick_broadcast_on);
 400                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
 401                         if (tick_broadcast_device.mode ==
 402                             TICKDEV_MODE_PERIODIC)
 403                                 tick_setup_periodic(dev, 0);
 404                 }
 405                 break;
 406         }
 407
 408         if (bc) {
 409                 if (cpumask_empty(tick_broadcast_mask)) {
 410                         if (!bc_stopped)
 411                                 clockevents_shutdown(bc);
 412                 } else if (bc_stopped) {
 413                         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 414                                 tick_broadcast_start_periodic(bc);
 415                         else
 416                                 tick_broadcast_setup_oneshot(bc);
 417                 }
 418         }
 419 out:
 420         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 421 }
 422 EXPORT_SYMBOL_GPL(tick_broadcast_control);
 423
 424 /*
 425  * Set the periodic handler depending on broadcast on/off
 426  */
 427 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 428 {
 429         if (!broadcast)
 430                 dev->event_handler = tick_handle_periodic;
 431         else
 432                 dev->event_handler = tick_handle_periodic_broadcast;
 433 }
 434
 435 #ifdef CONFIG_HOTPLUG_CPU
 436 /*
 437  * Remove a CPU from broadcasting
 438  */
 439 void tick_shutdown_broadcast(unsigned int cpu)
 440 {
 441         struct clock_event_device *bc;
 442         unsigned long flags;
 443
 444         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 445
 446         bc = tick_broadcast_device.evtdev;
 447         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 448         cpumask_clear_cpu(cpu, tick_broadcast_on);
 449
 450         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 451                 if (bc && cpumask_empty(tick_broadcast_mask))
 452                         clockevents_shutdown(bc);
 453         }
 454
 455         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 456 }
 457 #endif
 458
 459 void tick_suspend_broadcast(void)
 460 {
 461         struct clock_event_device *bc;
 462         unsigned long flags;
 463
 464         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 465
 466         bc = tick_broadcast_device.evtdev;
 467         if (bc)
 468                 clockevents_shutdown(bc);
 469
 470         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 471 }
 472
 473 /*
 474  * This is called from tick_resume_local() on a resuming CPU. That's
 475  * called from the core resume function, tick_unfreeze() and the magic XEN
 476  * resume hackery.
 477  *
 478  * In none of these cases the broadcast device mode can change and the
 479  * bit of the resuming CPU in the broadcast mask is safe as well.
 480  */
 481 bool tick_resume_check_broadcast(void)
 482 {
 483         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
 484                 return false;
 485         else
 486                 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
 487 }
 488
 489 void tick_resume_broadcast(void)
 490 {
 491         struct clock_event_device *bc;
 492         unsigned long flags;
 493
 494         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 495
 496         bc = tick_broadcast_device.evtdev;
 497
 498         if (bc) {
 499                 clockevents_tick_resume(bc);
 500
 501                 switch (tick_broadcast_device.mode) {
 502                 case TICKDEV_MODE_PERIODIC:
 503                         if (!cpumask_empty(tick_broadcast_mask))
 504                                 tick_broadcast_start_periodic(bc);
 505                         break;
 506                 case TICKDEV_MODE_ONESHOT:
 507                         if (!cpumask_empty(tick_broadcast_mask))
 508                                 tick_resume_broadcast_oneshot(bc);
 509                         break;
 510                 }
 511         }
 512         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 513 }
 514
 515 #ifdef CONFIG_TICK_ONESHOT
 516
 517 static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
 518 static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
 519 static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;
 520
 521 /*
 522  * Exposed for debugging: see timer_list.c
 523  */
 524 struct cpumask *tick_get_broadcast_oneshot_mask(void)
 525 {
 526         return tick_broadcast_oneshot_mask;
 527 }
 528
 529 /*
 530  * Called before going idle with interrupts disabled. Checks whether a
 531  * broadcast event from the other core is about to happen. We detected
 532  * that in tick_broadcast_oneshot_control(). The callsite can use this
 533  * to avoid a deep idle transition as we are about to get the
 534  * broadcast IPI right away.
 535  */
 536 int tick_check_broadcast_expired(void)
 537 {
 538         return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
 539 }
 540
 541 /*
 542  * Set broadcast interrupt affinity
 543  */
 544 static void tick_broadcast_set_affinity(struct clock_event_device *bc,
 545                                         const struct cpumask *cpumask)
 546 {
 547         if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
 548                 return;
 549
 550         if (cpumask_equal(bc->cpumask, cpumask))
 551                 return;
 552
 553         bc->cpumask = cpumask;
 554         irq_set_affinity(bc->irq, bc->cpumask);
 555 }
 556
 557 static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
 558                                      ktime_t expires)
 559 {
 560         if (!clockevent_state_oneshot(bc))
 561                 clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 562
 563         clockevents_program_event(bc, expires, 1);
 564         tick_broadcast_set_affinity(bc, cpumask_of(cpu));
 565 }
 566
 567 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
 568 {
 569         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 570 }
 571
 572 /*
 573  * Called from irq_enter() when idle was interrupted to reenable the
 574  * per cpu device.
 575  */
 576 void tick_check_oneshot_broadcast_this_cpu(void)
 577 {
 578         if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
 579                 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 580
 581                 /*
 582                  * We might be in the middle of switching over from
 583                  * periodic to oneshot. If the CPU has not yet
 584                  * switched over, leave the device alone.
 585                  */
 586                 if (td->mode == TICKDEV_MODE_ONESHOT) {
 587                         clockevents_switch_state(td->evtdev,
 588                                               CLOCK_EVT_STATE_ONESHOT);
 589                 }
 590         }
 591 }
 592
 593 /*
 594  * Handle oneshot mode broadcasting
 595  */
 596 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 597 {
 598         struct tick_device *td;
 599         ktime_t now, next_event;
 600         int cpu, next_cpu = 0;
 601         bool bc_local;
 602
 603         raw_spin_lock(&tick_broadcast_lock);
 604         dev->next_event = KTIME_MAX;
 605         next_event = KTIME_MAX;
 606         cpumask_clear(tmpmask);
 607         now = ktime_get();
 608         /* Find all expired events */
 609         for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
 610                 /*
 611                  * Required for !SMP because for_each_cpu() reports
 612                  * unconditionally CPU0 as set on UP kernels.
 613                  */
 614                 if (!IS_ENABLED(CONFIG_SMP) &&
 615                     cpumask_empty(tick_broadcast_oneshot_mask))
 616                         break;
 617
 618                 td = &per_cpu(tick_cpu_device, cpu);
 619                 if (td->evtdev->next_event <= now) {
 620                         cpumask_set_cpu(cpu, tmpmask);
 621                         /*
 622                          * Mark the remote cpu in the pending mask, so
 623                          * it can avoid reprogramming the cpu local
 624                          * timer in tick_broadcast_oneshot_control().
 625                          */
 626                         cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
 627                 } else if (td->evtdev->next_event < next_event) {
 628                         next_event = td->evtdev->next_event;
 629                         next_cpu = cpu;
 630                 }
 631         }
 632
 633         /*
 634          * Remove the current cpu from the pending mask. The event is
 635          * delivered immediately in tick_do_broadcast() !
 636          */
 637         cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
 638
 639         /* Take care of enforced broadcast requests */
 640         cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
 641         cpumask_clear(tick_broadcast_force_mask);
 642
 643         /*
 644          * Sanity check. Catch the case where we try to broadcast to
 645          * offline cpus.
 646          */
 647         if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
 648                 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
 649
 650         /*
 651          * Wakeup the cpus which have an expired event.
 652          */
 653         bc_local = tick_do_broadcast(tmpmask);
 654
 655         /*
 656          * Two reasons for reprogram:
 657          *
 658          * - The global event did not expire any CPU local
 659          * events. This happens in dyntick mode, as the maximum PIT
 660          * delta is quite small.
 661          *
 662          * - There are pending events on sleeping CPUs which were not
 663          * in the event mask
 664          */
 665         if (next_event != KTIME_MAX)
 666                 tick_broadcast_set_event(dev, next_cpu, next_event);
 667
 668         raw_spin_unlock(&tick_broadcast_lock);
 669
 670         if (bc_local) {
 671                 td = this_cpu_ptr(&tick_cpu_device);
 672                 td->evtdev->event_handler(td->evtdev);
 673         }
 674 }
 675
 676 static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
 677 {
 678         if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 679                 return 0;
 680         if (bc->next_event == KTIME_MAX)
 681                 return 0;
 682         return bc->bound_on == cpu ? -EBUSY : 0;
 683 }
 684
 685 static void broadcast_shutdown_local(struct clock_event_device *bc,
 686                                      struct clock_event_device *dev)
 687 {
 688         /*
 689          * For hrtimer based broadcasting we cannot shutdown the cpu
 690          * local device if our own event is the first one to expire or
 691          * if we own the broadcast timer.
 692          */
 693         if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
 694                 if (broadcast_needs_cpu(bc, smp_processor_id()))
 695                         return;
 696                 if (dev->next_event < bc->next_event)
 697                         return;
 698         }
 699         clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
 700 }
 701
 702 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 703 {
 704         struct clock_event_device *bc, *dev;
 705         int cpu, ret = 0;
 706         ktime_t now;
 707
 708         /*
 709          * If there is no broadcast device, tell the caller not to go
 710          * into deep idle.
 711          */
 712         if (!tick_broadcast_device.evtdev)
 713                 return -EBUSY;
 714
 715         dev = this_cpu_ptr(&tick_cpu_device)->evtdev;
 716
 717         raw_spin_lock(&tick_broadcast_lock);
 718         bc = tick_broadcast_device.evtdev;
 719         cpu = smp_processor_id();
 720
 721         if (state == TICK_BROADCAST_ENTER) {
 722                 /*
 723                  * If the current CPU owns the hrtimer broadcast
 724                  * mechanism, it cannot go deep idle and we do not add
 725                  * the CPU to the broadcast mask. We don't have to go
 726                  * through the EXIT path as the local timer is not
 727                  * shutdown.
 728                  */
 729                 ret = broadcast_needs_cpu(bc, cpu);
 730                 if (ret)
 731                         goto out;
 732
 733                 /*
 734                  * If the broadcast device is in periodic mode, we
 735                  * return.
 736                  */
 737                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 738                         /* If it is a hrtimer based broadcast, return busy */
 739                         if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
 740                                 ret = -EBUSY;
 741                         goto out;
 742                 }
 743
 744                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
 745                         WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
 746
 747                         /* Conditionally shut down the local timer. */
 748                         broadcast_shutdown_local(bc, dev);
 749
 750                         /*
 751                          * We only reprogram the broadcast timer if we
 752                          * did not mark ourself in the force mask and
 753                          * if the cpu local event is earlier than the
 754                          * broadcast event. If the current CPU is in
 755                          * the force mask, then we are going to be
 756                          * woken by the IPI right away; we return
 757                          * busy, so the CPU does not try to go deep
 758                          * idle.
 759                          */
 760                         if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
 761                                 ret = -EBUSY;
 762                         } else if (dev->next_event < bc->next_event) {
 763                                 tick_broadcast_set_event(bc, cpu, dev->next_event);
 764                                 /*
 765                                  * In case of hrtimer broadcasts the
 766                                  * programming might have moved the
 767                                  * timer to this cpu. If yes, remove
 768                                  * us from the broadcast mask and
 769                                  * return busy.
 770                                  */
 771                                 ret = broadcast_needs_cpu(bc, cpu);
 772                                 if (ret) {
 773                                         cpumask_clear_cpu(cpu,
 774                                                 tick_broadcast_oneshot_mask);
 775                                 }
 776                         }
 777                 }
 778         } else {
 779                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
 780                         clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
 781                         /*
 782                          * The cpu which was handling the broadcast
 783                          * timer marked this cpu in the broadcast
 784                          * pending mask and fired the broadcast
 785                          * IPI. So we are going to handle the expired
 786                          * event anyway via the broadcast IPI
 787                          * handler. No need to reprogram the timer
 788                          * with an already expired event.
 789                          */
 790                         if (cpumask_test_and_clear_cpu(cpu,
 791                                        tick_broadcast_pending_mask))
 792                                 goto out;
 793
 794                         /*
 795                          * Bail out if there is no next event.
 796                          */
 797                         if (dev->next_event == KTIME_MAX)
 798                                 goto out;
 799                         /*
 800                          * If the pending bit is not set, then we are
 801                          * either the CPU handling the broadcast
 802                          * interrupt or we got woken by something else.
 803                          *
 804                          * We are not longer in the broadcast mask, so
 805                          * if the cpu local expiry time is already
 806                          * reached, we would reprogram the cpu local
 807                          * timer with an already expired event.
 808                          *
 809                          * This can lead to a ping-pong when we return
 810                          * to idle and therefor rearm the broadcast
 811                          * timer before the cpu local timer was able
 812                          * to fire. This happens because the forced
 813                          * reprogramming makes sure that the event
 814                          * will happen in the future and depending on
 815                          * the min_delta setting this might be far
 816                          * enough out that the ping-pong starts.
 817                          *
 818                          * If the cpu local next_event has expired
 819                          * then we know that the broadcast timer
 820                          * next_event has expired as well and
 821                          * broadcast is about to be handled. So we
 822                          * avoid reprogramming and enforce that the
 823                          * broadcast handler, which did not run yet,
 824                          * will invoke the cpu local handler.
 825                          *
 826                          * We cannot call the handler directly from
 827                          * here, because we might be in a NOHZ phase
 828                          * and we did not go through the irq_enter()
 829                          * nohz fixups.
 830                          */
 831                         now = ktime_get();
 832                         if (dev->next_event <= now) {
 833                                 cpumask_set_cpu(cpu, tick_broadcast_force_mask);
 834                                 goto out;
 835                         }
 836                         /*
 837                          * We got woken by something else. Reprogram
 838                          * the cpu local timer device.
 839                          */
 840                         tick_program_event(dev->next_event, 1);
 841                 }
 842         }
 843 out:
 844         raw_spin_unlock(&tick_broadcast_lock);
 845         return ret;
 846 }
 847
 848 /*
 849  * Reset the one shot broadcast for a cpu
 850  *
 851  * Called with tick_broadcast_lock held
 852  */
 853 static void tick_broadcast_clear_oneshot(int cpu)
 854 {
 855         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 856         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 857 }
 858
 859 static void tick_broadcast_init_next_event(struct cpumask *mask,
 860                                            ktime_t expires)
 861 {
 862         struct tick_device *td;
 863         int cpu;
 864
 865         for_each_cpu(cpu, mask) {
 866                 td = &per_cpu(tick_cpu_device, cpu);
 867                 if (td->evtdev)
 868                         td->evtdev->next_event = expires;
 869         }
 870 }
 871
 872 /**
 873  * tick_broadcast_setup_oneshot - setup the broadcast device
 874  */
 875 static void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 876 {
 877         int cpu = smp_processor_id();
 878
 879         if (!bc)
 880                 return;
 881
 882         /* Set it up only once ! */
 883         if (bc->event_handler != tick_handle_oneshot_broadcast) {
 884                 int was_periodic = clockevent_state_periodic(bc);
 885
 886                 bc->event_handler = tick_handle_oneshot_broadcast;
 887
 888                 /*
 889                  * We must be careful here. There might be other CPUs
 890                  * waiting for periodic broadcast. We need to set the
 891                  * oneshot_mask bits for those and program the
 892                  * broadcast device to fire.
 893                  */
 894                 cpumask_copy(tmpmask, tick_broadcast_mask);
 895                 cpumask_clear_cpu(cpu, tmpmask);
 896                 cpumask_or(tick_broadcast_oneshot_mask,
 897                            tick_broadcast_oneshot_mask, tmpmask);
 898
 899                 if (was_periodic && !cpumask_empty(tmpmask)) {
 900                         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 901                         tick_broadcast_init_next_event(tmpmask,
 902                                                        tick_next_period);
 903                         tick_broadcast_set_event(bc, cpu, tick_next_period);
 904                 } else
 905                         bc->next_event = KTIME_MAX;
 906         } else {
 907                 /*
 908                  * The first cpu which switches to oneshot mode sets
 909                  * the bit for all other cpus which are in the general
 910                  * (periodic) broadcast mask. So the bit is set and
 911                  * would prevent the first broadcast enter after this
 912                  * to program the bc device.
 913                  */
 914                 tick_broadcast_clear_oneshot(cpu);
 915         }
 916 }
 917
 918 /*
 919  * Select oneshot operating mode for the broadcast device
 920  */
 921 void tick_broadcast_switch_to_oneshot(void)
 922 {
 923         struct clock_event_device *bc;
 924         unsigned long flags;
 925
 926         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 927
 928         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 929         bc = tick_broadcast_device.evtdev;
 930         if (bc)
 931                 tick_broadcast_setup_oneshot(bc);
 932
 933         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 934 }
 935
 936 #ifdef CONFIG_HOTPLUG_CPU
 937 void hotplug_cpu__broadcast_tick_pull(int deadcpu)
 938 {
 939         struct clock_event_device *bc;
 940         unsigned long flags;
 941
 942         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 943         bc = tick_broadcast_device.evtdev;
 944
 945         if (bc && broadcast_needs_cpu(bc, deadcpu)) {
 946                 /* This moves the broadcast assignment to this CPU: */
 947                 clockevents_program_event(bc, bc->next_event, 1);
 948         }
 949         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 950 }
 951
 952 /*
 953  * Remove a dead CPU from broadcasting
 954  */
 955 void tick_shutdown_broadcast_oneshot(unsigned int cpu)
 956 {
 957         unsigned long flags;
 958
 959         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 960
 961         /*
 962          * Clear the broadcast masks for the dead cpu, but do not stop
 963          * the broadcast device!
 964          */
 965         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 966         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 967         cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
 968
 969         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 970 }
 971 #endif
 972
 973 /*
 974  * Check, whether the broadcast device is in one shot mode
 975  */
 976 int tick_broadcast_oneshot_active(void)
 977 {
 978         return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
 979 }
 980
 981 /*
 982  * Check whether the broadcast device supports oneshot.
 983  */
 984 bool tick_broadcast_oneshot_available(void)
 985 {
 986         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 987
 988         return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
 989 }
 990
 991 #else
 992 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 993 {
 994         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 995
 996         if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
 997                 return -EBUSY;
 998
 999         return 0;
1000 }
1001 #endif
1002
1003 void __init tick_broadcast_init(void)
1004 {
1005         zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
1006         zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
1007         zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
1008 #ifdef CONFIG_TICK_ONESHOT
1009         zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
1010         zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
1011         zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
1012 #endif
1013 }