kernel/time/tick-broadcast.c

   1 /*
   2  * linux/kernel/time/tick-broadcast.c
   3  *
   4  * This file contains functions which emulate a local clock-event
   5  * device via a broadcast event source.
   6  *
   7  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   8  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   9  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10  *
  11  * This code is licenced under the GPL version 2. For details see
  12  * kernel-base/COPYING.
  13  */
  14 #include <linux/cpu.h>
  15 #include <linux/err.h>
  16 #include <linux/hrtimer.h>
  17 #include <linux/irq.h>
  18 #include <linux/percpu.h>
  19 #include <linux/profile.h>
  20 #include <linux/sched.h>
  21 #include <linux/tick.h>
  22
  23 #include "tick-internal.h"
  24
  25 /*
  26  * Broadcast support for broken x86 hardware, where the local apic
  27  * timer stops in C3 state.
  28  */
  29
  30 struct tick_device tick_broadcast_device;
  31 static cpumask_t tick_broadcast_mask;
  32 static DEFINE_SPINLOCK(tick_broadcast_lock);
  33
  34 /*
  35  * Debugging: see timer_list.c
  36  */
  37 struct tick_device *tick_get_broadcast_device(void)
  38 {
  39         return &tick_broadcast_device;
  40 }
  41
  42 cpumask_t *tick_get_broadcast_mask(void)
  43 {
  44         return &tick_broadcast_mask;
  45 }
  46
  47 /*
  48  * Start the device in periodic mode
  49  */
  50 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  51 {
  52         if (bc && bc->mode == CLOCK_EVT_MODE_SHUTDOWN)
  53                 tick_setup_periodic(bc, 1);
  54 }
  55
  56 /*
  57  * Check, if the device can be utilized as broadcast device:
  58  */
  59 int tick_check_broadcast_device(struct clock_event_device *dev)
  60 {
  61         if (tick_broadcast_device.evtdev ||
  62             (dev->features & CLOCK_EVT_FEAT_C3STOP))
  63                 return 0;
  64
  65         clockevents_exchange_device(NULL, dev);
  66         tick_broadcast_device.evtdev = dev;
  67         if (!cpus_empty(tick_broadcast_mask))
  68                 tick_broadcast_start_periodic(dev);
  69         return 1;
  70 }
  71
  72 /*
  73  * Check, if the device is the broadcast device
  74  */
  75 int tick_is_broadcast_device(struct clock_event_device *dev)
  76 {
  77         return (dev && tick_broadcast_device.evtdev == dev);
  78 }
  79
  80 /*
  81  * Check, if the device is disfunctional and a place holder, which
  82  * needs to be handled by the broadcast device.
  83  */
  84 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
  85 {
  86         unsigned long flags;
  87         int ret = 0;
  88
  89         spin_lock_irqsave(&tick_broadcast_lock, flags);
  90
  91         /*
  92          * Devices might be registered with both periodic and oneshot
  93          * mode disabled. This signals, that the device needs to be
  94          * operated from the broadcast device and is a placeholder for
  95          * the cpu local device.
  96          */
  97         if (!tick_device_is_functional(dev)) {
  98                 dev->event_handler = tick_handle_periodic;
  99                 cpu_set(cpu, tick_broadcast_mask);
 100                 tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
 101                 ret = 1;
 102         }
 103
 104         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 105         return ret;
 106 }
 107
 108 /*
 109  * Broadcast the event to the cpus, which are set in the mask
 110  */
 111 int tick_do_broadcast(cpumask_t mask)
 112 {
 113         int ret = 0, cpu = smp_processor_id();
 114         struct tick_device *td;
 115
 116         /*
 117          * Check, if the current cpu is in the mask
 118          */
 119         if (cpu_isset(cpu, mask)) {
 120                 cpu_clear(cpu, mask);
 121                 td = &per_cpu(tick_cpu_device, cpu);
 122                 td->evtdev->event_handler(td->evtdev);
 123                 ret = 1;
 124         }
 125
 126         if (!cpus_empty(mask)) {
 127                 /*
 128                  * It might be necessary to actually check whether the devices
 129                  * have different broadcast functions. For now, just use the
 130                  * one of the first device. This works as long as we have this
 131                  * misfeature only on x86 (lapic)
 132                  */
 133                 cpu = first_cpu(mask);
 134                 td = &per_cpu(tick_cpu_device, cpu);
 135                 td->evtdev->broadcast(mask);
 136                 ret = 1;
 137         }
 138         return ret;
 139 }
 140
 141 /*
 142  * Periodic broadcast:
 143  * - invoke the broadcast handlers
 144  */
 145 static void tick_do_periodic_broadcast(void)
 146 {
 147         cpumask_t mask;
 148
 149         spin_lock(&tick_broadcast_lock);
 150
 151         cpus_and(mask, cpu_online_map, tick_broadcast_mask);
 152         tick_do_broadcast(mask);
 153
 154         spin_unlock(&tick_broadcast_lock);
 155 }
 156
 157 /*
 158  * Event handler for periodic broadcast ticks
 159  */
 160 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 161 {
 162         dev->next_event.tv64 = KTIME_MAX;
 163
 164         tick_do_periodic_broadcast();
 165
 166         /*
 167          * The device is in periodic mode. No reprogramming necessary:
 168          */
 169         if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
 170                 return;
 171
 172         /*
 173          * Setup the next period for devices, which do not have
 174          * periodic mode:
 175          */
 176         for (;;) {
 177                 ktime_t next = ktime_add(dev->next_event, tick_period);
 178
 179                 if (!clockevents_program_event(dev, next, ktime_get()))
 180                         return;
 181                 tick_do_periodic_broadcast();
 182         }
 183 }
 184
 185 /*
 186  * Powerstate information: The system enters/leaves a state, where
 187  * affected devices might stop
 188  */
 189 static void tick_do_broadcast_on_off(void *why)
 190 {
 191         struct clock_event_device *bc, *dev;
 192         struct tick_device *td;
 193         unsigned long flags, *reason = why;
 194         int cpu;
 195
 196         spin_lock_irqsave(&tick_broadcast_lock, flags);
 197
 198         cpu = smp_processor_id();
 199         td = &per_cpu(tick_cpu_device, cpu);
 200         dev = td->evtdev;
 201         bc = tick_broadcast_device.evtdev;
 202
 203         /*
 204          * Is the device in broadcast mode forever or is it not
 205          * affected by the powerstate ?
 206          */
 207         if (!dev || !tick_device_is_functional(dev) ||
 208             !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 209                 goto out;
 210
 211         if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_ON) {
 212                 if (!cpu_isset(cpu, tick_broadcast_mask)) {
 213                         cpu_set(cpu, tick_broadcast_mask);
 214                         if (td->mode == TICKDEV_MODE_PERIODIC)
 215                                 clockevents_set_mode(dev,
 216                                                      CLOCK_EVT_MODE_SHUTDOWN);
 217                 }
 218         } else {
 219                 if (cpu_isset(cpu, tick_broadcast_mask)) {
 220                         cpu_clear(cpu, tick_broadcast_mask);
 221                         if (td->mode == TICKDEV_MODE_PERIODIC)
 222                                 tick_setup_periodic(dev, 0);
 223                 }
 224         }
 225
 226         if (cpus_empty(tick_broadcast_mask))
 227                 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 228         else {
 229                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 230                         tick_broadcast_start_periodic(bc);
 231                 else
 232                         tick_broadcast_setup_oneshot(bc);
 233         }
 234 out:
 235         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 236 }
 237
 238 /*
 239  * Powerstate information: The system enters/leaves a state, where
 240  * affected devices might stop.
 241  */
 242 void tick_broadcast_on_off(unsigned long reason, int *oncpu)
 243 {
 244         int cpu = get_cpu();
 245
 246         if (cpu == *oncpu)
 247                 tick_do_broadcast_on_off(&reason);
 248         else
 249                 smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
 250                                          &reason, 1, 1);
 251         put_cpu();
 252 }
 253
 254 /*
 255  * Set the periodic handler depending on broadcast on/off
 256  */
 257 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 258 {
 259         if (!broadcast)
 260                 dev->event_handler = tick_handle_periodic;
 261         else
 262                 dev->event_handler = tick_handle_periodic_broadcast;
 263 }
 264
 265 /*
 266  * Remove a CPU from broadcasting
 267  */
 268 void tick_shutdown_broadcast(unsigned int *cpup)
 269 {
 270         struct clock_event_device *bc;
 271         unsigned long flags;
 272         unsigned int cpu = *cpup;
 273
 274         spin_lock_irqsave(&tick_broadcast_lock, flags);
 275
 276         bc = tick_broadcast_device.evtdev;
 277         cpu_clear(cpu, tick_broadcast_mask);
 278
 279         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 280                 if (bc && cpus_empty(tick_broadcast_mask))
 281                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 282         }
 283
 284         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 285 }
 286
 287 void tick_suspend_broadcast(void)
 288 {
 289         struct clock_event_device *bc;
 290         unsigned long flags;
 291
 292         spin_lock_irqsave(&tick_broadcast_lock, flags);
 293
 294         bc = tick_broadcast_device.evtdev;
 295         if (bc && tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 296                 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 297
 298         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 299 }
 300
 301 int tick_resume_broadcast(void)
 302 {
 303         struct clock_event_device *bc;
 304         unsigned long flags;
 305         int broadcast = 0;
 306
 307         spin_lock_irqsave(&tick_broadcast_lock, flags);
 308
 309         bc = tick_broadcast_device.evtdev;
 310         if (bc) {
 311                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC &&
 312                     !cpus_empty(tick_broadcast_mask))
 313                         tick_broadcast_start_periodic(bc);
 314
 315                 broadcast = cpu_isset(smp_processor_id(), tick_broadcast_mask);
 316         }
 317         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 318
 319         return broadcast;
 320 }
 321
 322
 323 #ifdef CONFIG_TICK_ONESHOT
 324
 325 static cpumask_t tick_broadcast_oneshot_mask;
 326
 327 /*
 328  * Debugging: see timer_list.c
 329  */
 330 cpumask_t *tick_get_broadcast_oneshot_mask(void)
 331 {
 332         return &tick_broadcast_oneshot_mask;
 333 }
 334
 335 static int tick_broadcast_set_event(ktime_t expires, int force)
 336 {
 337         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 338         ktime_t now = ktime_get();
 339         int res;
 340
 341         for(;;) {
 342                 res = clockevents_program_event(bc, expires, now);
 343                 if (!res || !force)
 344                         return res;
 345                 now = ktime_get();
 346                 expires = ktime_add(now, ktime_set(0, bc->min_delta_ns));
 347         }
 348 }
 349
 350 /*
 351  * Reprogram the broadcast device:
 352  *
 353  * Called with tick_broadcast_lock held and interrupts disabled.
 354  */
 355 static int tick_broadcast_reprogram(void)
 356 {
 357         ktime_t expires = { .tv64 = KTIME_MAX };
 358         struct tick_device *td;
 359         int cpu;
 360
 361         /*
 362          * Find the event which expires next:
 363          */
 364         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
 365              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
 366                 td = &per_cpu(tick_cpu_device, cpu);
 367                 if (td->evtdev->next_event.tv64 < expires.tv64)
 368                         expires = td->evtdev->next_event;
 369         }
 370
 371         if (expires.tv64 == KTIME_MAX)
 372                 return 0;
 373
 374         return tick_broadcast_set_event(expires, 0);
 375 }
 376
 377 /*
 378  * Handle oneshot mode broadcasting
 379  */
 380 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 381 {
 382         struct tick_device *td;
 383         cpumask_t mask;
 384         ktime_t now;
 385         int cpu;
 386
 387         spin_lock(&tick_broadcast_lock);
 388 again:
 389         dev->next_event.tv64 = KTIME_MAX;
 390         mask = CPU_MASK_NONE;
 391         now = ktime_get();
 392         /* Find all expired events */
 393         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
 394              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
 395                 td = &per_cpu(tick_cpu_device, cpu);
 396                 if (td->evtdev->next_event.tv64 <= now.tv64)
 397                         cpu_set(cpu, mask);
 398         }
 399
 400         /*
 401          * Wakeup the cpus which have an expired event. The broadcast
 402          * device is reprogrammed in the return from idle code.
 403          */
 404         if (!tick_do_broadcast(mask)) {
 405                 /*
 406                  * The global event did not expire any CPU local
 407                  * events. This happens in dyntick mode, as the
 408                  * maximum PIT delta is quite small.
 409                  */
 410                 if (tick_broadcast_reprogram())
 411                         goto again;
 412         }
 413         spin_unlock(&tick_broadcast_lock);
 414 }
 415
 416 /*
 417  * Powerstate information: The system enters/leaves a state, where
 418  * affected devices might stop
 419  */
 420 void tick_broadcast_oneshot_control(unsigned long reason)
 421 {
 422         struct clock_event_device *bc, *dev;
 423         struct tick_device *td;
 424         unsigned long flags;
 425         int cpu;
 426
 427         spin_lock_irqsave(&tick_broadcast_lock, flags);
 428
 429         /*
 430          * Periodic mode does not care about the enter/exit of power
 431          * states
 432          */
 433         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 434                 goto out;
 435
 436         bc = tick_broadcast_device.evtdev;
 437         cpu = smp_processor_id();
 438         td = &per_cpu(tick_cpu_device, cpu);
 439         dev = td->evtdev;
 440
 441         if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 442                 goto out;
 443
 444         if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
 445                 if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
 446                         cpu_set(cpu, tick_broadcast_oneshot_mask);
 447                         clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
 448                         if (dev->next_event.tv64 < bc->next_event.tv64)
 449                                 tick_broadcast_set_event(dev->next_event, 1);
 450                 }
 451         } else {
 452                 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
 453                         cpu_clear(cpu, tick_broadcast_oneshot_mask);
 454                         clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
 455                         if (dev->next_event.tv64 != KTIME_MAX)
 456                                 tick_program_event(dev->next_event, 1);
 457                 }
 458         }
 459
 460 out:
 461         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 462 }
 463
 464 /**
 465  * tick_broadcast_setup_highres - setup the broadcast device for highres
 466  */
 467 void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 468 {
 469         if (bc->mode != CLOCK_EVT_MODE_ONESHOT) {
 470                 bc->event_handler = tick_handle_oneshot_broadcast;
 471                 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
 472                 bc->next_event.tv64 = KTIME_MAX;
 473         }
 474 }
 475
 476 /*
 477  * Select oneshot operating mode for the broadcast device
 478  */
 479 void tick_broadcast_switch_to_oneshot(void)
 480 {
 481         struct clock_event_device *bc;
 482         unsigned long flags;
 483
 484         spin_lock_irqsave(&tick_broadcast_lock, flags);
 485
 486         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 487         bc = tick_broadcast_device.evtdev;
 488         if (bc)
 489                 tick_broadcast_setup_oneshot(bc);
 490         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 491 }
 492
 493
 494 /*
 495  * Remove a dead CPU from broadcasting
 496  */
 497 void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
 498 {
 499         struct clock_event_device *bc;
 500         unsigned long flags;
 501         unsigned int cpu = *cpup;
 502
 503         spin_lock_irqsave(&tick_broadcast_lock, flags);
 504
 505         bc = tick_broadcast_device.evtdev;
 506         cpu_clear(cpu, tick_broadcast_oneshot_mask);
 507
 508         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT) {
 509                 if (bc && cpus_empty(tick_broadcast_oneshot_mask))
 510                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 511         }
 512
 513         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 514 }
 515
 516 #endif