kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28 #include <linux/irq.h>
  29
  30 #define CREATE_TRACE_POINTS
  31 #include <trace/events/irq.h>
  32
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
  53 EXPORT_PER_CPU_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 const char * const softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * If ksoftirqd is scheduled, we do not want to process pending softirqs
  82  * right now. Let ksoftirqd handle this at its own rate, to get fairness,
  83  * unless we're doing some of the synchronous softirqs.
  84  */
  85 #define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
  86 static bool ksoftirqd_running(unsigned long pending)
  87 {
  88         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  89
  90         if (pending & SOFTIRQ_NOW_MASK)
  91                 return false;
  92         return tsk && (tsk->state == TASK_RUNNING) &&
  93                 !__kthread_should_park(tsk);
  94 }
  95
  96 /*
  97  * preempt_count and SOFTIRQ_OFFSET usage:
  98  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  99  *   softirq processing.
 100  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
 101  *   on local_bh_disable or local_bh_enable.
 102  * This lets us distinguish between whether we are currently processing
 103  * softirq and whether we just have bh disabled.
 104  */
 105
 106 /*
 107  * This one is for softirq.c-internal use,
 108  * where hardirqs are disabled legitimately:
 109  */
 110 #ifdef CONFIG_TRACE_IRQFLAGS
 111 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
 112 {
 113         unsigned long flags;
 114
 115         WARN_ON_ONCE(in_irq());
 116
 117         raw_local_irq_save(flags);
 118         /*
 119          * The preempt tracer hooks into preempt_count_add and will break
 120          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 121          * is set and before current->softirq_enabled is cleared.
 122          * We must manually increment preempt_count here and manually
 123          * call the trace_preempt_off later.
 124          */
 125         __preempt_count_add(cnt);
 126         /*
 127          * Were softirqs turned off above:
 128          */
 129         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 130                 trace_softirqs_off(ip);
 131         raw_local_irq_restore(flags);
 132
 133         if (preempt_count() == cnt) {
 134 #ifdef CONFIG_DEBUG_PREEMPT
 135                 current->preempt_disable_ip = get_lock_parent_ip();
 136 #endif
 137                 trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
 138         }
 139 }
 140 EXPORT_SYMBOL(__local_bh_disable_ip);
 141 #endif /* CONFIG_TRACE_IRQFLAGS */
 142
 143 static void __local_bh_enable(unsigned int cnt)
 144 {
 145         lockdep_assert_irqs_disabled();
 146
 147         if (preempt_count() == cnt)
 148                 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
 149
 150         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 151                 trace_softirqs_on(_RET_IP_);
 152
 153         __preempt_count_sub(cnt);
 154 }
 155
 156 /*
 157  * Special-case - softirqs can safely be enabled by __do_softirq(),
 158  * without processing still-pending softirqs:
 159  */
 160 void _local_bh_enable(void)
 161 {
 162         WARN_ON_ONCE(in_irq());
 163         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 164 }
 165 EXPORT_SYMBOL(_local_bh_enable);
 166
 167 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 168 {
 169         WARN_ON_ONCE(in_irq());
 170         lockdep_assert_irqs_enabled();
 171 #ifdef CONFIG_TRACE_IRQFLAGS
 172         local_irq_disable();
 173 #endif
 174         /*
 175          * Are softirqs going to be turned on now:
 176          */
 177         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 178                 trace_softirqs_on(ip);
 179         /*
 180          * Keep preemption disabled until we are done with
 181          * softirq processing:
 182          */
 183         preempt_count_sub(cnt - 1);
 184
 185         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 186                 /*
 187                  * Run softirq if any pending. And do it in its own stack
 188                  * as we may be calling this deep in a task call stack already.
 189                  */
 190                 do_softirq();
 191         }
 192
 193         preempt_count_dec();
 194 #ifdef CONFIG_TRACE_IRQFLAGS
 195         local_irq_enable();
 196 #endif
 197         preempt_check_resched();
 198 }
 199 EXPORT_SYMBOL(__local_bh_enable_ip);
 200
 201 /*
 202  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 203  * but break the loop if need_resched() is set or after 2 ms.
 204  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 205  * certain cases, such as stop_machine(), jiffies may cease to
 206  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 207  * well to make sure we eventually return from this method.
 208  *
 209  * These limits have been established via experimentation.
 210  * The two things to balance is latency against fairness -
 211  * we want to handle softirqs as soon as possible, but they
 212  * should not be able to lock up the box.
 213  */
 214 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 215 #define MAX_SOFTIRQ_RESTART 10
 216
 217 #ifdef CONFIG_TRACE_IRQFLAGS
 218 /*
 219  * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 220  * to keep the lockdep irq context tracking as tight as possible in order to
 221  * not miss-qualify lock contexts and miss possible deadlocks.
 222  */
 223
 224 static inline bool lockdep_softirq_start(void)
 225 {
 226         bool in_hardirq = false;
 227
 228         if (trace_hardirq_context(current)) {
 229                 in_hardirq = true;
 230                 trace_hardirq_exit();
 231         }
 232
 233         lockdep_softirq_enter();
 234
 235         return in_hardirq;
 236 }
 237
 238 static inline void lockdep_softirq_end(bool in_hardirq)
 239 {
 240         lockdep_softirq_exit();
 241
 242         if (in_hardirq)
 243                 trace_hardirq_enter();
 244 }
 245 #else
 246 static inline bool lockdep_softirq_start(void) { return false; }
 247 static inline void lockdep_softirq_end(bool in_hardirq) { }
 248 #endif
 249
 250 asmlinkage __visible void __softirq_entry __do_softirq(void)
 251 {
 252         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 253         unsigned long old_flags = current->flags;
 254         int max_restart = MAX_SOFTIRQ_RESTART;
 255         struct softirq_action *h;
 256         bool in_hardirq;
 257         __u32 pending;
 258         int softirq_bit;
 259
 260         /*
 261          * Mask out PF_MEMALLOC as the current task context is borrowed for the
 262          * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
 263          * again if the socket is related to swapping.
 264          */
 265         current->flags &= ~PF_MEMALLOC;
 266
 267         pending = local_softirq_pending();
 268         account_irq_enter_time(current);
 269
 270         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 271         in_hardirq = lockdep_softirq_start();
 272
 273 restart:
 274         /* Reset the pending bitmask before enabling irqs */
 275         set_softirq_pending(0);
 276
 277         local_irq_enable();
 278
 279         h = softirq_vec;
 280
 281         while ((softirq_bit = ffs(pending))) {
 282                 unsigned int vec_nr;
 283                 int prev_count;
 284
 285                 h += softirq_bit - 1;
 286
 287                 vec_nr = h - softirq_vec;
 288                 prev_count = preempt_count();
 289
 290                 kstat_incr_softirqs_this_cpu(vec_nr);
 291
 292                 trace_softirq_entry(vec_nr);
 293                 h->action(h);
 294                 trace_softirq_exit(vec_nr);
 295                 if (unlikely(prev_count != preempt_count())) {
 296                         pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
 297                                vec_nr, softirq_to_name[vec_nr], h->action,
 298                                prev_count, preempt_count());
 299                         preempt_count_set(prev_count);
 300                 }
 301                 h++;
 302                 pending >>= softirq_bit;
 303         }
 304
 305         if (__this_cpu_read(ksoftirqd) == current)
 306                 rcu_softirq_qs();
 307         local_irq_disable();
 308
 309         pending = local_softirq_pending();
 310         if (pending) {
 311                 if (time_before(jiffies, end) && !need_resched() &&
 312                     --max_restart)
 313                         goto restart;
 314
 315                 wakeup_softirqd();
 316         }
 317
 318         lockdep_softirq_end(in_hardirq);
 319         account_irq_exit_time(current);
 320         __local_bh_enable(SOFTIRQ_OFFSET);
 321         WARN_ON_ONCE(in_interrupt());
 322         current_restore_flags(old_flags, PF_MEMALLOC);
 323 }
 324
 325 asmlinkage __visible void do_softirq(void)
 326 {
 327         __u32 pending;
 328         unsigned long flags;
 329
 330         if (in_interrupt())
 331                 return;
 332
 333         local_irq_save(flags);
 334
 335         pending = local_softirq_pending();
 336
 337         if (pending && !ksoftirqd_running(pending))
 338                 do_softirq_own_stack();
 339
 340         local_irq_restore(flags);
 341 }
 342
 343 /*
 344  * Enter an interrupt context.
 345  */
 346 void irq_enter(void)
 347 {
 348         rcu_irq_enter();
 349         if (is_idle_task(current) && !in_interrupt()) {
 350                 /*
 351                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 352                  * here, as softirq will be serviced on return from interrupt.
 353                  */
 354                 local_bh_disable();
 355                 tick_irq_enter();
 356                 _local_bh_enable();
 357         }
 358
 359         __irq_enter();
 360 }
 361
 362 static inline void invoke_softirq(void)
 363 {
 364         if (ksoftirqd_running(local_softirq_pending()))
 365                 return;
 366
 367         if (!force_irqthreads) {
 368 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 369                 /*
 370                  * We can safely execute softirq on the current stack if
 371                  * it is the irq stack, because it should be near empty
 372                  * at this stage.
 373                  */
 374                 __do_softirq();
 375 #else
 376                 /*
 377                  * Otherwise, irq_exit() is called on the task stack that can
 378                  * be potentially deep already. So call softirq in its own stack
 379                  * to prevent from any overrun.
 380                  */
 381                 do_softirq_own_stack();
 382 #endif
 383         } else {
 384                 wakeup_softirqd();
 385         }
 386 }
 387
 388 static inline void tick_irq_exit(void)
 389 {
 390 #ifdef CONFIG_NO_HZ_COMMON
 391         int cpu = smp_processor_id();
 392
 393         /* Make sure that timer wheel updates are propagated */
 394         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 395                 if (!in_irq())
 396                         tick_nohz_irq_exit();
 397         }
 398 #endif
 399 }
 400
 401 /*
 402  * Exit an interrupt context. Process softirqs if needed and possible:
 403  */
 404 void irq_exit(void)
 405 {
 406 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 407         local_irq_disable();
 408 #else
 409         lockdep_assert_irqs_disabled();
 410 #endif
 411         account_irq_exit_time(current);
 412         preempt_count_sub(HARDIRQ_OFFSET);
 413         if (!in_interrupt() && local_softirq_pending())
 414                 invoke_softirq();
 415
 416         tick_irq_exit();
 417         rcu_irq_exit();
 418         trace_hardirq_exit(); /* must be last! */
 419 }
 420
 421 /*
 422  * This function must run with irqs disabled!
 423  */
 424 inline void raise_softirq_irqoff(unsigned int nr)
 425 {
 426         __raise_softirq_irqoff(nr);
 427
 428         /*
 429          * If we're in an interrupt or softirq, we're done
 430          * (this also catches softirq-disabled code). We will
 431          * actually run the softirq once we return from
 432          * the irq or softirq.
 433          *
 434          * Otherwise we wake up ksoftirqd to make sure we
 435          * schedule the softirq soon.
 436          */
 437         if (!in_interrupt())
 438                 wakeup_softirqd();
 439 }
 440
 441 void raise_softirq(unsigned int nr)
 442 {
 443         unsigned long flags;
 444
 445         local_irq_save(flags);
 446         raise_softirq_irqoff(nr);
 447         local_irq_restore(flags);
 448 }
 449
 450 void __raise_softirq_irqoff(unsigned int nr)
 451 {
 452         trace_softirq_raise(nr);
 453         or_softirq_pending(1UL << nr);
 454 }
 455
 456 void open_softirq(int nr, void (*action)(struct softirq_action *))
 457 {
 458         softirq_vec[nr].action = action;
 459 }
 460
 461 /*
 462  * Tasklets
 463  */
 464 struct tasklet_head {
 465         struct tasklet_struct *head;
 466         struct tasklet_struct **tail;
 467 };
 468
 469 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 470 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 471
 472 static void __tasklet_schedule_common(struct tasklet_struct *t,
 473                                       struct tasklet_head __percpu *headp,
 474                                       unsigned int softirq_nr)
 475 {
 476         struct tasklet_head *head;
 477         unsigned long flags;
 478
 479         local_irq_save(flags);
 480         head = this_cpu_ptr(headp);
 481         t->next = NULL;
 482         *head->tail = t;
 483         head->tail = &(t->next);
 484         raise_softirq_irqoff(softirq_nr);
 485         local_irq_restore(flags);
 486 }
 487
 488 void __tasklet_schedule(struct tasklet_struct *t)
 489 {
 490         __tasklet_schedule_common(t, &tasklet_vec,
 491                                   TASKLET_SOFTIRQ);
 492 }
 493 EXPORT_SYMBOL(__tasklet_schedule);
 494
 495 void __tasklet_hi_schedule(struct tasklet_struct *t)
 496 {
 497         __tasklet_schedule_common(t, &tasklet_hi_vec,
 498                                   HI_SOFTIRQ);
 499 }
 500 EXPORT_SYMBOL(__tasklet_hi_schedule);
 501
 502 static void tasklet_action_common(struct softirq_action *a,
 503                                   struct tasklet_head *tl_head,
 504                                   unsigned int softirq_nr)
 505 {
 506         struct tasklet_struct *list;
 507
 508         local_irq_disable();
 509         list = tl_head->head;
 510         tl_head->head = NULL;
 511         tl_head->tail = &tl_head->head;
 512         local_irq_enable();
 513
 514         while (list) {
 515                 struct tasklet_struct *t = list;
 516
 517                 list = list->next;
 518
 519                 if (tasklet_trylock(t)) {
 520                         if (!atomic_read(&t->count)) {
 521                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
 522                                                         &t->state))
 523                                         BUG();
 524                                 t->func(t->data);
 525                                 tasklet_unlock(t);
 526                                 continue;
 527                         }
 528                         tasklet_unlock(t);
 529                 }
 530
 531                 local_irq_disable();
 532                 t->next = NULL;
 533                 *tl_head->tail = t;
 534                 tl_head->tail = &t->next;
 535                 __raise_softirq_irqoff(softirq_nr);
 536                 local_irq_enable();
 537         }
 538 }
 539
 540 static __latent_entropy void tasklet_action(struct softirq_action *a)
 541 {
 542         tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
 543 }
 544
 545 static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
 546 {
 547         tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
 548 }
 549
 550 void tasklet_init(struct tasklet_struct *t,
 551                   void (*func)(unsigned long), unsigned long data)
 552 {
 553         t->next = NULL;
 554         t->state = 0;
 555         atomic_set(&t->count, 0);
 556         t->func = func;
 557         t->data = data;
 558 }
 559 EXPORT_SYMBOL(tasklet_init);
 560
 561 void tasklet_kill(struct tasklet_struct *t)
 562 {
 563         if (in_interrupt())
 564                 pr_notice("Attempt to kill tasklet from interrupt\n");
 565
 566         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 567                 do {
 568                         yield();
 569                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 570         }
 571         tasklet_unlock_wait(t);
 572         clear_bit(TASKLET_STATE_SCHED, &t->state);
 573 }
 574 EXPORT_SYMBOL(tasklet_kill);
 575
 576 /*
 577  * tasklet_hrtimer
 578  */
 579
 580 /*
 581  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 582  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 583  * hrtimer callback, but from softirq context.
 584  */
 585 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 586 {
 587         struct tasklet_hrtimer *ttimer =
 588                 container_of(timer, struct tasklet_hrtimer, timer);
 589
 590         tasklet_hi_schedule(&ttimer->tasklet);
 591         return HRTIMER_NORESTART;
 592 }
 593
 594 /*
 595  * Helper function which calls the hrtimer callback from
 596  * tasklet/softirq context
 597  */
 598 static void __tasklet_hrtimer_trampoline(unsigned long data)
 599 {
 600         struct tasklet_hrtimer *ttimer = (void *)data;
 601         enum hrtimer_restart restart;
 602
 603         restart = ttimer->function(&ttimer->timer);
 604         if (restart != HRTIMER_NORESTART)
 605                 hrtimer_restart(&ttimer->timer);
 606 }
 607
 608 /**
 609  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 610  * @ttimer:      tasklet_hrtimer which is initialized
 611  * @function:    hrtimer callback function which gets called from softirq context
 612  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 613  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 614  */
 615 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 616                           enum hrtimer_restart (*function)(struct hrtimer *),
 617                           clockid_t which_clock, enum hrtimer_mode mode)
 618 {
 619         hrtimer_init(&ttimer->timer, which_clock, mode);
 620         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 621         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 622                      (unsigned long)ttimer);
 623         ttimer->function = function;
 624 }
 625 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 626
 627 void __init softirq_init(void)
 628 {
 629         int cpu;
 630
 631         for_each_possible_cpu(cpu) {
 632                 per_cpu(tasklet_vec, cpu).tail =
 633                         &per_cpu(tasklet_vec, cpu).head;
 634                 per_cpu(tasklet_hi_vec, cpu).tail =
 635                         &per_cpu(tasklet_hi_vec, cpu).head;
 636         }
 637
 638         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 639         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 640 }
 641
 642 static int ksoftirqd_should_run(unsigned int cpu)
 643 {
 644         return local_softirq_pending();
 645 }
 646
 647 static void run_ksoftirqd(unsigned int cpu)
 648 {
 649         local_irq_disable();
 650         if (local_softirq_pending()) {
 651                 /*
 652                  * We can safely run softirq on inline stack, as we are not deep
 653                  * in the task stack here.
 654                  */
 655                 __do_softirq();
 656                 local_irq_enable();
 657                 cond_resched();
 658                 return;
 659         }
 660         local_irq_enable();
 661 }
 662
 663 #ifdef CONFIG_HOTPLUG_CPU
 664 /*
 665  * tasklet_kill_immediate is called to remove a tasklet which can already be
 666  * scheduled for execution on @cpu.
 667  *
 668  * Unlike tasklet_kill, this function removes the tasklet
 669  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 670  *
 671  * When this function is called, @cpu must be in the CPU_DEAD state.
 672  */
 673 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 674 {
 675         struct tasklet_struct **i;
 676
 677         BUG_ON(cpu_online(cpu));
 678         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 679
 680         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 681                 return;
 682
 683         /* CPU is dead, so no lock needed. */
 684         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 685                 if (*i == t) {
 686                         *i = t->next;
 687                         /* If this was the tail element, move the tail ptr */
 688                         if (*i == NULL)
 689                                 per_cpu(tasklet_vec, cpu).tail = i;
 690                         return;
 691                 }
 692         }
 693         BUG();
 694 }
 695
 696 static int takeover_tasklets(unsigned int cpu)
 697 {
 698         /* CPU is dead, so no lock needed. */
 699         local_irq_disable();
 700
 701         /* Find end, append list for that CPU. */
 702         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 703                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 704                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 705                 per_cpu(tasklet_vec, cpu).head = NULL;
 706                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 707         }
 708         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 709
 710         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 711                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 712                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 713                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 714                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 715         }
 716         raise_softirq_irqoff(HI_SOFTIRQ);
 717
 718         local_irq_enable();
 719         return 0;
 720 }
 721 #else
 722 #define takeover_tasklets       NULL
 723 #endif /* CONFIG_HOTPLUG_CPU */
 724
 725 static struct smp_hotplug_thread softirq_threads = {
 726         .store                  = &ksoftirqd,
 727         .thread_should_run      = ksoftirqd_should_run,
 728         .thread_fn              = run_ksoftirqd,
 729         .thread_comm            = "ksoftirqd/%u",
 730 };
 731
 732 static __init int spawn_ksoftirqd(void)
 733 {
 734         cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
 735                                   takeover_tasklets);
 736         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 737
 738         return 0;
 739 }
 740 early_initcall(spawn_ksoftirqd);
 741
 742 /*
 743  * [ These __weak aliases are kept in a separate compilation unit, so that
 744  *   GCC does not inline them incorrectly. ]
 745  */
 746
 747 int __init __weak early_irq_init(void)
 748 {
 749         return 0;
 750 }
 751
 752 int __init __weak arch_probe_nr_irqs(void)
 753 {
 754         return NR_IRQS_LEGACY;
 755 }
 756
 757 int __init __weak arch_early_irq_init(void)
 758 {
 759         return 0;
 760 }
 761
 762 unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
 763 {
 764         return from;
 765 }