xtensa: replace CONFIG_DEBUG_KERNEL with CONFIG_DEBUG_MISC
[sfrench/cifs-2.6.git] / arch / xtensa / kernel / smp.c
1 /*
2  * Xtensa SMP support functions.
3  *
4  * This file is subject to the terms and conditions of the GNU General Public
5  * License.  See the file "COPYING" in the main directory of this archive
6  * for more details.
7  *
8  * Copyright (C) 2008 - 2013 Tensilica Inc.
9  *
10  * Chris Zankel <chris@zankel.net>
11  * Joe Taylor <joe@tensilica.com>
12  * Pete Delaney <piet@tensilica.com
13  */
14
15 #include <linux/cpu.h>
16 #include <linux/cpumask.h>
17 #include <linux/delay.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
20 #include <linux/irqdomain.h>
21 #include <linux/irq.h>
22 #include <linux/kdebug.h>
23 #include <linux/module.h>
24 #include <linux/sched/mm.h>
25 #include <linux/sched/hotplug.h>
26 #include <linux/sched/task_stack.h>
27 #include <linux/reboot.h>
28 #include <linux/seq_file.h>
29 #include <linux/smp.h>
30 #include <linux/thread_info.h>
31
32 #include <asm/cacheflush.h>
33 #include <asm/kdebug.h>
34 #include <asm/mmu_context.h>
35 #include <asm/mxregs.h>
36 #include <asm/platform.h>
37 #include <asm/tlbflush.h>
38 #include <asm/traps.h>
39
40 #ifdef CONFIG_SMP
41 # if XCHAL_HAVE_S32C1I == 0
42 #  error "The S32C1I option is required for SMP."
43 # endif
44 #endif
45
46 static void system_invalidate_dcache_range(unsigned long start,
47                 unsigned long size);
48 static void system_flush_invalidate_dcache_range(unsigned long start,
49                 unsigned long size);
50
51 /* IPI (Inter Process Interrupt) */
52
53 #define IPI_IRQ 0
54
55 static irqreturn_t ipi_interrupt(int irq, void *dev_id);
56 static struct irqaction ipi_irqaction = {
57         .handler =      ipi_interrupt,
58         .flags =        IRQF_PERCPU,
59         .name =         "ipi",
60 };
61
62 void ipi_init(void)
63 {
64         unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
65         setup_irq(irq, &ipi_irqaction);
66 }
67
68 static inline unsigned int get_core_count(void)
69 {
70         /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
71         unsigned int syscfgid = get_er(SYSCFGID);
72         return ((syscfgid >> 18) & 0xf) + 1;
73 }
74
75 static inline int get_core_id(void)
76 {
77         /* Bits 0...18 of SYSCFGID contain the core id  */
78         unsigned int core_id = get_er(SYSCFGID);
79         return core_id & 0x3fff;
80 }
81
82 void __init smp_prepare_cpus(unsigned int max_cpus)
83 {
84         unsigned i;
85
86         for_each_possible_cpu(i)
87                 set_cpu_present(i, true);
88 }
89
90 void __init smp_init_cpus(void)
91 {
92         unsigned i;
93         unsigned int ncpus = get_core_count();
94         unsigned int core_id = get_core_id();
95
96         pr_info("%s: Core Count = %d\n", __func__, ncpus);
97         pr_info("%s: Core Id = %d\n", __func__, core_id);
98
99         if (ncpus > NR_CPUS) {
100                 ncpus = NR_CPUS;
101                 pr_info("%s: limiting core count by %d\n", __func__, ncpus);
102         }
103
104         for (i = 0; i < ncpus; ++i)
105                 set_cpu_possible(i, true);
106 }
107
108 void __init smp_prepare_boot_cpu(void)
109 {
110         unsigned int cpu = smp_processor_id();
111         BUG_ON(cpu != 0);
112         cpu_asid_cache(cpu) = ASID_USER_FIRST;
113 }
114
115 void __init smp_cpus_done(unsigned int max_cpus)
116 {
117 }
118
119 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
120 static DECLARE_COMPLETION(cpu_running);
121
122 void secondary_start_kernel(void)
123 {
124         struct mm_struct *mm = &init_mm;
125         unsigned int cpu = smp_processor_id();
126
127         init_mmu();
128
129 #ifdef CONFIG_DEBUG_MISC
130         if (boot_secondary_processors == 0) {
131                 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
132                         __func__, boot_secondary_processors, cpu);
133                 for (;;)
134                         __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
135         }
136
137         pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
138                 __func__, boot_secondary_processors, cpu);
139 #endif
140         /* Init EXCSAVE1 */
141
142         secondary_trap_init();
143
144         /* All kernel threads share the same mm context. */
145
146         mmget(mm);
147         mmgrab(mm);
148         current->active_mm = mm;
149         cpumask_set_cpu(cpu, mm_cpumask(mm));
150         enter_lazy_tlb(mm, current);
151
152         preempt_disable();
153         trace_hardirqs_off();
154
155         calibrate_delay();
156
157         notify_cpu_starting(cpu);
158
159         secondary_init_irq();
160         local_timer_setup(cpu);
161
162         set_cpu_online(cpu, true);
163
164         local_irq_enable();
165
166         complete(&cpu_running);
167
168         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
169 }
170
171 static void mx_cpu_start(void *p)
172 {
173         unsigned cpu = (unsigned)p;
174         unsigned long run_stall_mask = get_er(MPSCORE);
175
176         set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
177         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
178                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
179 }
180
181 static void mx_cpu_stop(void *p)
182 {
183         unsigned cpu = (unsigned)p;
184         unsigned long run_stall_mask = get_er(MPSCORE);
185
186         set_er(run_stall_mask | (1u << cpu), MPSCORE);
187         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
188                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
189 }
190
191 #ifdef CONFIG_HOTPLUG_CPU
192 unsigned long cpu_start_id __cacheline_aligned;
193 #endif
194 unsigned long cpu_start_ccount;
195
196 static int boot_secondary(unsigned int cpu, struct task_struct *ts)
197 {
198         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
199         unsigned long ccount;
200         int i;
201
202 #ifdef CONFIG_HOTPLUG_CPU
203         WRITE_ONCE(cpu_start_id, cpu);
204         /* Pairs with the third memw in the cpu_restart */
205         mb();
206         system_flush_invalidate_dcache_range((unsigned long)&cpu_start_id,
207                                              sizeof(cpu_start_id));
208 #endif
209         smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
210
211         for (i = 0; i < 2; ++i) {
212                 do
213                         ccount = get_ccount();
214                 while (!ccount);
215
216                 WRITE_ONCE(cpu_start_ccount, ccount);
217
218                 do {
219                         /*
220                          * Pairs with the first two memws in the
221                          * .Lboot_secondary.
222                          */
223                         mb();
224                         ccount = READ_ONCE(cpu_start_ccount);
225                 } while (ccount && time_before(jiffies, timeout));
226
227                 if (ccount) {
228                         smp_call_function_single(0, mx_cpu_stop,
229                                                  (void *)cpu, 1);
230                         WRITE_ONCE(cpu_start_ccount, 0);
231                         return -EIO;
232                 }
233         }
234         return 0;
235 }
236
237 int __cpu_up(unsigned int cpu, struct task_struct *idle)
238 {
239         int ret = 0;
240
241         if (cpu_asid_cache(cpu) == 0)
242                 cpu_asid_cache(cpu) = ASID_USER_FIRST;
243
244         start_info.stack = (unsigned long)task_pt_regs(idle);
245         wmb();
246
247         pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
248                         __func__, cpu, idle, start_info.stack);
249
250         init_completion(&cpu_running);
251         ret = boot_secondary(cpu, idle);
252         if (ret == 0) {
253                 wait_for_completion_timeout(&cpu_running,
254                                 msecs_to_jiffies(1000));
255                 if (!cpu_online(cpu))
256                         ret = -EIO;
257         }
258
259         if (ret)
260                 pr_err("CPU %u failed to boot\n", cpu);
261
262         return ret;
263 }
264
265 #ifdef CONFIG_HOTPLUG_CPU
266
267 /*
268  * __cpu_disable runs on the processor to be shutdown.
269  */
270 int __cpu_disable(void)
271 {
272         unsigned int cpu = smp_processor_id();
273
274         /*
275          * Take this CPU offline.  Once we clear this, we can't return,
276          * and we must not schedule until we're ready to give up the cpu.
277          */
278         set_cpu_online(cpu, false);
279
280         /*
281          * OK - migrate IRQs away from this CPU
282          */
283         migrate_irqs();
284
285         /*
286          * Flush user cache and TLB mappings, and then remove this CPU
287          * from the vm mask set of all processes.
288          */
289         local_flush_cache_all();
290         local_flush_tlb_all();
291         invalidate_page_directory();
292
293         clear_tasks_mm_cpumask(cpu);
294
295         return 0;
296 }
297
298 static void platform_cpu_kill(unsigned int cpu)
299 {
300         smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
301 }
302
303 /*
304  * called on the thread which is asking for a CPU to be shutdown -
305  * waits until shutdown has completed, or it is timed out.
306  */
307 void __cpu_die(unsigned int cpu)
308 {
309         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
310         while (time_before(jiffies, timeout)) {
311                 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
312                                                sizeof(cpu_start_id));
313                 /* Pairs with the second memw in the cpu_restart */
314                 mb();
315                 if (READ_ONCE(cpu_start_id) == -cpu) {
316                         platform_cpu_kill(cpu);
317                         return;
318                 }
319         }
320         pr_err("CPU%u: unable to kill\n", cpu);
321 }
322
323 void arch_cpu_idle_dead(void)
324 {
325         cpu_die();
326 }
327 /*
328  * Called from the idle thread for the CPU which has been shutdown.
329  *
330  * Note that we disable IRQs here, but do not re-enable them
331  * before returning to the caller. This is also the behaviour
332  * of the other hotplug-cpu capable cores, so presumably coming
333  * out of idle fixes this.
334  */
335 void __ref cpu_die(void)
336 {
337         idle_task_exit();
338         local_irq_disable();
339         __asm__ __volatile__(
340                         "       movi    a2, cpu_restart\n"
341                         "       jx      a2\n");
342 }
343
344 #endif /* CONFIG_HOTPLUG_CPU */
345
346 enum ipi_msg_type {
347         IPI_RESCHEDULE = 0,
348         IPI_CALL_FUNC,
349         IPI_CPU_STOP,
350         IPI_MAX
351 };
352
353 static const struct {
354         const char *short_text;
355         const char *long_text;
356 } ipi_text[] = {
357         { .short_text = "RES", .long_text = "Rescheduling interrupts" },
358         { .short_text = "CAL", .long_text = "Function call interrupts" },
359         { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
360 };
361
362 struct ipi_data {
363         unsigned long ipi_count[IPI_MAX];
364 };
365
366 static DEFINE_PER_CPU(struct ipi_data, ipi_data);
367
368 static void send_ipi_message(const struct cpumask *callmask,
369                 enum ipi_msg_type msg_id)
370 {
371         int index;
372         unsigned long mask = 0;
373
374         for_each_cpu(index, callmask)
375                 mask |= 1 << index;
376
377         set_er(mask, MIPISET(msg_id));
378 }
379
380 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
381 {
382         send_ipi_message(mask, IPI_CALL_FUNC);
383 }
384
385 void arch_send_call_function_single_ipi(int cpu)
386 {
387         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
388 }
389
390 void smp_send_reschedule(int cpu)
391 {
392         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
393 }
394
395 void smp_send_stop(void)
396 {
397         struct cpumask targets;
398
399         cpumask_copy(&targets, cpu_online_mask);
400         cpumask_clear_cpu(smp_processor_id(), &targets);
401         send_ipi_message(&targets, IPI_CPU_STOP);
402 }
403
404 static void ipi_cpu_stop(unsigned int cpu)
405 {
406         set_cpu_online(cpu, false);
407         machine_halt();
408 }
409
410 irqreturn_t ipi_interrupt(int irq, void *dev_id)
411 {
412         unsigned int cpu = smp_processor_id();
413         struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
414
415         for (;;) {
416                 unsigned int msg;
417
418                 msg = get_er(MIPICAUSE(cpu));
419                 set_er(msg, MIPICAUSE(cpu));
420
421                 if (!msg)
422                         break;
423
424                 if (msg & (1 << IPI_CALL_FUNC)) {
425                         ++ipi->ipi_count[IPI_CALL_FUNC];
426                         generic_smp_call_function_interrupt();
427                 }
428
429                 if (msg & (1 << IPI_RESCHEDULE)) {
430                         ++ipi->ipi_count[IPI_RESCHEDULE];
431                         scheduler_ipi();
432                 }
433
434                 if (msg & (1 << IPI_CPU_STOP)) {
435                         ++ipi->ipi_count[IPI_CPU_STOP];
436                         ipi_cpu_stop(cpu);
437                 }
438         }
439
440         return IRQ_HANDLED;
441 }
442
443 void show_ipi_list(struct seq_file *p, int prec)
444 {
445         unsigned int cpu;
446         unsigned i;
447
448         for (i = 0; i < IPI_MAX; ++i) {
449                 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
450                 for_each_online_cpu(cpu)
451                         seq_printf(p, " %10lu",
452                                         per_cpu(ipi_data, cpu).ipi_count[i]);
453                 seq_printf(p, "   %s\n", ipi_text[i].long_text);
454         }
455 }
456
457 int setup_profiling_timer(unsigned int multiplier)
458 {
459         pr_debug("setup_profiling_timer %d\n", multiplier);
460         return 0;
461 }
462
463 /* TLB flush functions */
464
465 struct flush_data {
466         struct vm_area_struct *vma;
467         unsigned long addr1;
468         unsigned long addr2;
469 };
470
471 static void ipi_flush_tlb_all(void *arg)
472 {
473         local_flush_tlb_all();
474 }
475
476 void flush_tlb_all(void)
477 {
478         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
479 }
480
481 static void ipi_flush_tlb_mm(void *arg)
482 {
483         local_flush_tlb_mm(arg);
484 }
485
486 void flush_tlb_mm(struct mm_struct *mm)
487 {
488         on_each_cpu(ipi_flush_tlb_mm, mm, 1);
489 }
490
491 static void ipi_flush_tlb_page(void *arg)
492 {
493         struct flush_data *fd = arg;
494         local_flush_tlb_page(fd->vma, fd->addr1);
495 }
496
497 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
498 {
499         struct flush_data fd = {
500                 .vma = vma,
501                 .addr1 = addr,
502         };
503         on_each_cpu(ipi_flush_tlb_page, &fd, 1);
504 }
505
506 static void ipi_flush_tlb_range(void *arg)
507 {
508         struct flush_data *fd = arg;
509         local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
510 }
511
512 void flush_tlb_range(struct vm_area_struct *vma,
513                      unsigned long start, unsigned long end)
514 {
515         struct flush_data fd = {
516                 .vma = vma,
517                 .addr1 = start,
518                 .addr2 = end,
519         };
520         on_each_cpu(ipi_flush_tlb_range, &fd, 1);
521 }
522
523 static void ipi_flush_tlb_kernel_range(void *arg)
524 {
525         struct flush_data *fd = arg;
526         local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
527 }
528
529 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
530 {
531         struct flush_data fd = {
532                 .addr1 = start,
533                 .addr2 = end,
534         };
535         on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
536 }
537
538 /* Cache flush functions */
539
540 static void ipi_flush_cache_all(void *arg)
541 {
542         local_flush_cache_all();
543 }
544
545 void flush_cache_all(void)
546 {
547         on_each_cpu(ipi_flush_cache_all, NULL, 1);
548 }
549
550 static void ipi_flush_cache_page(void *arg)
551 {
552         struct flush_data *fd = arg;
553         local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
554 }
555
556 void flush_cache_page(struct vm_area_struct *vma,
557                      unsigned long address, unsigned long pfn)
558 {
559         struct flush_data fd = {
560                 .vma = vma,
561                 .addr1 = address,
562                 .addr2 = pfn,
563         };
564         on_each_cpu(ipi_flush_cache_page, &fd, 1);
565 }
566
567 static void ipi_flush_cache_range(void *arg)
568 {
569         struct flush_data *fd = arg;
570         local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
571 }
572
573 void flush_cache_range(struct vm_area_struct *vma,
574                      unsigned long start, unsigned long end)
575 {
576         struct flush_data fd = {
577                 .vma = vma,
578                 .addr1 = start,
579                 .addr2 = end,
580         };
581         on_each_cpu(ipi_flush_cache_range, &fd, 1);
582 }
583
584 static void ipi_flush_icache_range(void *arg)
585 {
586         struct flush_data *fd = arg;
587         local_flush_icache_range(fd->addr1, fd->addr2);
588 }
589
590 void flush_icache_range(unsigned long start, unsigned long end)
591 {
592         struct flush_data fd = {
593                 .addr1 = start,
594                 .addr2 = end,
595         };
596         on_each_cpu(ipi_flush_icache_range, &fd, 1);
597 }
598 EXPORT_SYMBOL(flush_icache_range);
599
600 /* ------------------------------------------------------------------------- */
601
602 static void ipi_invalidate_dcache_range(void *arg)
603 {
604         struct flush_data *fd = arg;
605         __invalidate_dcache_range(fd->addr1, fd->addr2);
606 }
607
608 static void system_invalidate_dcache_range(unsigned long start,
609                 unsigned long size)
610 {
611         struct flush_data fd = {
612                 .addr1 = start,
613                 .addr2 = size,
614         };
615         on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
616 }
617
618 static void ipi_flush_invalidate_dcache_range(void *arg)
619 {
620         struct flush_data *fd = arg;
621         __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
622 }
623
624 static void system_flush_invalidate_dcache_range(unsigned long start,
625                 unsigned long size)
626 {
627         struct flush_data fd = {
628                 .addr1 = start,
629                 .addr2 = size,
630         };
631         on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
632 }