arch/x86/kernel/smpboot.c

   1 #include <linux/init.h>
   2 #include <linux/smp.h>
   3 #include <linux/module.h>
   4 #include <linux/sched.h>
   5 #include <linux/percpu.h>
   6 #include <linux/bootmem.h>
   7
   8 #include <asm/nmi.h>
   9 #include <asm/irq.h>
  10 #include <asm/smp.h>
  11 #include <asm/cpu.h>
  12 #include <asm/numa.h>
  13
  14 /* Number of siblings per CPU package */
  15 int smp_num_siblings = 1;
  16 EXPORT_SYMBOL(smp_num_siblings);
  17
  18 /* Last level cache ID of each logical CPU */
  19 DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
  20
  21 /* bitmap of online cpus */
  22 cpumask_t cpu_online_map __read_mostly;
  23 EXPORT_SYMBOL(cpu_online_map);
  24
  25 cpumask_t cpu_callin_map;
  26 cpumask_t cpu_callout_map;
  27 cpumask_t cpu_possible_map;
  28 EXPORT_SYMBOL(cpu_possible_map);
  29
  30 /* representing HT siblings of each logical CPU */
  31 DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
  32 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
  33
  34 /* representing HT and core siblings of each logical CPU */
  35 DEFINE_PER_CPU(cpumask_t, cpu_core_map);
  36 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
  37
  38 /* Per CPU bogomips and other parameters */
  39 DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
  40 EXPORT_PER_CPU_SYMBOL(cpu_info);
  41
  42 /* ready for x86_64, no harm for x86, since it will overwrite after alloc */
  43 unsigned char *trampoline_base = __va(SMP_TRAMPOLINE_BASE);
  44
  45 /* representing cpus for which sibling maps can be computed */
  46 static cpumask_t cpu_sibling_setup_map;
  47
  48 void __cpuinit set_cpu_sibling_map(int cpu)
  49 {
  50         int i;
  51         struct cpuinfo_x86 *c = &cpu_data(cpu);
  52
  53         cpu_set(cpu, cpu_sibling_setup_map);
  54
  55         if (smp_num_siblings > 1) {
  56                 for_each_cpu_mask(i, cpu_sibling_setup_map) {
  57                         if (c->phys_proc_id == cpu_data(i).phys_proc_id &&
  58                             c->cpu_core_id == cpu_data(i).cpu_core_id) {
  59                                 cpu_set(i, per_cpu(cpu_sibling_map, cpu));
  60                                 cpu_set(cpu, per_cpu(cpu_sibling_map, i));
  61                                 cpu_set(i, per_cpu(cpu_core_map, cpu));
  62                                 cpu_set(cpu, per_cpu(cpu_core_map, i));
  63                                 cpu_set(i, c->llc_shared_map);
  64                                 cpu_set(cpu, cpu_data(i).llc_shared_map);
  65                         }
  66                 }
  67         } else {
  68                 cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
  69         }
  70
  71         cpu_set(cpu, c->llc_shared_map);
  72
  73         if (current_cpu_data.x86_max_cores == 1) {
  74                 per_cpu(cpu_core_map, cpu) = per_cpu(cpu_sibling_map, cpu);
  75                 c->booted_cores = 1;
  76                 return;
  77         }
  78
  79         for_each_cpu_mask(i, cpu_sibling_setup_map) {
  80                 if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
  81                     per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
  82                         cpu_set(i, c->llc_shared_map);
  83                         cpu_set(cpu, cpu_data(i).llc_shared_map);
  84                 }
  85                 if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
  86                         cpu_set(i, per_cpu(cpu_core_map, cpu));
  87                         cpu_set(cpu, per_cpu(cpu_core_map, i));
  88                         /*
  89                          *  Does this new cpu bringup a new core?
  90                          */
  91                         if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) {
  92                                 /*
  93                                  * for each core in package, increment
  94                                  * the booted_cores for this new cpu
  95                                  */
  96                                 if (first_cpu(per_cpu(cpu_sibling_map, i)) == i)
  97                                         c->booted_cores++;
  98                                 /*
  99                                  * increment the core count for all
 100                                  * the other cpus in this package
 101                                  */
 102                                 if (i != cpu)
 103                                         cpu_data(i).booted_cores++;
 104                         } else if (i != cpu && !c->booted_cores)
 105                                 c->booted_cores = cpu_data(i).booted_cores;
 106                 }
 107         }
 108 }
 109
 110 /* maps the cpu to the sched domain representing multi-core */
 111 cpumask_t cpu_coregroup_map(int cpu)
 112 {
 113         struct cpuinfo_x86 *c = &cpu_data(cpu);
 114         /*
 115          * For perf, we return last level cache shared map.
 116          * And for power savings, we return cpu_core_map
 117          */
 118         if (sched_mc_power_savings || sched_smt_power_savings)
 119                 return per_cpu(cpu_core_map, cpu);
 120         else
 121                 return c->llc_shared_map;
 122 }
 123
 124 /*
 125  * Currently trivial. Write the real->protected mode
 126  * bootstrap into the page concerned. The caller
 127  * has made sure it's suitably aligned.
 128  */
 129
 130 unsigned long __cpuinit setup_trampoline(void)
 131 {
 132         memcpy(trampoline_base, trampoline_data,
 133                trampoline_end - trampoline_data);
 134         return virt_to_phys(trampoline_base);
 135 }
 136
 137 #ifdef CONFIG_X86_32
 138 /*
 139  * We are called very early to get the low memory for the
 140  * SMP bootup trampoline page.
 141  */
 142 void __init smp_alloc_memory(void)
 143 {
 144         trampoline_base = alloc_bootmem_low_pages(PAGE_SIZE);
 145         /*
 146          * Has to be in very low memory so we can execute
 147          * real-mode AP code.
 148          */
 149         if (__pa(trampoline_base) >= 0x9F000)
 150                 BUG();
 151 }
 152 #endif
 153
 154 #ifdef CONFIG_HOTPLUG_CPU
 155 void remove_siblinginfo(int cpu)
 156 {
 157         int sibling;
 158         struct cpuinfo_x86 *c = &cpu_data(cpu);
 159
 160         for_each_cpu_mask(sibling, per_cpu(cpu_core_map, cpu)) {
 161                 cpu_clear(cpu, per_cpu(cpu_core_map, sibling));
 162                 /*/
 163                  * last thread sibling in this cpu core going down
 164                  */
 165                 if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1)
 166                         cpu_data(sibling).booted_cores--;
 167         }
 168
 169         for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu))
 170                 cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling));
 171         cpus_clear(per_cpu(cpu_sibling_map, cpu));
 172         cpus_clear(per_cpu(cpu_core_map, cpu));
 173         c->phys_proc_id = 0;
 174         c->cpu_core_id = 0;
 175         cpu_clear(cpu, cpu_sibling_setup_map);
 176 }
 177
 178 int additional_cpus __initdata = -1;
 179
 180 static __init int setup_additional_cpus(char *s)
 181 {
 182         return s && get_option(&s, &additional_cpus) ? 0 : -EINVAL;
 183 }
 184 early_param("additional_cpus", setup_additional_cpus);
 185
 186 /*
 187  * cpu_possible_map should be static, it cannot change as cpu's
 188  * are onlined, or offlined. The reason is per-cpu data-structures
 189  * are allocated by some modules at init time, and dont expect to
 190  * do this dynamically on cpu arrival/departure.
 191  * cpu_present_map on the other hand can change dynamically.
 192  * In case when cpu_hotplug is not compiled, then we resort to current
 193  * behaviour, which is cpu_possible == cpu_present.
 194  * - Ashok Raj
 195  *
 196  * Three ways to find out the number of additional hotplug CPUs:
 197  * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
 198  * - The user can overwrite it with additional_cpus=NUM
 199  * - Otherwise don't reserve additional CPUs.
 200  * We do this because additional CPUs waste a lot of memory.
 201  * -AK
 202  */
 203 __init void prefill_possible_map(void)
 204 {
 205         int i;
 206         int possible;
 207
 208         if (additional_cpus == -1) {
 209                 if (disabled_cpus > 0)
 210                         additional_cpus = disabled_cpus;
 211                 else
 212                         additional_cpus = 0;
 213         }
 214         possible = num_processors + additional_cpus;
 215         if (possible > NR_CPUS)
 216                 possible = NR_CPUS;
 217
 218         printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
 219                 possible, max_t(int, possible - num_processors, 0));
 220
 221         for (i = 0; i < possible; i++)
 222                 cpu_set(i, cpu_possible_map);
 223 }
 224
 225 static void __ref remove_cpu_from_maps(int cpu)
 226 {
 227         cpu_clear(cpu, cpu_online_map);
 228 #ifdef CONFIG_X86_64
 229         cpu_clear(cpu, cpu_callout_map);
 230         cpu_clear(cpu, cpu_callin_map);
 231         /* was set by cpu_init() */
 232         clear_bit(cpu, (unsigned long *)&cpu_initialized);
 233         clear_node_cpumask(cpu);
 234 #endif
 235 }
 236
 237 int __cpu_disable(void)
 238 {
 239         int cpu = smp_processor_id();
 240
 241         /*
 242          * Perhaps use cpufreq to drop frequency, but that could go
 243          * into generic code.
 244          *
 245          * We won't take down the boot processor on i386 due to some
 246          * interrupts only being able to be serviced by the BSP.
 247          * Especially so if we're not using an IOAPIC   -zwane
 248          */
 249         if (cpu == 0)
 250                 return -EBUSY;
 251
 252         if (nmi_watchdog == NMI_LOCAL_APIC)
 253                 stop_apic_nmi_watchdog(NULL);
 254         clear_local_APIC();
 255
 256         /*
 257          * HACK:
 258          * Allow any queued timer interrupts to get serviced
 259          * This is only a temporary solution until we cleanup
 260          * fixup_irqs as we do for IA64.
 261          */
 262         local_irq_enable();
 263         mdelay(1);
 264
 265         local_irq_disable();
 266         remove_siblinginfo(cpu);
 267
 268         /* It's now safe to remove this processor from the online map */
 269         remove_cpu_from_maps(cpu);
 270         fixup_irqs(cpu_online_map);
 271         return 0;
 272 }
 273
 274 void __cpu_die(unsigned int cpu)
 275 {
 276         /* We don't do anything here: idle task is faking death itself. */
 277         unsigned int i;
 278
 279         for (i = 0; i < 10; i++) {
 280                 /* They ack this in play_dead by setting CPU_DEAD */
 281                 if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
 282                         printk(KERN_INFO "CPU %d is now offline\n", cpu);
 283                         if (1 == num_online_cpus())
 284                                 alternatives_smp_switch(0);
 285                         return;
 286                 }
 287                 msleep(100);
 288         }
 289         printk(KERN_ERR "CPU %u didn't die...\n", cpu);
 290 }
 291 #else /* ... !CONFIG_HOTPLUG_CPU */
 292 int __cpu_disable(void)
 293 {
 294         return -ENOSYS;
 295 }
 296
 297 void __cpu_die(unsigned int cpu)
 298 {
 299         /* We said "no" in __cpu_disable */
 300         BUG();
 301 }
 302 #endif
 303
 304 /*
 305  * If the BIOS enumerates physical processors before logical,
 306  * maxcpus=N at enumeration-time can be used to disable HT.
 307  */
 308 static int __init parse_maxcpus(char *arg)
 309 {
 310         extern unsigned int maxcpus;
 311
 312         maxcpus = simple_strtoul(arg, NULL, 0);
 313         return 0;
 314 }
 315 early_param("maxcpus", parse_maxcpus);