4 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
5 * deal of code from the sparc and intel versions.
7 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
9 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
10 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version
15 * 2 of the License, or (at your option) any later version.
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/sched.h>
23 #include <linux/smp.h>
24 #include <linux/interrupt.h>
25 #include <linux/delay.h>
26 #include <linux/init.h>
27 #include <linux/spinlock.h>
28 #include <linux/cache.h>
29 #include <linux/err.h>
30 #include <linux/sysdev.h>
31 #include <linux/cpu.h>
32 #include <linux/notifier.h>
33 #include <linux/topology.h>
35 #include <asm/ptrace.h>
36 #include <asm/atomic.h>
39 #include <asm/pgtable.h>
43 #include <asm/machdep.h>
44 #include <asm/cputhreads.h>
45 #include <asm/cputable.h>
46 #include <asm/system.h>
48 #include <asm/vdso_datapage.h>
55 #define DBG(fmt...) udbg_printf(fmt)
60 int smp_hw_index
[NR_CPUS
];
61 struct thread_info
*secondary_ti
;
63 cpumask_t cpu_possible_map
= CPU_MASK_NONE
;
64 cpumask_t cpu_online_map
= CPU_MASK_NONE
;
65 DEFINE_PER_CPU(cpumask_t
, cpu_sibling_map
) = CPU_MASK_NONE
;
66 DEFINE_PER_CPU(cpumask_t
, cpu_core_map
) = CPU_MASK_NONE
;
68 EXPORT_SYMBOL(cpu_online_map
);
69 EXPORT_SYMBOL(cpu_possible_map
);
70 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map
);
71 EXPORT_PER_CPU_SYMBOL(cpu_core_map
);
73 /* SMP operations for this machine */
74 struct smp_ops_t
*smp_ops
;
76 static volatile unsigned int cpu_callin_map
[NR_CPUS
];
78 int smt_enabled_at_boot
= 1;
80 static void (*crash_ipi_function_ptr
)(struct pt_regs
*) = NULL
;
83 void __devinit
smp_generic_kick_cpu(int nr
)
85 BUG_ON(nr
< 0 || nr
>= NR_CPUS
);
88 * The processor is currently spinning, waiting for the
89 * cpu_start field to become non-zero After we set cpu_start,
90 * the processor will continue on to secondary_start
92 paca
[nr
].cpu_start
= 1;
97 void smp_message_recv(int msg
)
100 case PPC_MSG_CALL_FUNCTION
:
101 generic_smp_call_function_interrupt();
103 case PPC_MSG_RESCHEDULE
:
104 /* XXX Do we have to do this? */
107 case PPC_MSG_CALL_FUNC_SINGLE
:
108 generic_smp_call_function_single_interrupt();
110 case PPC_MSG_DEBUGGER_BREAK
:
111 if (crash_ipi_function_ptr
) {
112 crash_ipi_function_ptr(get_irq_regs());
115 #ifdef CONFIG_DEBUGGER
116 debugger_ipi(get_irq_regs());
118 #endif /* CONFIG_DEBUGGER */
121 printk("SMP %d: smp_message_recv(): unknown msg %d\n",
122 smp_processor_id(), msg
);
127 void smp_send_reschedule(int cpu
)
130 smp_ops
->message_pass(cpu
, PPC_MSG_RESCHEDULE
);
133 void arch_send_call_function_single_ipi(int cpu
)
135 smp_ops
->message_pass(cpu
, PPC_MSG_CALL_FUNC_SINGLE
);
138 void arch_send_call_function_ipi(cpumask_t mask
)
142 for_each_cpu_mask(cpu
, mask
)
143 smp_ops
->message_pass(cpu
, PPC_MSG_CALL_FUNCTION
);
146 #ifdef CONFIG_DEBUGGER
147 void smp_send_debugger_break(int cpu
)
150 smp_ops
->message_pass(cpu
, PPC_MSG_DEBUGGER_BREAK
);
155 void crash_send_ipi(void (*crash_ipi_callback
)(struct pt_regs
*))
157 crash_ipi_function_ptr
= crash_ipi_callback
;
158 if (crash_ipi_callback
&& smp_ops
) {
160 smp_ops
->message_pass(MSG_ALL_BUT_SELF
, PPC_MSG_DEBUGGER_BREAK
);
165 static void stop_this_cpu(void *dummy
)
172 void smp_send_stop(void)
174 smp_call_function(stop_this_cpu
, NULL
, 0);
177 struct thread_info
*current_set
[NR_CPUS
];
179 static void __devinit
smp_store_cpu_info(int id
)
181 per_cpu(pvr
, id
) = mfspr(SPRN_PVR
);
184 static void __init
smp_create_idle(unsigned int cpu
)
186 struct task_struct
*p
;
188 /* create a process for the processor */
191 panic("failed fork for CPU %u: %li", cpu
, PTR_ERR(p
));
193 paca
[cpu
].__current
= p
;
194 paca
[cpu
].kstack
= (unsigned long) task_thread_info(p
)
195 + THREAD_SIZE
- STACK_FRAME_OVERHEAD
;
197 current_set
[cpu
] = task_thread_info(p
);
198 task_thread_info(p
)->cpu
= cpu
;
201 void __init
smp_prepare_cpus(unsigned int max_cpus
)
205 DBG("smp_prepare_cpus\n");
208 * setup_cpu may need to be called on the boot cpu. We havent
209 * spun any cpus up but lets be paranoid.
211 BUG_ON(boot_cpuid
!= smp_processor_id());
214 smp_store_cpu_info(boot_cpuid
);
215 cpu_callin_map
[boot_cpuid
] = 1;
218 max_cpus
= smp_ops
->probe();
222 smp_space_timers(max_cpus
);
224 for_each_possible_cpu(cpu
)
225 if (cpu
!= boot_cpuid
)
226 smp_create_idle(cpu
);
229 void __devinit
smp_prepare_boot_cpu(void)
231 BUG_ON(smp_processor_id() != boot_cpuid
);
233 cpu_set(boot_cpuid
, cpu_online_map
);
234 cpu_set(boot_cpuid
, per_cpu(cpu_sibling_map
, boot_cpuid
));
235 cpu_set(boot_cpuid
, per_cpu(cpu_core_map
, boot_cpuid
));
237 paca
[boot_cpuid
].__current
= current
;
239 current_set
[boot_cpuid
] = task_thread_info(current
);
242 #ifdef CONFIG_HOTPLUG_CPU
243 /* State of each CPU during hotplug phases */
244 DEFINE_PER_CPU(int, cpu_state
) = { 0 };
246 int generic_cpu_disable(void)
248 unsigned int cpu
= smp_processor_id();
250 if (cpu
== boot_cpuid
)
253 cpu_clear(cpu
, cpu_online_map
);
255 vdso_data
->processorCount
--;
256 fixup_irqs(cpu_online_map
);
261 int generic_cpu_enable(unsigned int cpu
)
263 /* Do the normal bootup if we haven't
264 * already bootstrapped. */
265 if (system_state
!= SYSTEM_RUNNING
)
268 /* get the target out of it's holding state */
269 per_cpu(cpu_state
, cpu
) = CPU_UP_PREPARE
;
272 while (!cpu_online(cpu
))
276 fixup_irqs(cpu_online_map
);
277 /* counter the irq disable in fixup_irqs */
283 void generic_cpu_die(unsigned int cpu
)
287 for (i
= 0; i
< 100; i
++) {
289 if (per_cpu(cpu_state
, cpu
) == CPU_DEAD
)
293 printk(KERN_ERR
"CPU%d didn't die...\n", cpu
);
296 void generic_mach_cpu_die(void)
301 cpu
= smp_processor_id();
302 printk(KERN_DEBUG
"CPU%d offline\n", cpu
);
303 __get_cpu_var(cpu_state
) = CPU_DEAD
;
305 while (__get_cpu_var(cpu_state
) != CPU_UP_PREPARE
)
307 cpu_set(cpu
, cpu_online_map
);
312 static int __devinit
cpu_enable(unsigned int cpu
)
314 if (smp_ops
&& smp_ops
->cpu_enable
)
315 return smp_ops
->cpu_enable(cpu
);
320 int __cpuinit
__cpu_up(unsigned int cpu
)
324 secondary_ti
= current_set
[cpu
];
325 if (!cpu_enable(cpu
))
328 if (smp_ops
== NULL
||
329 (smp_ops
->cpu_bootable
&& !smp_ops
->cpu_bootable(cpu
)))
332 /* Make sure callin-map entry is 0 (can be leftover a CPU
335 cpu_callin_map
[cpu
] = 0;
337 /* The information for processor bringup must
338 * be written out to main store before we release
344 DBG("smp: kicking cpu %d\n", cpu
);
345 smp_ops
->kick_cpu(cpu
);
348 * wait to see if the cpu made a callin (is actually up).
349 * use this value that I found through experimentation.
352 if (system_state
< SYSTEM_RUNNING
)
353 for (c
= 50000; c
&& !cpu_callin_map
[cpu
]; c
--)
355 #ifdef CONFIG_HOTPLUG_CPU
358 * CPUs can take much longer to come up in the
359 * hotplug case. Wait five seconds.
361 for (c
= 25; c
&& !cpu_callin_map
[cpu
]; c
--) {
366 if (!cpu_callin_map
[cpu
]) {
367 printk("Processor %u is stuck.\n", cpu
);
371 printk("Processor %u found.\n", cpu
);
373 if (smp_ops
->give_timebase
)
374 smp_ops
->give_timebase();
376 /* Wait until cpu puts itself in the online map */
377 while (!cpu_online(cpu
))
383 /* Return the value of the reg property corresponding to the given
386 int cpu_to_core_id(int cpu
)
388 struct device_node
*np
;
392 np
= of_get_cpu_node(cpu
, NULL
);
396 reg
= of_get_property(np
, "reg", NULL
);
406 /* Must be called when no change can occur to cpu_present_map,
407 * i.e. during cpu online or offline.
409 static struct device_node
*cpu_to_l2cache(int cpu
)
411 struct device_node
*np
;
415 if (!cpu_present(cpu
))
418 np
= of_get_cpu_node(cpu
, NULL
);
422 php
= of_get_property(np
, "l2-cache", NULL
);
428 return of_find_node_by_phandle(ph
);
431 /* Activate a secondary processor. */
432 int __devinit
start_secondary(void *unused
)
434 unsigned int cpu
= smp_processor_id();
435 struct device_node
*l2_cache
;
438 atomic_inc(&init_mm
.mm_count
);
439 current
->active_mm
= &init_mm
;
441 smp_store_cpu_info(cpu
);
442 set_dec(tb_ticks_per_jiffy
);
444 cpu_callin_map
[cpu
] = 1;
446 smp_ops
->setup_cpu(cpu
);
447 if (smp_ops
->take_timebase
)
448 smp_ops
->take_timebase();
450 if (system_state
> SYSTEM_BOOTING
)
453 secondary_cpu_time_init();
456 cpu_set(cpu
, cpu_online_map
);
457 /* Update sibling maps */
458 base
= cpu_first_thread_in_core(cpu
);
459 for (i
= 0; i
< threads_per_core
; i
++) {
460 if (cpu_is_offline(base
+ i
))
462 cpu_set(cpu
, per_cpu(cpu_sibling_map
, base
+ i
));
463 cpu_set(base
+ i
, per_cpu(cpu_sibling_map
, cpu
));
465 /* cpu_core_map should be a superset of
466 * cpu_sibling_map even if we don't have cache
467 * information, so update the former here, too.
469 cpu_set(cpu
, per_cpu(cpu_core_map
, base
+i
));
470 cpu_set(base
+ i
, per_cpu(cpu_core_map
, cpu
));
472 l2_cache
= cpu_to_l2cache(cpu
);
473 for_each_online_cpu(i
) {
474 struct device_node
*np
= cpu_to_l2cache(i
);
477 if (np
== l2_cache
) {
478 cpu_set(cpu
, per_cpu(cpu_core_map
, i
));
479 cpu_set(i
, per_cpu(cpu_core_map
, cpu
));
483 of_node_put(l2_cache
);
492 int setup_profiling_timer(unsigned int multiplier
)
497 void __init
smp_cpus_done(unsigned int max_cpus
)
501 /* We want the setup_cpu() here to be called from CPU 0, but our
502 * init thread may have been "borrowed" by another CPU in the meantime
503 * se we pin us down to CPU 0 for a short while
505 old_mask
= current
->cpus_allowed
;
506 set_cpus_allowed(current
, cpumask_of_cpu(boot_cpuid
));
509 smp_ops
->setup_cpu(boot_cpuid
);
511 set_cpus_allowed(current
, old_mask
);
513 snapshot_timebases();
515 dump_numa_cpu_topology();
518 #ifdef CONFIG_HOTPLUG_CPU
519 int __cpu_disable(void)
521 struct device_node
*l2_cache
;
522 int cpu
= smp_processor_id();
526 if (!smp_ops
->cpu_disable
)
529 err
= smp_ops
->cpu_disable();
533 /* Update sibling maps */
534 base
= cpu_first_thread_in_core(cpu
);
535 for (i
= 0; i
< threads_per_core
; i
++) {
536 cpu_clear(cpu
, per_cpu(cpu_sibling_map
, base
+ i
));
537 cpu_clear(base
+ i
, per_cpu(cpu_sibling_map
, cpu
));
538 cpu_clear(cpu
, per_cpu(cpu_core_map
, base
+i
));
539 cpu_clear(base
+ i
, per_cpu(cpu_core_map
, cpu
));
542 l2_cache
= cpu_to_l2cache(cpu
);
543 for_each_present_cpu(i
) {
544 struct device_node
*np
= cpu_to_l2cache(i
);
547 if (np
== l2_cache
) {
548 cpu_clear(cpu
, per_cpu(cpu_core_map
, i
));
549 cpu_clear(i
, per_cpu(cpu_core_map
, cpu
));
553 of_node_put(l2_cache
);
559 void __cpu_die(unsigned int cpu
)
561 if (smp_ops
->cpu_die
)
562 smp_ops
->cpu_die(cpu
);