2 * linux/arch/arm/kernel/smp.c
4 * Copyright (C) 2002 ARM Limited, All Rights Reserved.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 #include <linux/module.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/cache.h>
17 #include <linux/profile.h>
18 #include <linux/errno.h>
20 #include <linux/err.h>
21 #include <linux/cpu.h>
22 #include <linux/smp.h>
23 #include <linux/seq_file.h>
24 #include <linux/irq.h>
26 #include <asm/atomic.h>
27 #include <asm/cacheflush.h>
29 #include <asm/mmu_context.h>
30 #include <asm/pgtable.h>
31 #include <asm/pgalloc.h>
32 #include <asm/processor.h>
33 #include <asm/tlbflush.h>
34 #include <asm/ptrace.h>
37 * bitmask of present and online CPUs.
38 * The present bitmask indicates that the CPU is physically present.
39 * The online bitmask indicates that the CPU is up and running.
41 cpumask_t cpu_possible_map
;
42 EXPORT_SYMBOL(cpu_possible_map
);
43 cpumask_t cpu_online_map
;
44 EXPORT_SYMBOL(cpu_online_map
);
47 * as from 2.5, kernels no longer have an init_tasks structure
48 * so we need some other way of telling a new secondary core
49 * where to place its SVC stack
51 struct secondary_data secondary_data
;
54 * structures for inter-processor calls
55 * - A collection of single bit ipi messages.
59 unsigned long ipi_count
;
63 static DEFINE_PER_CPU(struct ipi_data
, ipi_data
) = {
64 .lock
= SPIN_LOCK_UNLOCKED
,
75 int __cpuinit
__cpu_up(unsigned int cpu
)
77 struct cpuinfo_arm
*ci
= &per_cpu(cpu_data
, cpu
);
78 struct task_struct
*idle
= ci
->idle
;
84 * Spawn a new process manually, if not already done.
85 * Grab a pointer to its task struct so we can mess with it
88 idle
= fork_idle(cpu
);
90 printk(KERN_ERR
"CPU%u: fork() failed\n", cpu
);
97 * Allocate initial page tables to allow the new CPU to
98 * enable the MMU safely. This essentially means a set
99 * of our "standard" page tables, with the addition of
100 * a 1:1 mapping for the physical address of the kernel.
102 pgd
= pgd_alloc(&init_mm
);
103 pmd
= pmd_offset(pgd
+ pgd_index(PHYS_OFFSET
), PHYS_OFFSET
);
104 *pmd
= __pmd((PHYS_OFFSET
& PGDIR_MASK
) |
105 PMD_TYPE_SECT
| PMD_SECT_AP_WRITE
);
108 * We need to tell the secondary core where to find
109 * its stack and the page tables.
111 secondary_data
.stack
= task_stack_page(idle
) + THREAD_START_SP
;
112 secondary_data
.pgdir
= virt_to_phys(pgd
);
116 * Now bring the CPU into our world.
118 ret
= boot_secondary(cpu
, idle
);
120 unsigned long timeout
;
123 * CPU was successfully started, wait for it
124 * to come online or time out.
126 timeout
= jiffies
+ HZ
;
127 while (time_before(jiffies
, timeout
)) {
135 if (!cpu_online(cpu
))
139 secondary_data
.stack
= NULL
;
140 secondary_data
.pgdir
= 0;
143 pgd_free(&init_mm
, pgd
);
146 printk(KERN_CRIT
"CPU%u: processor failed to boot\n", cpu
);
149 * FIXME: We need to clean up the new idle thread. --rmk
156 #ifdef CONFIG_HOTPLUG_CPU
158 * __cpu_disable runs on the processor to be shutdown.
160 int __cpuexit
__cpu_disable(void)
162 unsigned int cpu
= smp_processor_id();
163 struct task_struct
*p
;
166 ret
= mach_cpu_disable(cpu
);
171 * Take this CPU offline. Once we clear this, we can't return,
172 * and we must not schedule until we're ready to give up the cpu.
174 cpu_clear(cpu
, cpu_online_map
);
177 * OK - migrate IRQs away from this CPU
182 * Stop the local timer for this CPU.
184 local_timer_stop(cpu
);
187 * Flush user cache and TLB mappings, and then remove this CPU
188 * from the vm mask set of all processes.
191 local_flush_tlb_all();
193 read_lock(&tasklist_lock
);
194 for_each_process(p
) {
196 cpu_clear(cpu
, p
->mm
->cpu_vm_mask
);
198 read_unlock(&tasklist_lock
);
204 * called on the thread which is asking for a CPU to be shutdown -
205 * waits until shutdown has completed, or it is timed out.
207 void __cpuexit
__cpu_die(unsigned int cpu
)
209 if (!platform_cpu_kill(cpu
))
210 printk("CPU%u: unable to kill\n", cpu
);
214 * Called from the idle thread for the CPU which has been shutdown.
216 * Note that we disable IRQs here, but do not re-enable them
217 * before returning to the caller. This is also the behaviour
218 * of the other hotplug-cpu capable cores, so presumably coming
219 * out of idle fixes this.
221 void __cpuexit
cpu_die(void)
223 unsigned int cpu
= smp_processor_id();
229 * actual CPU shutdown procedure is at least platform (if not
232 platform_cpu_die(cpu
);
235 * Do not return to the idle loop - jump back to the secondary
236 * cpu initialisation. There's some initialisation which needs
237 * to be repeated to undo the effects of taking the CPU offline.
239 __asm__("mov sp, %0\n"
240 " b secondary_start_kernel"
242 : "r" (task_stack_page(current
) + THREAD_SIZE
- 8));
244 #endif /* CONFIG_HOTPLUG_CPU */
247 * This is the secondary CPU boot entry. We're using this CPUs
248 * idle thread stack, but a set of temporary page tables.
250 asmlinkage
void __cpuinit
secondary_start_kernel(void)
252 struct mm_struct
*mm
= &init_mm
;
253 unsigned int cpu
= smp_processor_id();
255 printk("CPU%u: Booted secondary processor\n", cpu
);
258 * All kernel threads share the same mm context; grab a
259 * reference and switch to it.
261 atomic_inc(&mm
->mm_users
);
262 atomic_inc(&mm
->mm_count
);
263 current
->active_mm
= mm
;
264 cpu_set(cpu
, mm
->cpu_vm_mask
);
265 cpu_switch_mm(mm
->pgd
, mm
);
266 enter_lazy_tlb(mm
, current
);
267 local_flush_tlb_all();
273 * Give the platform a chance to do its own initialisation.
275 platform_secondary_init(cpu
);
278 * Enable local interrupts.
284 * Setup local timer for this CPU.
286 local_timer_setup(cpu
);
290 smp_store_cpu_info(cpu
);
293 * OK, now it's safe to let the boot CPU continue
295 cpu_set(cpu
, cpu_online_map
);
298 * OK, it's off to the idle thread for us
304 * Called by both boot and secondaries to move global data into
305 * per-processor storage.
307 void __cpuinit
smp_store_cpu_info(unsigned int cpuid
)
309 struct cpuinfo_arm
*cpu_info
= &per_cpu(cpu_data
, cpuid
);
311 cpu_info
->loops_per_jiffy
= loops_per_jiffy
;
314 void __init
smp_cpus_done(unsigned int max_cpus
)
317 unsigned long bogosum
= 0;
319 for_each_online_cpu(cpu
)
320 bogosum
+= per_cpu(cpu_data
, cpu
).loops_per_jiffy
;
322 printk(KERN_INFO
"SMP: Total of %d processors activated "
323 "(%lu.%02lu BogoMIPS).\n",
325 bogosum
/ (500000/HZ
),
326 (bogosum
/ (5000/HZ
)) % 100);
329 void __init
smp_prepare_boot_cpu(void)
331 unsigned int cpu
= smp_processor_id();
333 per_cpu(cpu_data
, cpu
).idle
= current
;
336 static void send_ipi_message(cpumask_t callmap
, enum ipi_msg_type msg
)
341 local_irq_save(flags
);
343 for_each_cpu_mask(cpu
, callmap
) {
344 struct ipi_data
*ipi
= &per_cpu(ipi_data
, cpu
);
346 spin_lock(&ipi
->lock
);
347 ipi
->bits
|= 1 << msg
;
348 spin_unlock(&ipi
->lock
);
352 * Call the platform specific cross-CPU call function.
354 smp_cross_call(callmap
);
356 local_irq_restore(flags
);
359 void arch_send_call_function_ipi(cpumask_t mask
)
361 send_ipi_message(mask
, IPI_CALL_FUNC
);
364 void arch_send_call_function_single_ipi(int cpu
)
366 send_ipi_message(cpumask_of_cpu(cpu
), IPI_CALL_FUNC_SINGLE
);
369 void show_ipi_list(struct seq_file
*p
)
375 for_each_present_cpu(cpu
)
376 seq_printf(p
, " %10lu", per_cpu(ipi_data
, cpu
).ipi_count
);
381 void show_local_irqs(struct seq_file
*p
)
385 seq_printf(p
, "LOC: ");
387 for_each_present_cpu(cpu
)
388 seq_printf(p
, "%10u ", irq_stat
[cpu
].local_timer_irqs
);
393 static void ipi_timer(void)
396 local_timer_interrupt();
400 #ifdef CONFIG_LOCAL_TIMERS
401 asmlinkage
void __exception
do_local_timer(struct pt_regs
*regs
)
403 struct pt_regs
*old_regs
= set_irq_regs(regs
);
404 int cpu
= smp_processor_id();
406 if (local_timer_ack()) {
407 irq_stat
[cpu
].local_timer_irqs
++;
411 set_irq_regs(old_regs
);
415 static DEFINE_SPINLOCK(stop_lock
);
418 * ipi_cpu_stop - handle IPI from smp_send_stop()
420 static void ipi_cpu_stop(unsigned int cpu
)
422 spin_lock(&stop_lock
);
423 printk(KERN_CRIT
"CPU%u: stopping\n", cpu
);
425 spin_unlock(&stop_lock
);
427 cpu_clear(cpu
, cpu_online_map
);
437 * Main handler for inter-processor interrupts
439 * For ARM, the ipimask now only identifies a single
440 * category of IPI (Bit 1 IPIs have been replaced by a
441 * different mechanism):
443 * Bit 0 - Inter-processor function call
445 asmlinkage
void __exception
do_IPI(struct pt_regs
*regs
)
447 unsigned int cpu
= smp_processor_id();
448 struct ipi_data
*ipi
= &per_cpu(ipi_data
, cpu
);
449 struct pt_regs
*old_regs
= set_irq_regs(regs
);
456 spin_lock(&ipi
->lock
);
459 spin_unlock(&ipi
->lock
);
467 nextmsg
= msgs
& -msgs
;
469 nextmsg
= ffz(~nextmsg
);
478 * nothing more to do - eveything is
479 * done on the interrupt return path
484 generic_smp_call_function_interrupt();
487 case IPI_CALL_FUNC_SINGLE
:
488 generic_smp_call_function_single_interrupt();
496 printk(KERN_CRIT
"CPU%u: Unknown IPI message 0x%x\n",
503 set_irq_regs(old_regs
);
506 void smp_send_reschedule(int cpu
)
508 send_ipi_message(cpumask_of_cpu(cpu
), IPI_RESCHEDULE
);
511 void smp_send_timer(void)
513 cpumask_t mask
= cpu_online_map
;
514 cpu_clear(smp_processor_id(), mask
);
515 send_ipi_message(mask
, IPI_TIMER
);
518 void smp_timer_broadcast(cpumask_t mask
)
520 send_ipi_message(mask
, IPI_TIMER
);
523 void smp_send_stop(void)
525 cpumask_t mask
= cpu_online_map
;
526 cpu_clear(smp_processor_id(), mask
);
527 send_ipi_message(mask
, IPI_CPU_STOP
);
533 int setup_profiling_timer(unsigned int multiplier
)
539 on_each_cpu_mask(void (*func
)(void *), void *info
, int wait
, cpumask_t mask
)
545 ret
= smp_call_function_mask(mask
, func
, info
, wait
);
546 if (cpu_isset(smp_processor_id(), mask
))
554 /**********************************************************************/
560 struct vm_area_struct
*ta_vma
;
561 unsigned long ta_start
;
562 unsigned long ta_end
;
565 static inline void ipi_flush_tlb_all(void *ignored
)
567 local_flush_tlb_all();
570 static inline void ipi_flush_tlb_mm(void *arg
)
572 struct mm_struct
*mm
= (struct mm_struct
*)arg
;
574 local_flush_tlb_mm(mm
);
577 static inline void ipi_flush_tlb_page(void *arg
)
579 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
581 local_flush_tlb_page(ta
->ta_vma
, ta
->ta_start
);
584 static inline void ipi_flush_tlb_kernel_page(void *arg
)
586 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
588 local_flush_tlb_kernel_page(ta
->ta_start
);
591 static inline void ipi_flush_tlb_range(void *arg
)
593 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
595 local_flush_tlb_range(ta
->ta_vma
, ta
->ta_start
, ta
->ta_end
);
598 static inline void ipi_flush_tlb_kernel_range(void *arg
)
600 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
602 local_flush_tlb_kernel_range(ta
->ta_start
, ta
->ta_end
);
605 void flush_tlb_all(void)
607 on_each_cpu(ipi_flush_tlb_all
, NULL
, 1);
610 void flush_tlb_mm(struct mm_struct
*mm
)
612 cpumask_t mask
= mm
->cpu_vm_mask
;
614 on_each_cpu_mask(ipi_flush_tlb_mm
, mm
, 1, mask
);
617 void flush_tlb_page(struct vm_area_struct
*vma
, unsigned long uaddr
)
619 cpumask_t mask
= vma
->vm_mm
->cpu_vm_mask
;
625 on_each_cpu_mask(ipi_flush_tlb_page
, &ta
, 1, mask
);
628 void flush_tlb_kernel_page(unsigned long kaddr
)
634 on_each_cpu(ipi_flush_tlb_kernel_page
, &ta
, 1);
637 void flush_tlb_range(struct vm_area_struct
*vma
,
638 unsigned long start
, unsigned long end
)
640 cpumask_t mask
= vma
->vm_mm
->cpu_vm_mask
;
647 on_each_cpu_mask(ipi_flush_tlb_range
, &ta
, 1, mask
);
650 void flush_tlb_kernel_range(unsigned long start
, unsigned long end
)
657 on_each_cpu(ipi_flush_tlb_kernel_range
, &ta
, 1);