2 * linux/arch/arm/kernel/smp.c
4 * Copyright (C) 2002 ARM Limited, All Rights Reserved.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 #include <linux/config.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/cache.h>
17 #include <linux/profile.h>
18 #include <linux/errno.h>
20 #include <linux/cpu.h>
21 #include <linux/smp.h>
22 #include <linux/seq_file.h>
24 #include <asm/atomic.h>
25 #include <asm/cacheflush.h>
27 #include <asm/mmu_context.h>
28 #include <asm/pgtable.h>
29 #include <asm/pgalloc.h>
30 #include <asm/processor.h>
31 #include <asm/tlbflush.h>
32 #include <asm/ptrace.h>
35 * bitmask of present and online CPUs.
36 * The present bitmask indicates that the CPU is physically present.
37 * The online bitmask indicates that the CPU is up and running.
39 cpumask_t cpu_possible_map
;
40 cpumask_t cpu_online_map
;
43 * as from 2.5, kernels no longer have an init_tasks structure
44 * so we need some other way of telling a new secondary core
45 * where to place its SVC stack
47 struct secondary_data secondary_data
;
50 * structures for inter-processor calls
51 * - A collection of single bit ipi messages.
55 unsigned long ipi_count
;
59 static DEFINE_PER_CPU(struct ipi_data
, ipi_data
) = {
60 .lock
= SPIN_LOCK_UNLOCKED
,
70 struct smp_call_struct
{
71 void (*func
)(void *info
);
78 static struct smp_call_struct
* volatile smp_call_function_data
;
79 static DEFINE_SPINLOCK(smp_call_function_lock
);
81 int __cpuinit
__cpu_up(unsigned int cpu
)
83 struct cpuinfo_arm
*ci
= &per_cpu(cpu_data
, cpu
);
84 struct task_struct
*idle
= ci
->idle
;
90 * Spawn a new process manually, if not already done.
91 * Grab a pointer to its task struct so we can mess with it
94 idle
= fork_idle(cpu
);
96 printk(KERN_ERR
"CPU%u: fork() failed\n", cpu
);
103 * Allocate initial page tables to allow the new CPU to
104 * enable the MMU safely. This essentially means a set
105 * of our "standard" page tables, with the addition of
106 * a 1:1 mapping for the physical address of the kernel.
108 pgd
= pgd_alloc(&init_mm
);
109 pmd
= pmd_offset(pgd
, PHYS_OFFSET
);
110 *pmd
= __pmd((PHYS_OFFSET
& PGDIR_MASK
) |
111 PMD_TYPE_SECT
| PMD_SECT_AP_WRITE
);
114 * We need to tell the secondary core where to find
115 * its stack and the page tables.
117 secondary_data
.stack
= task_stack_page(idle
) + THREAD_START_SP
;
118 secondary_data
.pgdir
= virt_to_phys(pgd
);
122 * Now bring the CPU into our world.
124 ret
= boot_secondary(cpu
, idle
);
126 unsigned long timeout
;
129 * CPU was successfully started, wait for it
130 * to come online or time out.
132 timeout
= jiffies
+ HZ
;
133 while (time_before(jiffies
, timeout
)) {
141 if (!cpu_online(cpu
))
145 secondary_data
.stack
= NULL
;
146 secondary_data
.pgdir
= 0;
148 *pmd_offset(pgd
, PHYS_OFFSET
) = __pmd(0);
152 printk(KERN_CRIT
"CPU%u: processor failed to boot\n", cpu
);
155 * FIXME: We need to clean up the new idle thread. --rmk
162 #ifdef CONFIG_HOTPLUG_CPU
164 * __cpu_disable runs on the processor to be shutdown.
166 int __cpuexit
__cpu_disable(void)
168 unsigned int cpu
= smp_processor_id();
169 struct task_struct
*p
;
172 ret
= mach_cpu_disable(cpu
);
177 * Take this CPU offline. Once we clear this, we can't return,
178 * and we must not schedule until we're ready to give up the cpu.
180 cpu_clear(cpu
, cpu_online_map
);
183 * OK - migrate IRQs away from this CPU
188 * Stop the local timer for this CPU.
190 local_timer_stop(cpu
);
193 * Flush user cache and TLB mappings, and then remove this CPU
194 * from the vm mask set of all processes.
197 local_flush_tlb_all();
199 read_lock(&tasklist_lock
);
200 for_each_process(p
) {
202 cpu_clear(cpu
, p
->mm
->cpu_vm_mask
);
204 read_unlock(&tasklist_lock
);
210 * called on the thread which is asking for a CPU to be shutdown -
211 * waits until shutdown has completed, or it is timed out.
213 void __cpuexit
__cpu_die(unsigned int cpu
)
215 if (!platform_cpu_kill(cpu
))
216 printk("CPU%u: unable to kill\n", cpu
);
220 * Called from the idle thread for the CPU which has been shutdown.
222 * Note that we disable IRQs here, but do not re-enable them
223 * before returning to the caller. This is also the behaviour
224 * of the other hotplug-cpu capable cores, so presumably coming
225 * out of idle fixes this.
227 void __cpuexit
cpu_die(void)
229 unsigned int cpu
= smp_processor_id();
235 * actual CPU shutdown procedure is at least platform (if not
238 platform_cpu_die(cpu
);
241 * Do not return to the idle loop - jump back to the secondary
242 * cpu initialisation. There's some initialisation which needs
243 * to be repeated to undo the effects of taking the CPU offline.
245 __asm__("mov sp, %0\n"
246 " b secondary_start_kernel"
248 : "r" (task_stack_page(current
) + THREAD_SIZE
- 8));
250 #endif /* CONFIG_HOTPLUG_CPU */
253 * This is the secondary CPU boot entry. We're using this CPUs
254 * idle thread stack, but a set of temporary page tables.
256 asmlinkage
void __cpuinit
secondary_start_kernel(void)
258 struct mm_struct
*mm
= &init_mm
;
259 unsigned int cpu
= smp_processor_id();
261 printk("CPU%u: Booted secondary processor\n", cpu
);
264 * All kernel threads share the same mm context; grab a
265 * reference and switch to it.
267 atomic_inc(&mm
->mm_users
);
268 atomic_inc(&mm
->mm_count
);
269 current
->active_mm
= mm
;
270 cpu_set(cpu
, mm
->cpu_vm_mask
);
271 cpu_switch_mm(mm
->pgd
, mm
);
272 enter_lazy_tlb(mm
, current
);
273 local_flush_tlb_all();
279 * Give the platform a chance to do its own initialisation.
281 platform_secondary_init(cpu
);
284 * Enable local interrupts.
291 smp_store_cpu_info(cpu
);
294 * OK, now it's safe to let the boot CPU continue
296 cpu_set(cpu
, cpu_online_map
);
299 * Setup local timer for this CPU.
301 local_timer_setup(cpu
);
304 * OK, it's off to the idle thread for us
310 * Called by both boot and secondaries to move global data into
311 * per-processor storage.
313 void __cpuinit
smp_store_cpu_info(unsigned int cpuid
)
315 struct cpuinfo_arm
*cpu_info
= &per_cpu(cpu_data
, cpuid
);
317 cpu_info
->loops_per_jiffy
= loops_per_jiffy
;
320 void __init
smp_cpus_done(unsigned int max_cpus
)
323 unsigned long bogosum
= 0;
325 for_each_online_cpu(cpu
)
326 bogosum
+= per_cpu(cpu_data
, cpu
).loops_per_jiffy
;
328 printk(KERN_INFO
"SMP: Total of %d processors activated "
329 "(%lu.%02lu BogoMIPS).\n",
331 bogosum
/ (500000/HZ
),
332 (bogosum
/ (5000/HZ
)) % 100);
335 void __init
smp_prepare_boot_cpu(void)
337 unsigned int cpu
= smp_processor_id();
339 per_cpu(cpu_data
, cpu
).idle
= current
;
341 cpu_set(cpu
, cpu_possible_map
);
342 cpu_set(cpu
, cpu_present_map
);
343 cpu_set(cpu
, cpu_online_map
);
346 static void send_ipi_message(cpumask_t callmap
, enum ipi_msg_type msg
)
351 local_irq_save(flags
);
353 for_each_cpu_mask(cpu
, callmap
) {
354 struct ipi_data
*ipi
= &per_cpu(ipi_data
, cpu
);
356 spin_lock(&ipi
->lock
);
357 ipi
->bits
|= 1 << msg
;
358 spin_unlock(&ipi
->lock
);
362 * Call the platform specific cross-CPU call function.
364 smp_cross_call(callmap
);
366 local_irq_restore(flags
);
370 * You must not call this function with disabled interrupts, from a
371 * hardware interrupt handler, nor from a bottom half handler.
373 static int smp_call_function_on_cpu(void (*func
)(void *info
), void *info
,
374 int retry
, int wait
, cpumask_t callmap
)
376 struct smp_call_struct data
;
377 unsigned long timeout
;
384 cpu_clear(smp_processor_id(), callmap
);
385 if (cpus_empty(callmap
))
388 data
.pending
= callmap
;
390 data
.unfinished
= callmap
;
393 * try to get the mutex on smp_call_function_data
395 spin_lock(&smp_call_function_lock
);
396 smp_call_function_data
= &data
;
398 send_ipi_message(callmap
, IPI_CALL_FUNC
);
400 timeout
= jiffies
+ HZ
;
401 while (!cpus_empty(data
.pending
) && time_before(jiffies
, timeout
))
407 if (!cpus_empty(data
.pending
)) {
409 * this may be causing our panic - report it
412 "CPU%u: smp_call_function timeout for %p(%p)\n"
413 " callmap %lx pending %lx, %swait\n",
414 smp_processor_id(), func
, info
, *cpus_addr(callmap
),
415 *cpus_addr(data
.pending
), wait
? "" : "no ");
420 timeout
= jiffies
+ (5 * HZ
);
421 while (!cpus_empty(data
.pending
) && time_before(jiffies
, timeout
))
424 if (cpus_empty(data
.pending
))
425 printk(KERN_CRIT
" RESOLVED\n");
427 printk(KERN_CRIT
" STILL STUCK\n");
431 * whatever happened, we're done with the data, so release it
433 smp_call_function_data
= NULL
;
434 spin_unlock(&smp_call_function_lock
);
436 if (!cpus_empty(data
.pending
)) {
442 while (!cpus_empty(data
.unfinished
))
449 int smp_call_function(void (*func
)(void *info
), void *info
, int retry
,
452 return smp_call_function_on_cpu(func
, info
, retry
, wait
,
456 void show_ipi_list(struct seq_file
*p
)
462 for_each_present_cpu(cpu
)
463 seq_printf(p
, " %10lu", per_cpu(ipi_data
, cpu
).ipi_count
);
468 void show_local_irqs(struct seq_file
*p
)
472 seq_printf(p
, "LOC: ");
474 for_each_present_cpu(cpu
)
475 seq_printf(p
, "%10u ", irq_stat
[cpu
].local_timer_irqs
);
480 static void ipi_timer(struct pt_regs
*regs
)
482 int user
= user_mode(regs
);
485 profile_tick(CPU_PROFILING
, regs
);
486 update_process_times(user
);
490 #ifdef CONFIG_LOCAL_TIMERS
491 asmlinkage
void do_local_timer(struct pt_regs
*regs
)
493 int cpu
= smp_processor_id();
495 if (local_timer_ack()) {
496 irq_stat
[cpu
].local_timer_irqs
++;
503 * ipi_call_function - handle IPI from smp_call_function()
505 * Note that we copy data out of the cross-call structure and then
506 * let the caller know that we're here and have done with their data
508 static void ipi_call_function(unsigned int cpu
)
510 struct smp_call_struct
*data
= smp_call_function_data
;
511 void (*func
)(void *info
) = data
->func
;
512 void *info
= data
->info
;
513 int wait
= data
->wait
;
515 cpu_clear(cpu
, data
->pending
);
520 cpu_clear(cpu
, data
->unfinished
);
523 static DEFINE_SPINLOCK(stop_lock
);
526 * ipi_cpu_stop - handle IPI from smp_send_stop()
528 static void ipi_cpu_stop(unsigned int cpu
)
530 spin_lock(&stop_lock
);
531 printk(KERN_CRIT
"CPU%u: stopping\n", cpu
);
533 spin_unlock(&stop_lock
);
535 cpu_clear(cpu
, cpu_online_map
);
545 * Main handler for inter-processor interrupts
547 * For ARM, the ipimask now only identifies a single
548 * category of IPI (Bit 1 IPIs have been replaced by a
549 * different mechanism):
551 * Bit 0 - Inter-processor function call
553 asmlinkage
void do_IPI(struct pt_regs
*regs
)
555 unsigned int cpu
= smp_processor_id();
556 struct ipi_data
*ipi
= &per_cpu(ipi_data
, cpu
);
563 spin_lock(&ipi
->lock
);
566 spin_unlock(&ipi
->lock
);
574 nextmsg
= msgs
& -msgs
;
576 nextmsg
= ffz(~nextmsg
);
585 * nothing more to do - eveything is
586 * done on the interrupt return path
591 ipi_call_function(cpu
);
599 printk(KERN_CRIT
"CPU%u: Unknown IPI message 0x%x\n",
607 void smp_send_reschedule(int cpu
)
609 send_ipi_message(cpumask_of_cpu(cpu
), IPI_RESCHEDULE
);
612 void smp_send_timer(void)
614 cpumask_t mask
= cpu_online_map
;
615 cpu_clear(smp_processor_id(), mask
);
616 send_ipi_message(mask
, IPI_TIMER
);
619 void smp_send_stop(void)
621 cpumask_t mask
= cpu_online_map
;
622 cpu_clear(smp_processor_id(), mask
);
623 send_ipi_message(mask
, IPI_CPU_STOP
);
629 int __init
setup_profiling_timer(unsigned int multiplier
)
635 on_each_cpu_mask(void (*func
)(void *), void *info
, int retry
, int wait
,
642 ret
= smp_call_function_on_cpu(func
, info
, retry
, wait
, mask
);
643 if (cpu_isset(smp_processor_id(), mask
))
651 /**********************************************************************/
657 struct vm_area_struct
*ta_vma
;
658 unsigned long ta_start
;
659 unsigned long ta_end
;
662 static inline void ipi_flush_tlb_all(void *ignored
)
664 local_flush_tlb_all();
667 static inline void ipi_flush_tlb_mm(void *arg
)
669 struct mm_struct
*mm
= (struct mm_struct
*)arg
;
671 local_flush_tlb_mm(mm
);
674 static inline void ipi_flush_tlb_page(void *arg
)
676 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
678 local_flush_tlb_page(ta
->ta_vma
, ta
->ta_start
);
681 static inline void ipi_flush_tlb_kernel_page(void *arg
)
683 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
685 local_flush_tlb_kernel_page(ta
->ta_start
);
688 static inline void ipi_flush_tlb_range(void *arg
)
690 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
692 local_flush_tlb_range(ta
->ta_vma
, ta
->ta_start
, ta
->ta_end
);
695 static inline void ipi_flush_tlb_kernel_range(void *arg
)
697 struct tlb_args
*ta
= (struct tlb_args
*)arg
;
699 local_flush_tlb_kernel_range(ta
->ta_start
, ta
->ta_end
);
702 void flush_tlb_all(void)
704 on_each_cpu(ipi_flush_tlb_all
, NULL
, 1, 1);
707 void flush_tlb_mm(struct mm_struct
*mm
)
709 cpumask_t mask
= mm
->cpu_vm_mask
;
711 on_each_cpu_mask(ipi_flush_tlb_mm
, mm
, 1, 1, mask
);
714 void flush_tlb_page(struct vm_area_struct
*vma
, unsigned long uaddr
)
716 cpumask_t mask
= vma
->vm_mm
->cpu_vm_mask
;
722 on_each_cpu_mask(ipi_flush_tlb_page
, &ta
, 1, 1, mask
);
725 void flush_tlb_kernel_page(unsigned long kaddr
)
731 on_each_cpu(ipi_flush_tlb_kernel_page
, &ta
, 1, 1);
734 void flush_tlb_range(struct vm_area_struct
*vma
,
735 unsigned long start
, unsigned long end
)
737 cpumask_t mask
= vma
->vm_mm
->cpu_vm_mask
;
744 on_each_cpu_mask(ipi_flush_tlb_range
, &ta
, 1, 1, mask
);
747 void flush_tlb_kernel_range(unsigned long start
, unsigned long end
)
754 on_each_cpu(ipi_flush_tlb_kernel_range
, &ta
, 1, 1);