2 * linux/arch/m32r/kernel/smp.c
4 * M32R SMP support routines.
6 * Copyright (c) 2001, 2002 Hitoshi Yamamoto
8 * Taken from i386 version.
9 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
10 * (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
12 * This code is released under the GNU General Public License version 2 or
18 #include <linux/irq.h>
19 #include <linux/interrupt.h>
20 #include <linux/sched.h>
21 #include <linux/spinlock.h>
23 #include <linux/smp.h>
24 #include <linux/profile.h>
25 #include <linux/cpu.h>
27 #include <asm/cacheflush.h>
28 #include <asm/pgalloc.h>
29 #include <asm/atomic.h>
31 #include <asm/mmu_context.h>
34 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
35 /* Data structures and variables */
36 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
39 * For flush_cache_all()
41 static DEFINE_SPINLOCK(flushcache_lock
);
42 static volatile unsigned long flushcache_cpumask
= 0;
45 * For flush_tlb_others()
47 static volatile cpumask_t flush_cpumask
;
48 static struct mm_struct
*flush_mm
;
49 static struct vm_area_struct
*flush_vma
;
50 static volatile unsigned long flush_va
;
51 static DEFINE_SPINLOCK(tlbstate_lock
);
52 #define FLUSH_ALL 0xffffffff
54 DECLARE_PER_CPU(int, prof_multiplier
);
55 DECLARE_PER_CPU(int, prof_old_multiplier
);
56 DECLARE_PER_CPU(int, prof_counter
);
58 extern spinlock_t ipi_lock
[];
60 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
61 /* Function Prototypes */
62 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
64 void smp_send_reschedule(int);
65 void smp_reschedule_interrupt(void);
67 void smp_flush_cache_all(void);
68 void smp_flush_cache_all_interrupt(void);
70 void smp_flush_tlb_all(void);
71 static void flush_tlb_all_ipi(void *);
73 void smp_flush_tlb_mm(struct mm_struct
*);
74 void smp_flush_tlb_range(struct vm_area_struct
*, unsigned long, \
76 void smp_flush_tlb_page(struct vm_area_struct
*, unsigned long);
77 static void flush_tlb_others(cpumask_t
, struct mm_struct
*,
78 struct vm_area_struct
*, unsigned long);
79 void smp_invalidate_interrupt(void);
81 void smp_send_stop(void);
82 static void stop_this_cpu(void *);
84 void smp_send_timer(void);
85 void smp_ipi_timer_interrupt(struct pt_regs
*);
86 void smp_local_timer_interrupt(void);
88 static void send_IPI_allbutself(int, int);
89 static void send_IPI_mask(const struct cpumask
*, int, int);
90 unsigned long send_IPI_mask_phys(cpumask_t
, int, int);
92 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
93 /* Rescheduling request Routines */
94 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
96 /*==========================================================================*
97 * Name: smp_send_reschedule
99 * Description: This routine requests other CPU to execute rescheduling.
100 * 1.Send 'RESCHEDULE_IPI' to other CPU.
101 * Request other CPU to execute 'smp_reschedule_interrupt()'.
103 * Born on Date: 2002.02.05
105 * Arguments: cpu_id - Target CPU ID
107 * Returns: void (cannot fail)
110 * Date Who Description
111 * ---------- --- --------------------------------------------------------
113 *==========================================================================*/
114 void smp_send_reschedule(int cpu_id
)
116 WARN_ON(cpu_is_offline(cpu_id
));
117 send_IPI_mask(cpumask_of(cpu_id
), RESCHEDULE_IPI
, 1);
120 /*==========================================================================*
121 * Name: smp_reschedule_interrupt
123 * Description: This routine executes on CPU which received
125 * Rescheduling is processed at the exit of interrupt
128 * Born on Date: 2002.02.05
132 * Returns: void (cannot fail)
135 * Date Who Description
136 * ---------- --- --------------------------------------------------------
138 *==========================================================================*/
139 void smp_reschedule_interrupt(void)
144 /*==========================================================================*
145 * Name: smp_flush_cache_all
147 * Description: This routine sends a 'INVALIDATE_CACHE_IPI' to all other
148 * CPUs in the system.
150 * Born on Date: 2003-05-28
154 * Returns: void (cannot fail)
157 * Date Who Description
158 * ---------- --- --------------------------------------------------------
160 *==========================================================================*/
161 void smp_flush_cache_all(void)
167 cpumask
= cpu_online_map
;
168 cpu_clear(smp_processor_id(), cpumask
);
169 spin_lock(&flushcache_lock
);
170 mask
=cpus_addr(cpumask
);
171 atomic_set_mask(*mask
, (atomic_t
*)&flushcache_cpumask
);
172 send_IPI_mask(&cpumask
, INVALIDATE_CACHE_IPI
, 0);
173 _flush_cache_copyback_all();
174 while (flushcache_cpumask
)
176 spin_unlock(&flushcache_lock
);
180 void smp_flush_cache_all_interrupt(void)
182 _flush_cache_copyback_all();
183 clear_bit(smp_processor_id(), &flushcache_cpumask
);
186 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
187 /* TLB flush request Routines */
188 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
190 /*==========================================================================*
191 * Name: smp_flush_tlb_all
193 * Description: This routine flushes all processes TLBs.
194 * 1.Request other CPU to execute 'flush_tlb_all_ipi()'.
195 * 2.Execute 'do_flush_tlb_all_local()'.
197 * Born on Date: 2002.02.05
201 * Returns: void (cannot fail)
204 * Date Who Description
205 * ---------- --- --------------------------------------------------------
207 *==========================================================================*/
208 void smp_flush_tlb_all(void)
213 local_irq_save(flags
);
215 local_irq_restore(flags
);
216 smp_call_function(flush_tlb_all_ipi
, NULL
, 1);
220 /*==========================================================================*
221 * Name: flush_tlb_all_ipi
223 * Description: This routine flushes all local TLBs.
224 * 1.Execute 'do_flush_tlb_all_local()'.
226 * Born on Date: 2002.02.05
228 * Arguments: *info - not used
230 * Returns: void (cannot fail)
233 * Date Who Description
234 * ---------- --- --------------------------------------------------------
236 *==========================================================================*/
237 static void flush_tlb_all_ipi(void *info
)
242 /*==========================================================================*
243 * Name: smp_flush_tlb_mm
245 * Description: This routine flushes the specified mm context TLB's.
247 * Born on Date: 2002.02.05
249 * Arguments: *mm - a pointer to the mm struct for flush TLB
251 * Returns: void (cannot fail)
254 * Date Who Description
255 * ---------- --- --------------------------------------------------------
257 *==========================================================================*/
258 void smp_flush_tlb_mm(struct mm_struct
*mm
)
266 cpu_id
= smp_processor_id();
267 mmc
= &mm
->context
[cpu_id
];
268 cpu_mask
= *mm_cpumask(mm
);
269 cpu_clear(cpu_id
, cpu_mask
);
271 if (*mmc
!= NO_CONTEXT
) {
272 local_irq_save(flags
);
274 if (mm
== current
->mm
)
275 activate_context(mm
);
277 cpumask_clear_cpu(cpu_id
, mm_cpumask(mm
));
278 local_irq_restore(flags
);
280 if (!cpus_empty(cpu_mask
))
281 flush_tlb_others(cpu_mask
, mm
, NULL
, FLUSH_ALL
);
286 /*==========================================================================*
287 * Name: smp_flush_tlb_range
289 * Description: This routine flushes a range of pages.
291 * Born on Date: 2002.02.05
293 * Arguments: *mm - a pointer to the mm struct for flush TLB
297 * Returns: void (cannot fail)
300 * Date Who Description
301 * ---------- --- --------------------------------------------------------
303 *==========================================================================*/
304 void smp_flush_tlb_range(struct vm_area_struct
*vma
, unsigned long start
,
307 smp_flush_tlb_mm(vma
->vm_mm
);
310 /*==========================================================================*
311 * Name: smp_flush_tlb_page
313 * Description: This routine flushes one page.
315 * Born on Date: 2002.02.05
317 * Arguments: *vma - a pointer to the vma struct include va
318 * va - virtual address for flush TLB
320 * Returns: void (cannot fail)
323 * Date Who Description
324 * ---------- --- --------------------------------------------------------
326 *==========================================================================*/
327 void smp_flush_tlb_page(struct vm_area_struct
*vma
, unsigned long va
)
329 struct mm_struct
*mm
= vma
->vm_mm
;
336 cpu_id
= smp_processor_id();
337 mmc
= &mm
->context
[cpu_id
];
338 cpu_mask
= *mm_cpumask(mm
);
339 cpu_clear(cpu_id
, cpu_mask
);
346 if (*mmc
!= NO_CONTEXT
) {
347 local_irq_save(flags
);
349 va
|= (*mmc
& MMU_CONTEXT_ASID_MASK
);
350 __flush_tlb_page(va
);
351 local_irq_restore(flags
);
353 if (!cpus_empty(cpu_mask
))
354 flush_tlb_others(cpu_mask
, mm
, vma
, va
);
359 /*==========================================================================*
360 * Name: flush_tlb_others
362 * Description: This routine requests other CPU to execute flush TLB.
363 * 1.Setup parameters.
364 * 2.Send 'INVALIDATE_TLB_IPI' to other CPU.
365 * Request other CPU to execute 'smp_invalidate_interrupt()'.
366 * 3.Wait for other CPUs operation finished.
368 * Born on Date: 2002.02.05
370 * Arguments: cpumask - bitmap of target CPUs
371 * *mm - a pointer to the mm struct for flush TLB
372 * *vma - a pointer to the vma struct include va
373 * va - virtual address for flush TLB
375 * Returns: void (cannot fail)
378 * Date Who Description
379 * ---------- --- --------------------------------------------------------
381 *==========================================================================*/
382 static void flush_tlb_others(cpumask_t cpumask
, struct mm_struct
*mm
,
383 struct vm_area_struct
*vma
, unsigned long va
)
389 if (!(flags
& 0x0040)) /* Interrupt Disable NONONO */
391 #endif /* DEBUG_SMP */
394 * A couple of (to be removed) sanity checks:
396 * - we do not send IPIs to not-yet booted CPUs.
397 * - current CPU must not be in mask
398 * - mask must exist :)
400 BUG_ON(cpus_empty(cpumask
));
402 BUG_ON(cpu_isset(smp_processor_id(), cpumask
));
405 /* If a CPU which we ran on has gone down, OK. */
406 cpus_and(cpumask
, cpumask
, cpu_online_map
);
407 if (cpus_empty(cpumask
))
411 * i'm not happy about this global shared spinlock in the
412 * MM hot path, but we'll see how contended it is.
413 * Temporarily this turns IRQs off, so that lockups are
414 * detected by the NMI watchdog.
416 spin_lock(&tlbstate_lock
);
421 mask
=cpus_addr(cpumask
);
422 atomic_set_mask(*mask
, (atomic_t
*)&flush_cpumask
);
425 * We have to send the IPI only to
428 send_IPI_mask(&cpumask
, INVALIDATE_TLB_IPI
, 0);
430 while (!cpus_empty(flush_cpumask
)) {
431 /* nothing. lockup detection does not belong here */
438 spin_unlock(&tlbstate_lock
);
441 /*==========================================================================*
442 * Name: smp_invalidate_interrupt
444 * Description: This routine executes on CPU which received
445 * 'INVALIDATE_TLB_IPI'.
447 * 2.Report flush TLB process was finished.
449 * Born on Date: 2002.02.05
453 * Returns: void (cannot fail)
456 * Date Who Description
457 * ---------- --- --------------------------------------------------------
459 *==========================================================================*/
460 void smp_invalidate_interrupt(void)
462 int cpu_id
= smp_processor_id();
463 unsigned long *mmc
= &flush_mm
->context
[cpu_id
];
465 if (!cpu_isset(cpu_id
, flush_cpumask
))
468 if (flush_va
== FLUSH_ALL
) {
470 if (flush_mm
== current
->active_mm
)
471 activate_context(flush_mm
);
473 cpumask_clear_cpu(cpu_id
, mm_cpumask(flush_mm
));
475 unsigned long va
= flush_va
;
477 if (*mmc
!= NO_CONTEXT
) {
479 va
|= (*mmc
& MMU_CONTEXT_ASID_MASK
);
480 __flush_tlb_page(va
);
483 cpu_clear(cpu_id
, flush_cpumask
);
486 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
487 /* Stop CPU request Routines */
488 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
490 /*==========================================================================*
491 * Name: smp_send_stop
493 * Description: This routine requests stop all CPUs.
494 * 1.Request other CPU to execute 'stop_this_cpu()'.
496 * Born on Date: 2002.02.05
500 * Returns: void (cannot fail)
503 * Date Who Description
504 * ---------- --- --------------------------------------------------------
506 *==========================================================================*/
507 void smp_send_stop(void)
509 smp_call_function(stop_this_cpu
, NULL
, 0);
512 /*==========================================================================*
513 * Name: stop_this_cpu
515 * Description: This routine halt CPU.
517 * Born on Date: 2002.02.05
521 * Returns: void (cannot fail)
524 * Date Who Description
525 * ---------- --- --------------------------------------------------------
527 *==========================================================================*/
528 static void stop_this_cpu(void *dummy
)
530 int cpu_id
= smp_processor_id();
535 cpu_clear(cpu_id
, cpu_online_map
);
543 outl(0, M32R_ICU_IMASK_PORTL
);
544 inl(M32R_ICU_IMASK_PORTL
); /* dummy read */
550 void arch_send_call_function_ipi_mask(const struct cpumask
*mask
)
552 send_IPI_mask(mask
, CALL_FUNCTION_IPI
, 0);
555 void arch_send_call_function_single_ipi(int cpu
)
557 send_IPI_mask(cpumask_of(cpu
), CALL_FUNC_SINGLE_IPI
, 0);
560 /*==========================================================================*
561 * Name: smp_call_function_interrupt
563 * Description: This routine executes on CPU which received
564 * 'CALL_FUNCTION_IPI'.
566 * Born on Date: 2002.02.05
570 * Returns: void (cannot fail)
573 * Date Who Description
574 * ---------- --- --------------------------------------------------------
576 *==========================================================================*/
577 void smp_call_function_interrupt(void)
580 generic_smp_call_function_interrupt();
584 void smp_call_function_single_interrupt(void)
587 generic_smp_call_function_single_interrupt();
591 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
593 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
595 /*==========================================================================*
596 * Name: smp_send_timer
598 * Description: This routine sends a 'LOCAL_TIMER_IPI' to all other CPUs
601 * Born on Date: 2002.02.05
605 * Returns: void (cannot fail)
608 * Date Who Description
609 * ---------- --- --------------------------------------------------------
611 *==========================================================================*/
612 void smp_send_timer(void)
614 send_IPI_allbutself(LOCAL_TIMER_IPI
, 1);
617 /*==========================================================================*
618 * Name: smp_send_timer
620 * Description: This routine executes on CPU which received
623 * Born on Date: 2002.02.05
625 * Arguments: *regs - a pointer to the saved regster info
627 * Returns: void (cannot fail)
630 * Date Who Description
631 * ---------- --- --------------------------------------------------------
633 *==========================================================================*/
634 void smp_ipi_timer_interrupt(struct pt_regs
*regs
)
636 struct pt_regs
*old_regs
;
637 old_regs
= set_irq_regs(regs
);
639 smp_local_timer_interrupt();
641 set_irq_regs(old_regs
);
644 /*==========================================================================*
645 * Name: smp_local_timer_interrupt
647 * Description: Local timer interrupt handler. It does both profiling and
648 * process statistics/rescheduling.
649 * We do profiling in every local tick, statistics/rescheduling
650 * happen only every 'profiling multiplier' ticks. The default
651 * multiplier is 1 and it can be changed by writing the new
652 * multiplier value into /proc/profile.
654 * Born on Date: 2002.02.05
656 * Arguments: *regs - a pointer to the saved regster info
658 * Returns: void (cannot fail)
660 * Original: arch/i386/kernel/apic.c
663 * Date Who Description
664 * ---------- --- --------------------------------------------------------
665 * 2003-06-24 hy use per_cpu structure.
666 *==========================================================================*/
667 void smp_local_timer_interrupt(void)
669 int user
= user_mode(get_irq_regs());
670 int cpu_id
= smp_processor_id();
673 * The profiling function is SMP safe. (nothing can mess
674 * around with "current", and the profiling counters are
675 * updated with atomic operations). This is especially
676 * useful with a profiling multiplier != 1
679 profile_tick(CPU_PROFILING
);
681 if (--per_cpu(prof_counter
, cpu_id
) <= 0) {
683 * The multiplier may have changed since the last time we got
684 * to this point as a result of the user writing to
685 * /proc/profile. In this case we need to adjust the APIC
688 * Interrupts are already masked off at this point.
690 per_cpu(prof_counter
, cpu_id
)
691 = per_cpu(prof_multiplier
, cpu_id
);
692 if (per_cpu(prof_counter
, cpu_id
)
693 != per_cpu(prof_old_multiplier
, cpu_id
))
695 per_cpu(prof_old_multiplier
, cpu_id
)
696 = per_cpu(prof_counter
, cpu_id
);
699 update_process_times(user
);
703 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
704 /* Send IPI Routines */
705 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
707 /*==========================================================================*
708 * Name: send_IPI_allbutself
710 * Description: This routine sends a IPI to all other CPUs in the system.
712 * Born on Date: 2002.02.05
714 * Arguments: ipi_num - Number of IPI
715 * try - 0 : Send IPI certainly.
716 * !0 : The following IPI is not sent when Target CPU
717 * has not received the before IPI.
719 * Returns: void (cannot fail)
722 * Date Who Description
723 * ---------- --- --------------------------------------------------------
725 *==========================================================================*/
726 static void send_IPI_allbutself(int ipi_num
, int try)
730 cpumask
= cpu_online_map
;
731 cpu_clear(smp_processor_id(), cpumask
);
733 send_IPI_mask(&cpumask
, ipi_num
, try);
736 /*==========================================================================*
737 * Name: send_IPI_mask
739 * Description: This routine sends a IPI to CPUs in the system.
741 * Born on Date: 2002.02.05
743 * Arguments: cpu_mask - Bitmap of target CPUs logical ID
744 * ipi_num - Number of IPI
745 * try - 0 : Send IPI certainly.
746 * !0 : The following IPI is not sent when Target CPU
747 * has not received the before IPI.
749 * Returns: void (cannot fail)
752 * Date Who Description
753 * ---------- --- --------------------------------------------------------
755 *==========================================================================*/
756 static void send_IPI_mask(const struct cpumask
*cpumask
, int ipi_num
, int try)
758 cpumask_t physid_mask
, tmp
;
760 int num_cpus
= num_online_cpus();
762 if (num_cpus
<= 1) /* NO MP */
765 cpumask_and(&tmp
, cpumask
, cpu_online_mask
);
766 BUG_ON(!cpumask_equal(cpumask
, &tmp
));
768 physid_mask
= CPU_MASK_NONE
;
769 for_each_cpu(cpu_id
, cpumask
) {
770 if ((phys_id
= cpu_to_physid(cpu_id
)) != -1)
771 cpu_set(phys_id
, physid_mask
);
774 send_IPI_mask_phys(physid_mask
, ipi_num
, try);
777 /*==========================================================================*
778 * Name: send_IPI_mask_phys
780 * Description: This routine sends a IPI to other CPUs in the system.
782 * Born on Date: 2002.02.05
784 * Arguments: cpu_mask - Bitmap of target CPUs physical ID
785 * ipi_num - Number of IPI
786 * try - 0 : Send IPI certainly.
787 * !0 : The following IPI is not sent when Target CPU
788 * has not received the before IPI.
790 * Returns: IPICRi regster value.
793 * Date Who Description
794 * ---------- --- --------------------------------------------------------
796 *==========================================================================*/
797 unsigned long send_IPI_mask_phys(cpumask_t physid_mask
, int ipi_num
,
801 volatile unsigned long *ipicr_addr
;
802 unsigned long ipicr_val
;
803 unsigned long my_physid_mask
;
804 unsigned long mask
= cpus_addr(physid_mask
)[0];
807 if (mask
& ~physids_coerce(phys_cpu_present_map
))
809 if (ipi_num
>= NR_IPIS
|| ipi_num
< 0)
813 ipilock
= &ipi_lock
[ipi_num
];
814 ipicr_addr
= (volatile unsigned long *)(M32R_ICU_IPICR_ADDR
816 my_physid_mask
= ~(1 << smp_processor_id());
821 * write IPICRi (send IPIi)
825 __asm__
__volatile__ (
826 ";; CHECK IPICRi == 0 \n\t"
834 ";; WRITE IPICRi (send IPIi) \n\t"
841 : "r"(ipicr_addr
), "r"(mask
), "r"(try), "r"(my_physid_mask
)
844 spin_unlock(ipilock
);