arch/m32r/kernel/smp.c

   1 /*
   2  *  linux/arch/m32r/kernel/smp.c
   3  *
   4  *  M32R SMP support routines.
   5  *
   6  *  Copyright (c) 2001, 2002  Hitoshi Yamamoto
   7  *
   8  *  Taken from i386 version.
   9  *    (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
  10  *    (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
  11  *
  12  *  This code is released under the GNU General Public License version 2 or
  13  *  later.
  14  */
  15
  16 #undef DEBUG_SMP
  17
  18 #include <linux/irq.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/sched.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/mm.h>
  23 #include <linux/smp.h>
  24 #include <linux/profile.h>
  25 #include <linux/cpu.h>
  26
  27 #include <asm/cacheflush.h>
  28 #include <asm/pgalloc.h>
  29 #include <asm/atomic.h>
  30 #include <asm/io.h>
  31 #include <asm/mmu_context.h>
  32 #include <asm/m32r.h>
  33
  34 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
  35 /* Data structures and variables                                             */
  36 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
  37
  38 /*
  39  * For flush_cache_all()
  40  */
  41 static DEFINE_SPINLOCK(flushcache_lock);
  42 static volatile unsigned long flushcache_cpumask = 0;
  43
  44 /*
  45  * For flush_tlb_others()
  46  */
  47 static volatile cpumask_t flush_cpumask;
  48 static struct mm_struct *flush_mm;
  49 static struct vm_area_struct *flush_vma;
  50 static volatile unsigned long flush_va;
  51 static DEFINE_SPINLOCK(tlbstate_lock);
  52 #define FLUSH_ALL 0xffffffff
  53
  54 DECLARE_PER_CPU(int, prof_multiplier);
  55 DECLARE_PER_CPU(int, prof_old_multiplier);
  56 DECLARE_PER_CPU(int, prof_counter);
  57
  58 extern spinlock_t ipi_lock[];
  59
  60 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
  61 /* Function Prototypes                                                       */
  62 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
  63
  64 void smp_send_reschedule(int);
  65 void smp_reschedule_interrupt(void);
  66
  67 void smp_flush_cache_all(void);
  68 void smp_flush_cache_all_interrupt(void);
  69
  70 void smp_flush_tlb_all(void);
  71 static void flush_tlb_all_ipi(void *);
  72
  73 void smp_flush_tlb_mm(struct mm_struct *);
  74 void smp_flush_tlb_range(struct vm_area_struct *, unsigned long, \
  75         unsigned long);
  76 void smp_flush_tlb_page(struct vm_area_struct *, unsigned long);
  77 static void flush_tlb_others(cpumask_t, struct mm_struct *,
  78         struct vm_area_struct *, unsigned long);
  79 void smp_invalidate_interrupt(void);
  80
  81 void smp_send_stop(void);
  82 static void stop_this_cpu(void *);
  83
  84 void smp_send_timer(void);
  85 void smp_ipi_timer_interrupt(struct pt_regs *);
  86 void smp_local_timer_interrupt(void);
  87
  88 static void send_IPI_allbutself(int, int);
  89 static void send_IPI_mask(const struct cpumask *, int, int);
  90
  91 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
  92 /* Rescheduling request Routines                                             */
  93 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
  94
  95 /*==========================================================================*
  96  * Name:         smp_send_reschedule
  97  *
  98  * Description:  This routine requests other CPU to execute rescheduling.
  99  *               1.Send 'RESCHEDULE_IPI' to other CPU.
 100  *                 Request other CPU to execute 'smp_reschedule_interrupt()'.
 101  *
 102  * Born on Date: 2002.02.05
 103  *
 104  * Arguments:    cpu_id - Target CPU ID
 105  *
 106  * Returns:      void (cannot fail)
 107  *
 108  * Modification log:
 109  * Date       Who Description
 110  * ---------- --- --------------------------------------------------------
 111  *
 112  *==========================================================================*/
 113 void smp_send_reschedule(int cpu_id)
 114 {
 115         WARN_ON(cpu_is_offline(cpu_id));
 116         send_IPI_mask(cpumask_of(cpu_id), RESCHEDULE_IPI, 1);
 117 }
 118
 119 /*==========================================================================*
 120  * Name:         smp_reschedule_interrupt
 121  *
 122  * Description:  This routine executes on CPU which received
 123  *               'RESCHEDULE_IPI'.
 124  *
 125  * Born on Date: 2002.02.05
 126  *
 127  * Arguments:    NONE
 128  *
 129  * Returns:      void (cannot fail)
 130  *
 131  * Modification log:
 132  * Date       Who Description
 133  * ---------- --- --------------------------------------------------------
 134  *
 135  *==========================================================================*/
 136 void smp_reschedule_interrupt(void)
 137 {
 138         scheduler_ipi();
 139 }
 140
 141 /*==========================================================================*
 142  * Name:         smp_flush_cache_all
 143  *
 144  * Description:  This routine sends a 'INVALIDATE_CACHE_IPI' to all other
 145  *               CPUs in the system.
 146  *
 147  * Born on Date: 2003-05-28
 148  *
 149  * Arguments:    NONE
 150  *
 151  * Returns:      void (cannot fail)
 152  *
 153  * Modification log:
 154  * Date       Who Description
 155  * ---------- --- --------------------------------------------------------
 156  *
 157  *==========================================================================*/
 158 void smp_flush_cache_all(void)
 159 {
 160         cpumask_t cpumask;
 161         unsigned long *mask;
 162
 163         preempt_disable();
 164         cpumask_copy(&cpumask, cpu_online_mask);
 165         cpumask_clear_cpu(smp_processor_id(), &cpumask);
 166         spin_lock(&flushcache_lock);
 167         mask=cpumask_bits(&cpumask);
 168         atomic_set_mask(*mask, (atomic_t *)&flushcache_cpumask);
 169         send_IPI_mask(&cpumask, INVALIDATE_CACHE_IPI, 0);
 170         _flush_cache_copyback_all();
 171         while (flushcache_cpumask)
 172                 mb();
 173         spin_unlock(&flushcache_lock);
 174         preempt_enable();
 175 }
 176
 177 void smp_flush_cache_all_interrupt(void)
 178 {
 179         _flush_cache_copyback_all();
 180         clear_bit(smp_processor_id(), &flushcache_cpumask);
 181 }
 182
 183 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 184 /* TLB flush request Routines                                                */
 185 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 186
 187 /*==========================================================================*
 188  * Name:         smp_flush_tlb_all
 189  *
 190  * Description:  This routine flushes all processes TLBs.
 191  *               1.Request other CPU to execute 'flush_tlb_all_ipi()'.
 192  *               2.Execute 'do_flush_tlb_all_local()'.
 193  *
 194  * Born on Date: 2002.02.05
 195  *
 196  * Arguments:    NONE
 197  *
 198  * Returns:      void (cannot fail)
 199  *
 200  * Modification log:
 201  * Date       Who Description
 202  * ---------- --- --------------------------------------------------------
 203  *
 204  *==========================================================================*/
 205 void smp_flush_tlb_all(void)
 206 {
 207         unsigned long flags;
 208
 209         preempt_disable();
 210         local_irq_save(flags);
 211         __flush_tlb_all();
 212         local_irq_restore(flags);
 213         smp_call_function(flush_tlb_all_ipi, NULL, 1);
 214         preempt_enable();
 215 }
 216
 217 /*==========================================================================*
 218  * Name:         flush_tlb_all_ipi
 219  *
 220  * Description:  This routine flushes all local TLBs.
 221  *               1.Execute 'do_flush_tlb_all_local()'.
 222  *
 223  * Born on Date: 2002.02.05
 224  *
 225  * Arguments:    *info - not used
 226  *
 227  * Returns:      void (cannot fail)
 228  *
 229  * Modification log:
 230  * Date       Who Description
 231  * ---------- --- --------------------------------------------------------
 232  *
 233  *==========================================================================*/
 234 static void flush_tlb_all_ipi(void *info)
 235 {
 236         __flush_tlb_all();
 237 }
 238
 239 /*==========================================================================*
 240  * Name:         smp_flush_tlb_mm
 241  *
 242  * Description:  This routine flushes the specified mm context TLB's.
 243  *
 244  * Born on Date: 2002.02.05
 245  *
 246  * Arguments:    *mm - a pointer to the mm struct for flush TLB
 247  *
 248  * Returns:      void (cannot fail)
 249  *
 250  * Modification log:
 251  * Date       Who Description
 252  * ---------- --- --------------------------------------------------------
 253  *
 254  *==========================================================================*/
 255 void smp_flush_tlb_mm(struct mm_struct *mm)
 256 {
 257         int cpu_id;
 258         cpumask_t cpu_mask;
 259         unsigned long *mmc;
 260         unsigned long flags;
 261
 262         preempt_disable();
 263         cpu_id = smp_processor_id();
 264         mmc = &mm->context[cpu_id];
 265         cpumask_copy(&cpu_mask, mm_cpumask(mm));
 266         cpumask_clear_cpu(cpu_id, &cpu_mask);
 267
 268         if (*mmc != NO_CONTEXT) {
 269                 local_irq_save(flags);
 270                 *mmc = NO_CONTEXT;
 271                 if (mm == current->mm)
 272                         activate_context(mm);
 273                 else
 274                         cpumask_clear_cpu(cpu_id, mm_cpumask(mm));
 275                 local_irq_restore(flags);
 276         }
 277         if (!cpumask_empty(&cpu_mask))
 278                 flush_tlb_others(cpu_mask, mm, NULL, FLUSH_ALL);
 279
 280         preempt_enable();
 281 }
 282
 283 /*==========================================================================*
 284  * Name:         smp_flush_tlb_range
 285  *
 286  * Description:  This routine flushes a range of pages.
 287  *
 288  * Born on Date: 2002.02.05
 289  *
 290  * Arguments:    *mm - a pointer to the mm struct for flush TLB
 291  *               start - not used
 292  *               end - not used
 293  *
 294  * Returns:      void (cannot fail)
 295  *
 296  * Modification log:
 297  * Date       Who Description
 298  * ---------- --- --------------------------------------------------------
 299  *
 300  *==========================================================================*/
 301 void smp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 302         unsigned long end)
 303 {
 304         smp_flush_tlb_mm(vma->vm_mm);
 305 }
 306
 307 /*==========================================================================*
 308  * Name:         smp_flush_tlb_page
 309  *
 310  * Description:  This routine flushes one page.
 311  *
 312  * Born on Date: 2002.02.05
 313  *
 314  * Arguments:    *vma - a pointer to the vma struct include va
 315  *               va - virtual address for flush TLB
 316  *
 317  * Returns:      void (cannot fail)
 318  *
 319  * Modification log:
 320  * Date       Who Description
 321  * ---------- --- --------------------------------------------------------
 322  *
 323  *==========================================================================*/
 324 void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
 325 {
 326         struct mm_struct *mm = vma->vm_mm;
 327         int cpu_id;
 328         cpumask_t cpu_mask;
 329         unsigned long *mmc;
 330         unsigned long flags;
 331
 332         preempt_disable();
 333         cpu_id = smp_processor_id();
 334         mmc = &mm->context[cpu_id];
 335         cpumask_copy(&cpu_mask, mm_cpumask(mm));
 336         cpumask_clear_cpu(cpu_id, &cpu_mask);
 337
 338 #ifdef DEBUG_SMP
 339         if (!mm)
 340                 BUG();
 341 #endif
 342
 343         if (*mmc != NO_CONTEXT) {
 344                 local_irq_save(flags);
 345                 va &= PAGE_MASK;
 346                 va |= (*mmc & MMU_CONTEXT_ASID_MASK);
 347                 __flush_tlb_page(va);
 348                 local_irq_restore(flags);
 349         }
 350         if (!cpumask_empty(&cpu_mask))
 351                 flush_tlb_others(cpu_mask, mm, vma, va);
 352
 353         preempt_enable();
 354 }
 355
 356 /*==========================================================================*
 357  * Name:         flush_tlb_others
 358  *
 359  * Description:  This routine requests other CPU to execute flush TLB.
 360  *               1.Setup parameters.
 361  *               2.Send 'INVALIDATE_TLB_IPI' to other CPU.
 362  *                 Request other CPU to execute 'smp_invalidate_interrupt()'.
 363  *               3.Wait for other CPUs operation finished.
 364  *
 365  * Born on Date: 2002.02.05
 366  *
 367  * Arguments:    cpumask - bitmap of target CPUs
 368  *               *mm -  a pointer to the mm struct for flush TLB
 369  *               *vma -  a pointer to the vma struct include va
 370  *               va - virtual address for flush TLB
 371  *
 372  * Returns:      void (cannot fail)
 373  *
 374  * Modification log:
 375  * Date       Who Description
 376  * ---------- --- --------------------------------------------------------
 377  *
 378  *==========================================================================*/
 379 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
 380         struct vm_area_struct *vma, unsigned long va)
 381 {
 382         unsigned long *mask;
 383 #ifdef DEBUG_SMP
 384         unsigned long flags;
 385         __save_flags(flags);
 386         if (!(flags & 0x0040))  /* Interrupt Disable NONONO */
 387                 BUG();
 388 #endif /* DEBUG_SMP */
 389
 390         /*
 391          * A couple of (to be removed) sanity checks:
 392          *
 393          * - we do not send IPIs to not-yet booted CPUs.
 394          * - current CPU must not be in mask
 395          * - mask must exist :)
 396          */
 397         BUG_ON(cpumask_empty(&cpumask));
 398
 399         BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask));
 400         BUG_ON(!mm);
 401
 402         /* If a CPU which we ran on has gone down, OK. */
 403         cpumask_and(&cpumask, &cpumask, cpu_online_mask);
 404         if (cpumask_empty(&cpumask))
 405                 return;
 406
 407         /*
 408          * i'm not happy about this global shared spinlock in the
 409          * MM hot path, but we'll see how contended it is.
 410          * Temporarily this turns IRQs off, so that lockups are
 411          * detected by the NMI watchdog.
 412          */
 413         spin_lock(&tlbstate_lock);
 414
 415         flush_mm = mm;
 416         flush_vma = vma;
 417         flush_va = va;
 418         mask=cpumask_bits(&cpumask);
 419         atomic_set_mask(*mask, (atomic_t *)&flush_cpumask);
 420
 421         /*
 422          * We have to send the IPI only to
 423          * CPUs affected.
 424          */
 425         send_IPI_mask(&cpumask, INVALIDATE_TLB_IPI, 0);
 426
 427         while (!cpumask_empty((cpumask_t*)&flush_cpumask)) {
 428                 /* nothing. lockup detection does not belong here */
 429                 mb();
 430         }
 431
 432         flush_mm = NULL;
 433         flush_vma = NULL;
 434         flush_va = 0;
 435         spin_unlock(&tlbstate_lock);
 436 }
 437
 438 /*==========================================================================*
 439  * Name:         smp_invalidate_interrupt
 440  *
 441  * Description:  This routine executes on CPU which received
 442  *               'INVALIDATE_TLB_IPI'.
 443  *               1.Flush local TLB.
 444  *               2.Report flush TLB process was finished.
 445  *
 446  * Born on Date: 2002.02.05
 447  *
 448  * Arguments:    NONE
 449  *
 450  * Returns:      void (cannot fail)
 451  *
 452  * Modification log:
 453  * Date       Who Description
 454  * ---------- --- --------------------------------------------------------
 455  *
 456  *==========================================================================*/
 457 void smp_invalidate_interrupt(void)
 458 {
 459         int cpu_id = smp_processor_id();
 460         unsigned long *mmc = &flush_mm->context[cpu_id];
 461
 462         if (!cpumask_test_cpu(cpu_id, &flush_cpumask))
 463                 return;
 464
 465         if (flush_va == FLUSH_ALL) {
 466                 *mmc = NO_CONTEXT;
 467                 if (flush_mm == current->active_mm)
 468                         activate_context(flush_mm);
 469                 else
 470                         cpumask_clear_cpu(cpu_id, mm_cpumask(flush_mm));
 471         } else {
 472                 unsigned long va = flush_va;
 473
 474                 if (*mmc != NO_CONTEXT) {
 475                         va &= PAGE_MASK;
 476                         va |= (*mmc & MMU_CONTEXT_ASID_MASK);
 477                         __flush_tlb_page(va);
 478                 }
 479         }
 480         cpumask_clear_cpu(cpu_id, (cpumask_t*)&flush_cpumask);
 481 }
 482
 483 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 484 /* Stop CPU request Routines                                                 */
 485 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 486
 487 /*==========================================================================*
 488  * Name:         smp_send_stop
 489  *
 490  * Description:  This routine requests stop all CPUs.
 491  *               1.Request other CPU to execute 'stop_this_cpu()'.
 492  *
 493  * Born on Date: 2002.02.05
 494  *
 495  * Arguments:    NONE
 496  *
 497  * Returns:      void (cannot fail)
 498  *
 499  * Modification log:
 500  * Date       Who Description
 501  * ---------- --- --------------------------------------------------------
 502  *
 503  *==========================================================================*/
 504 void smp_send_stop(void)
 505 {
 506         smp_call_function(stop_this_cpu, NULL, 0);
 507 }
 508
 509 /*==========================================================================*
 510  * Name:         stop_this_cpu
 511  *
 512  * Description:  This routine halt CPU.
 513  *
 514  * Born on Date: 2002.02.05
 515  *
 516  * Arguments:    NONE
 517  *
 518  * Returns:      void (cannot fail)
 519  *
 520  * Modification log:
 521  * Date       Who Description
 522  * ---------- --- --------------------------------------------------------
 523  *
 524  *==========================================================================*/
 525 static void stop_this_cpu(void *dummy)
 526 {
 527         int cpu_id = smp_processor_id();
 528
 529         /*
 530          * Remove this CPU:
 531          */
 532         set_cpu_online(cpu_id, false);
 533
 534         /*
 535          * PSW IE = 1;
 536          * IMASK = 0;
 537          * goto SLEEP
 538          */
 539         local_irq_disable();
 540         outl(0, M32R_ICU_IMASK_PORTL);
 541         inl(M32R_ICU_IMASK_PORTL);      /* dummy read */
 542         local_irq_enable();
 543
 544         for ( ; ; );
 545 }
 546
 547 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 548 {
 549         send_IPI_mask(mask, CALL_FUNCTION_IPI, 0);
 550 }
 551
 552 void arch_send_call_function_single_ipi(int cpu)
 553 {
 554         send_IPI_mask(cpumask_of(cpu), CALL_FUNC_SINGLE_IPI, 0);
 555 }
 556
 557 /*==========================================================================*
 558  * Name:         smp_call_function_interrupt
 559  *
 560  * Description:  This routine executes on CPU which received
 561  *               'CALL_FUNCTION_IPI'.
 562  *
 563  * Born on Date: 2002.02.05
 564  *
 565  * Arguments:    NONE
 566  *
 567  * Returns:      void (cannot fail)
 568  *
 569  * Modification log:
 570  * Date       Who Description
 571  * ---------- --- --------------------------------------------------------
 572  *
 573  *==========================================================================*/
 574 void smp_call_function_interrupt(void)
 575 {
 576         irq_enter();
 577         generic_smp_call_function_interrupt();
 578         irq_exit();
 579 }
 580
 581 void smp_call_function_single_interrupt(void)
 582 {
 583         irq_enter();
 584         generic_smp_call_function_single_interrupt();
 585         irq_exit();
 586 }
 587
 588 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 589 /* Timer Routines                                                            */
 590 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 591
 592 /*==========================================================================*
 593  * Name:         smp_send_timer
 594  *
 595  * Description:  This routine sends a 'LOCAL_TIMER_IPI' to all other CPUs
 596  *               in the system.
 597  *
 598  * Born on Date: 2002.02.05
 599  *
 600  * Arguments:    NONE
 601  *
 602  * Returns:      void (cannot fail)
 603  *
 604  * Modification log:
 605  * Date       Who Description
 606  * ---------- --- --------------------------------------------------------
 607  *
 608  *==========================================================================*/
 609 void smp_send_timer(void)
 610 {
 611         send_IPI_allbutself(LOCAL_TIMER_IPI, 1);
 612 }
 613
 614 /*==========================================================================*
 615  * Name:         smp_send_timer
 616  *
 617  * Description:  This routine executes on CPU which received
 618  *               'LOCAL_TIMER_IPI'.
 619  *
 620  * Born on Date: 2002.02.05
 621  *
 622  * Arguments:    *regs - a pointer to the saved regster info
 623  *
 624  * Returns:      void (cannot fail)
 625  *
 626  * Modification log:
 627  * Date       Who Description
 628  * ---------- --- --------------------------------------------------------
 629  *
 630  *==========================================================================*/
 631 void smp_ipi_timer_interrupt(struct pt_regs *regs)
 632 {
 633         struct pt_regs *old_regs;
 634         old_regs = set_irq_regs(regs);
 635         irq_enter();
 636         smp_local_timer_interrupt();
 637         irq_exit();
 638         set_irq_regs(old_regs);
 639 }
 640
 641 /*==========================================================================*
 642  * Name:         smp_local_timer_interrupt
 643  *
 644  * Description:  Local timer interrupt handler. It does both profiling and
 645  *               process statistics/rescheduling.
 646  *               We do profiling in every local tick, statistics/rescheduling
 647  *               happen only every 'profiling multiplier' ticks. The default
 648  *               multiplier is 1 and it can be changed by writing the new
 649  *               multiplier value into /proc/profile.
 650  *
 651  * Born on Date: 2002.02.05
 652  *
 653  * Arguments:    *regs - a pointer to the saved regster info
 654  *
 655  * Returns:      void (cannot fail)
 656  *
 657  * Original:     arch/i386/kernel/apic.c
 658  *
 659  * Modification log:
 660  * Date       Who Description
 661  * ---------- --- --------------------------------------------------------
 662  * 2003-06-24 hy  use per_cpu structure.
 663  *==========================================================================*/
 664 void smp_local_timer_interrupt(void)
 665 {
 666         int user = user_mode(get_irq_regs());
 667         int cpu_id = smp_processor_id();
 668
 669         /*
 670          * The profiling function is SMP safe. (nothing can mess
 671          * around with "current", and the profiling counters are
 672          * updated with atomic operations). This is especially
 673          * useful with a profiling multiplier != 1
 674          */
 675
 676         profile_tick(CPU_PROFILING);
 677
 678         if (--per_cpu(prof_counter, cpu_id) <= 0) {
 679                 /*
 680                  * The multiplier may have changed since the last time we got
 681                  * to this point as a result of the user writing to
 682                  * /proc/profile. In this case we need to adjust the APIC
 683                  * timer accordingly.
 684                  *
 685                  * Interrupts are already masked off at this point.
 686                  */
 687                 per_cpu(prof_counter, cpu_id)
 688                         = per_cpu(prof_multiplier, cpu_id);
 689                 if (per_cpu(prof_counter, cpu_id)
 690                         != per_cpu(prof_old_multiplier, cpu_id))
 691                 {
 692                         per_cpu(prof_old_multiplier, cpu_id)
 693                                 = per_cpu(prof_counter, cpu_id);
 694                 }
 695
 696                 update_process_times(user);
 697         }
 698 }
 699
 700 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 701 /* Send IPI Routines                                                         */
 702 /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/
 703
 704 /*==========================================================================*
 705  * Name:         send_IPI_allbutself
 706  *
 707  * Description:  This routine sends a IPI to all other CPUs in the system.
 708  *
 709  * Born on Date: 2002.02.05
 710  *
 711  * Arguments:    ipi_num - Number of IPI
 712  *               try -  0 : Send IPI certainly.
 713  *                     !0 : The following IPI is not sent when Target CPU
 714  *                          has not received the before IPI.
 715  *
 716  * Returns:      void (cannot fail)
 717  *
 718  * Modification log:
 719  * Date       Who Description
 720  * ---------- --- --------------------------------------------------------
 721  *
 722  *==========================================================================*/
 723 static void send_IPI_allbutself(int ipi_num, int try)
 724 {
 725         cpumask_t cpumask;
 726
 727         cpumask_copy(&cpumask, cpu_online_mask);
 728         cpumask_clear_cpu(smp_processor_id(), &cpumask);
 729
 730         send_IPI_mask(&cpumask, ipi_num, try);
 731 }
 732
 733 /*==========================================================================*
 734  * Name:         send_IPI_mask
 735  *
 736  * Description:  This routine sends a IPI to CPUs in the system.
 737  *
 738  * Born on Date: 2002.02.05
 739  *
 740  * Arguments:    cpu_mask - Bitmap of target CPUs logical ID
 741  *               ipi_num - Number of IPI
 742  *               try -  0 : Send IPI certainly.
 743  *                     !0 : The following IPI is not sent when Target CPU
 744  *                          has not received the before IPI.
 745  *
 746  * Returns:      void (cannot fail)
 747  *
 748  * Modification log:
 749  * Date       Who Description
 750  * ---------- --- --------------------------------------------------------
 751  *
 752  *==========================================================================*/
 753 static void send_IPI_mask(const struct cpumask *cpumask, int ipi_num, int try)
 754 {
 755         cpumask_t physid_mask, tmp;
 756         int cpu_id, phys_id;
 757         int num_cpus = num_online_cpus();
 758
 759         if (num_cpus <= 1)      /* NO MP */
 760                 return;
 761
 762         cpumask_and(&tmp, cpumask, cpu_online_mask);
 763         BUG_ON(!cpumask_equal(cpumask, &tmp));
 764
 765         cpumask_clear(&physid_mask);
 766         for_each_cpu(cpu_id, cpumask) {
 767                 if ((phys_id = cpu_to_physid(cpu_id)) != -1)
 768                         cpumask_set_cpu(phys_id, &physid_mask);
 769         }
 770
 771         send_IPI_mask_phys(&physid_mask, ipi_num, try);
 772 }
 773
 774 /*==========================================================================*
 775  * Name:         send_IPI_mask_phys
 776  *
 777  * Description:  This routine sends a IPI to other CPUs in the system.
 778  *
 779  * Born on Date: 2002.02.05
 780  *
 781  * Arguments:    cpu_mask - Bitmap of target CPUs physical ID
 782  *               ipi_num - Number of IPI
 783  *               try -  0 : Send IPI certainly.
 784  *                     !0 : The following IPI is not sent when Target CPU
 785  *                          has not received the before IPI.
 786  *
 787  * Returns:      IPICRi regster value.
 788  *
 789  * Modification log:
 790  * Date       Who Description
 791  * ---------- --- --------------------------------------------------------
 792  *
 793  *==========================================================================*/
 794 unsigned long send_IPI_mask_phys(const cpumask_t *physid_mask, int ipi_num,
 795         int try)
 796 {
 797         spinlock_t *ipilock;
 798         volatile unsigned long *ipicr_addr;
 799         unsigned long ipicr_val;
 800         unsigned long my_physid_mask;
 801         unsigned long mask = cpumask_bits(physid_mask)[0];
 802
 803
 804         if (mask & ~physids_coerce(phys_cpu_present_map))
 805                 BUG();
 806         if (ipi_num >= NR_IPIS || ipi_num < 0)
 807                 BUG();
 808
 809         mask <<= IPI_SHIFT;
 810         ipilock = &ipi_lock[ipi_num];
 811         ipicr_addr = (volatile unsigned long *)(M32R_ICU_IPICR_ADDR
 812                 + (ipi_num << 2));
 813         my_physid_mask = ~(1 << smp_processor_id());
 814
 815         /*
 816          * lock ipi_lock[i]
 817          * check IPICRi == 0
 818          * write IPICRi (send IPIi)
 819          * unlock ipi_lock[i]
 820          */
 821         spin_lock(ipilock);
 822         __asm__ __volatile__ (
 823                 ";; CHECK IPICRi == 0           \n\t"
 824                 ".fillinsn                      \n"
 825                 "1:                             \n\t"
 826                 "ld     %0, @%1                 \n\t"
 827                 "and    %0, %4                  \n\t"
 828                 "beqz   %0, 2f                  \n\t"
 829                 "bnez   %3, 3f                  \n\t"
 830                 "bra    1b                      \n\t"
 831                 ";; WRITE IPICRi (send IPIi)    \n\t"
 832                 ".fillinsn                      \n"
 833                 "2:                             \n\t"
 834                 "st     %2, @%1                 \n\t"
 835                 ".fillinsn                      \n"
 836                 "3:                             \n\t"
 837                 : "=&r"(ipicr_val)
 838                 : "r"(ipicr_addr), "r"(mask), "r"(try), "r"(my_physid_mask)
 839                 : "memory"
 840         );
 841         spin_unlock(ipilock);
 842
 843         return ipicr_val;
 844 }