arch/blackfin/mach-common/smp.c

   1 /*
   2  * IPI management based on arch/arm/kernel/smp.c (Copyright 2002 ARM Limited)
   3  *
   4  * Copyright 2007-2009 Analog Devices Inc.
   5  *                         Philippe Gerum <rpm@xenomai.org>
   6  *
   7  * Licensed under the GPL-2.
   8  */
   9
  10 #include <linux/module.h>
  11 #include <linux/delay.h>
  12 #include <linux/init.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/sched.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/cache.h>
  17 #include <linux/profile.h>
  18 #include <linux/errno.h>
  19 #include <linux/mm.h>
  20 #include <linux/cpu.h>
  21 #include <linux/smp.h>
  22 #include <linux/cpumask.h>
  23 #include <linux/seq_file.h>
  24 #include <linux/irq.h>
  25 #include <linux/slab.h>
  26 #include <asm/atomic.h>
  27 #include <asm/cacheflush.h>
  28 #include <asm/mmu_context.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/pgalloc.h>
  31 #include <asm/processor.h>
  32 #include <asm/ptrace.h>
  33 #include <asm/cpu.h>
  34 #include <asm/time.h>
  35 #include <linux/err.h>
  36
  37 /*
  38  * Anomaly notes:
  39  * 05000120 - we always define corelock as 32-bit integer in L2
  40  */
  41 struct corelock_slot corelock __attribute__ ((__section__(".l2.bss")));
  42
  43 void __cpuinitdata *init_retx_coreb, *init_saved_retx_coreb,
  44         *init_saved_seqstat_coreb, *init_saved_icplb_fault_addr_coreb,
  45         *init_saved_dcplb_fault_addr_coreb;
  46
  47 #define BFIN_IPI_RESCHEDULE   0
  48 #define BFIN_IPI_CALL_FUNC    1
  49 #define BFIN_IPI_CPU_STOP     2
  50
  51 struct blackfin_flush_data {
  52         unsigned long start;
  53         unsigned long end;
  54 };
  55
  56 void *secondary_stack;
  57
  58
  59 struct smp_call_struct {
  60         void (*func)(void *info);
  61         void *info;
  62         int wait;
  63         cpumask_t pending;
  64         cpumask_t waitmask;
  65 };
  66
  67 static struct blackfin_flush_data smp_flush_data;
  68
  69 static DEFINE_SPINLOCK(stop_lock);
  70
  71 struct ipi_message {
  72         struct list_head list;
  73         unsigned long type;
  74         struct smp_call_struct call_struct;
  75 };
  76
  77 struct ipi_message_queue {
  78         struct list_head head;
  79         spinlock_t lock;
  80         unsigned long count;
  81 };
  82
  83 static DEFINE_PER_CPU(struct ipi_message_queue, ipi_msg_queue);
  84
  85 static void ipi_cpu_stop(unsigned int cpu)
  86 {
  87         spin_lock(&stop_lock);
  88         printk(KERN_CRIT "CPU%u: stopping\n", cpu);
  89         dump_stack();
  90         spin_unlock(&stop_lock);
  91
  92         cpu_clear(cpu, cpu_online_map);
  93
  94         local_irq_disable();
  95
  96         while (1)
  97                 SSYNC();
  98 }
  99
 100 static void ipi_flush_icache(void *info)
 101 {
 102         struct blackfin_flush_data *fdata = info;
 103
 104         /* Invalidate the memory holding the bounds of the flushed region. */
 105         blackfin_dcache_invalidate_range((unsigned long)fdata,
 106                                          (unsigned long)fdata + sizeof(*fdata));
 107
 108         blackfin_icache_flush_range(fdata->start, fdata->end);
 109 }
 110
 111 static void ipi_call_function(unsigned int cpu, struct ipi_message *msg)
 112 {
 113         int wait;
 114         void (*func)(void *info);
 115         void *info;
 116         func = msg->call_struct.func;
 117         info = msg->call_struct.info;
 118         wait = msg->call_struct.wait;
 119         cpu_clear(cpu, msg->call_struct.pending);
 120         func(info);
 121         if (wait) {
 122 #ifdef __ARCH_SYNC_CORE_DCACHE
 123                 /*
 124                  * 'wait' usually means synchronization between CPUs.
 125                  * Invalidate D cache in case shared data was changed
 126                  * by func() to ensure cache coherence.
 127                  */
 128                 resync_core_dcache();
 129 #endif
 130                 cpu_clear(cpu, msg->call_struct.waitmask);
 131         } else
 132                 kfree(msg);
 133 }
 134
 135 static irqreturn_t ipi_handler(int irq, void *dev_instance)
 136 {
 137         struct ipi_message *msg;
 138         struct ipi_message_queue *msg_queue;
 139         unsigned int cpu = smp_processor_id();
 140
 141         platform_clear_ipi(cpu);
 142
 143         msg_queue = &__get_cpu_var(ipi_msg_queue);
 144         msg_queue->count++;
 145
 146         spin_lock(&msg_queue->lock);
 147         while (!list_empty(&msg_queue->head)) {
 148                 msg = list_entry(msg_queue->head.next, typeof(*msg), list);
 149                 list_del(&msg->list);
 150                 switch (msg->type) {
 151                 case BFIN_IPI_RESCHEDULE:
 152                         /* That's the easiest one; leave it to
 153                          * return_from_int. */
 154                         kfree(msg);
 155                         break;
 156                 case BFIN_IPI_CALL_FUNC:
 157                         spin_unlock(&msg_queue->lock);
 158                         ipi_call_function(cpu, msg);
 159                         spin_lock(&msg_queue->lock);
 160                         break;
 161                 case BFIN_IPI_CPU_STOP:
 162                         spin_unlock(&msg_queue->lock);
 163                         ipi_cpu_stop(cpu);
 164                         spin_lock(&msg_queue->lock);
 165                         kfree(msg);
 166                         break;
 167                 default:
 168                         printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%lx\n",
 169                                cpu, msg->type);
 170                         kfree(msg);
 171                         break;
 172                 }
 173         }
 174         spin_unlock(&msg_queue->lock);
 175         return IRQ_HANDLED;
 176 }
 177
 178 static void ipi_queue_init(void)
 179 {
 180         unsigned int cpu;
 181         struct ipi_message_queue *msg_queue;
 182         for_each_possible_cpu(cpu) {
 183                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
 184                 INIT_LIST_HEAD(&msg_queue->head);
 185                 spin_lock_init(&msg_queue->lock);
 186                 msg_queue->count = 0;
 187         }
 188 }
 189
 190 int smp_call_function(void (*func)(void *info), void *info, int wait)
 191 {
 192         unsigned int cpu;
 193         cpumask_t callmap;
 194         unsigned long flags;
 195         struct ipi_message_queue *msg_queue;
 196         struct ipi_message *msg;
 197
 198         callmap = cpu_online_map;
 199         cpu_clear(smp_processor_id(), callmap);
 200         if (cpus_empty(callmap))
 201                 return 0;
 202
 203         msg = kmalloc(sizeof(*msg), GFP_ATOMIC);
 204         if (!msg)
 205                 return -ENOMEM;
 206         INIT_LIST_HEAD(&msg->list);
 207         msg->call_struct.func = func;
 208         msg->call_struct.info = info;
 209         msg->call_struct.wait = wait;
 210         msg->call_struct.pending = callmap;
 211         msg->call_struct.waitmask = callmap;
 212         msg->type = BFIN_IPI_CALL_FUNC;
 213
 214         for_each_cpu_mask(cpu, callmap) {
 215                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
 216                 spin_lock_irqsave(&msg_queue->lock, flags);
 217                 list_add_tail(&msg->list, &msg_queue->head);
 218                 spin_unlock_irqrestore(&msg_queue->lock, flags);
 219                 platform_send_ipi_cpu(cpu);
 220         }
 221         if (wait) {
 222                 while (!cpus_empty(msg->call_struct.waitmask))
 223                         blackfin_dcache_invalidate_range(
 224                                 (unsigned long)(&msg->call_struct.waitmask),
 225                                 (unsigned long)(&msg->call_struct.waitmask));
 226 #ifdef __ARCH_SYNC_CORE_DCACHE
 227                 /*
 228                  * Invalidate D cache in case shared data was changed by
 229                  * other processors to ensure cache coherence.
 230                  */
 231                 resync_core_dcache();
 232 #endif
 233                 kfree(msg);
 234         }
 235         return 0;
 236 }
 237 EXPORT_SYMBOL_GPL(smp_call_function);
 238
 239 int smp_call_function_single(int cpuid, void (*func) (void *info), void *info,
 240                                 int wait)
 241 {
 242         unsigned int cpu = cpuid;
 243         cpumask_t callmap;
 244         unsigned long flags;
 245         struct ipi_message_queue *msg_queue;
 246         struct ipi_message *msg;
 247
 248         if (cpu_is_offline(cpu))
 249                 return 0;
 250         cpus_clear(callmap);
 251         cpu_set(cpu, callmap);
 252
 253         msg = kmalloc(sizeof(*msg), GFP_ATOMIC);
 254         if (!msg)
 255                 return -ENOMEM;
 256         INIT_LIST_HEAD(&msg->list);
 257         msg->call_struct.func = func;
 258         msg->call_struct.info = info;
 259         msg->call_struct.wait = wait;
 260         msg->call_struct.pending = callmap;
 261         msg->call_struct.waitmask = callmap;
 262         msg->type = BFIN_IPI_CALL_FUNC;
 263
 264         msg_queue = &per_cpu(ipi_msg_queue, cpu);
 265         spin_lock_irqsave(&msg_queue->lock, flags);
 266         list_add_tail(&msg->list, &msg_queue->head);
 267         spin_unlock_irqrestore(&msg_queue->lock, flags);
 268         platform_send_ipi_cpu(cpu);
 269
 270         if (wait) {
 271                 while (!cpus_empty(msg->call_struct.waitmask))
 272                         blackfin_dcache_invalidate_range(
 273                                 (unsigned long)(&msg->call_struct.waitmask),
 274                                 (unsigned long)(&msg->call_struct.waitmask));
 275 #ifdef __ARCH_SYNC_CORE_DCACHE
 276                 /*
 277                  * Invalidate D cache in case shared data was changed by
 278                  * other processors to ensure cache coherence.
 279                  */
 280                 resync_core_dcache();
 281 #endif
 282                 kfree(msg);
 283         }
 284         return 0;
 285 }
 286 EXPORT_SYMBOL_GPL(smp_call_function_single);
 287
 288 void smp_send_reschedule(int cpu)
 289 {
 290         unsigned long flags;
 291         struct ipi_message_queue *msg_queue;
 292         struct ipi_message *msg;
 293
 294         if (cpu_is_offline(cpu))
 295                 return;
 296
 297         msg = kzalloc(sizeof(*msg), GFP_ATOMIC);
 298         if (!msg)
 299                 return;
 300         INIT_LIST_HEAD(&msg->list);
 301         msg->type = BFIN_IPI_RESCHEDULE;
 302
 303         msg_queue = &per_cpu(ipi_msg_queue, cpu);
 304         spin_lock_irqsave(&msg_queue->lock, flags);
 305         list_add_tail(&msg->list, &msg_queue->head);
 306         spin_unlock_irqrestore(&msg_queue->lock, flags);
 307         platform_send_ipi_cpu(cpu);
 308
 309         return;
 310 }
 311
 312 void smp_send_stop(void)
 313 {
 314         unsigned int cpu;
 315         cpumask_t callmap;
 316         unsigned long flags;
 317         struct ipi_message_queue *msg_queue;
 318         struct ipi_message *msg;
 319
 320         callmap = cpu_online_map;
 321         cpu_clear(smp_processor_id(), callmap);
 322         if (cpus_empty(callmap))
 323                 return;
 324
 325         msg = kzalloc(sizeof(*msg), GFP_ATOMIC);
 326         if (!msg)
 327                 return;
 328         INIT_LIST_HEAD(&msg->list);
 329         msg->type = BFIN_IPI_CPU_STOP;
 330
 331         for_each_cpu_mask(cpu, callmap) {
 332                 msg_queue = &per_cpu(ipi_msg_queue, cpu);
 333                 spin_lock_irqsave(&msg_queue->lock, flags);
 334                 list_add_tail(&msg->list, &msg_queue->head);
 335                 spin_unlock_irqrestore(&msg_queue->lock, flags);
 336                 platform_send_ipi_cpu(cpu);
 337         }
 338         return;
 339 }
 340
 341 int __cpuinit __cpu_up(unsigned int cpu)
 342 {
 343         int ret;
 344         static struct task_struct *idle;
 345
 346         if (idle)
 347                 free_task(idle);
 348
 349         idle = fork_idle(cpu);
 350         if (IS_ERR(idle)) {
 351                 printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
 352                 return PTR_ERR(idle);
 353         }
 354
 355         secondary_stack = task_stack_page(idle) + THREAD_SIZE;
 356
 357         ret = platform_boot_secondary(cpu, idle);
 358
 359         secondary_stack = NULL;
 360
 361         return ret;
 362 }
 363
 364 static void __cpuinit setup_secondary(unsigned int cpu)
 365 {
 366         unsigned long ilat;
 367
 368         bfin_write_IMASK(0);
 369         CSYNC();
 370         ilat = bfin_read_ILAT();
 371         CSYNC();
 372         bfin_write_ILAT(ilat);
 373         CSYNC();
 374
 375         /* Enable interrupt levels IVG7-15. IARs have been already
 376          * programmed by the boot CPU.  */
 377         bfin_irq_flags |= IMASK_IVG15 |
 378             IMASK_IVG14 | IMASK_IVG13 | IMASK_IVG12 | IMASK_IVG11 |
 379             IMASK_IVG10 | IMASK_IVG9 | IMASK_IVG8 | IMASK_IVG7 | IMASK_IVGHW;
 380 }
 381
 382 void __cpuinit secondary_start_kernel(void)
 383 {
 384         unsigned int cpu = smp_processor_id();
 385         struct mm_struct *mm = &init_mm;
 386
 387         if (_bfin_swrst & SWRST_DBL_FAULT_B) {
 388                 printk(KERN_EMERG "CoreB Recovering from DOUBLE FAULT event\n");
 389 #ifdef CONFIG_DEBUG_DOUBLEFAULT
 390                 printk(KERN_EMERG " While handling exception (EXCAUSE = 0x%x) at %pF\n",
 391                         (int)init_saved_seqstat_coreb & SEQSTAT_EXCAUSE, init_saved_retx_coreb);
 392                 printk(KERN_NOTICE "   DCPLB_FAULT_ADDR: %pF\n", init_saved_dcplb_fault_addr_coreb);
 393                 printk(KERN_NOTICE "   ICPLB_FAULT_ADDR: %pF\n", init_saved_icplb_fault_addr_coreb);
 394 #endif
 395                 printk(KERN_NOTICE " The instruction at %pF caused a double exception\n",
 396                         init_retx_coreb);
 397         }
 398
 399         /*
 400          * We want the D-cache to be enabled early, in case the atomic
 401          * support code emulates cache coherence (see
 402          * __ARCH_SYNC_CORE_DCACHE).
 403          */
 404         init_exception_vectors();
 405
 406         bfin_setup_caches(cpu);
 407
 408         local_irq_disable();
 409
 410         /* Attach the new idle task to the global mm. */
 411         atomic_inc(&mm->mm_users);
 412         atomic_inc(&mm->mm_count);
 413         current->active_mm = mm;
 414
 415         preempt_disable();
 416
 417         setup_secondary(cpu);
 418
 419         platform_secondary_init(cpu);
 420
 421         /* setup local core timer */
 422         bfin_local_timer_setup();
 423
 424         local_irq_enable();
 425
 426         /*
 427          * Calibrate loops per jiffy value.
 428          * IRQs need to be enabled here - D-cache can be invalidated
 429          * in timer irq handler, so core B can read correct jiffies.
 430          */
 431         calibrate_delay();
 432
 433         cpu_idle();
 434 }
 435
 436 void __init smp_prepare_boot_cpu(void)
 437 {
 438 }
 439
 440 void __init smp_prepare_cpus(unsigned int max_cpus)
 441 {
 442         platform_prepare_cpus(max_cpus);
 443         ipi_queue_init();
 444         platform_request_ipi(ipi_handler);
 445 }
 446
 447 void __init smp_cpus_done(unsigned int max_cpus)
 448 {
 449         unsigned long bogosum = 0;
 450         unsigned int cpu;
 451
 452         for_each_online_cpu(cpu)
 453                 bogosum += loops_per_jiffy;
 454
 455         printk(KERN_INFO "SMP: Total of %d processors activated "
 456                "(%lu.%02lu BogoMIPS).\n",
 457                num_online_cpus(),
 458                bogosum / (500000/HZ),
 459                (bogosum / (5000/HZ)) % 100);
 460 }
 461
 462 void smp_icache_flush_range_others(unsigned long start, unsigned long end)
 463 {
 464         smp_flush_data.start = start;
 465         smp_flush_data.end = end;
 466
 467         if (smp_call_function(&ipi_flush_icache, &smp_flush_data, 0))
 468                 printk(KERN_WARNING "SMP: failed to run I-cache flush request on other CPUs\n");
 469 }
 470 EXPORT_SYMBOL_GPL(smp_icache_flush_range_others);
 471
 472 #ifdef __ARCH_SYNC_CORE_ICACHE
 473 unsigned long icache_invld_count[NR_CPUS];
 474 void resync_core_icache(void)
 475 {
 476         unsigned int cpu = get_cpu();
 477         blackfin_invalidate_entire_icache();
 478         icache_invld_count[cpu]++;
 479         put_cpu();
 480 }
 481 EXPORT_SYMBOL(resync_core_icache);
 482 #endif
 483
 484 #ifdef __ARCH_SYNC_CORE_DCACHE
 485 unsigned long dcache_invld_count[NR_CPUS];
 486 unsigned long barrier_mask __attribute__ ((__section__(".l2.bss")));
 487
 488 void resync_core_dcache(void)
 489 {
 490         unsigned int cpu = get_cpu();
 491         blackfin_invalidate_entire_dcache();
 492         dcache_invld_count[cpu]++;
 493         put_cpu();
 494 }
 495 EXPORT_SYMBOL(resync_core_dcache);
 496 #endif
 497
 498 #ifdef CONFIG_HOTPLUG_CPU
 499 int __cpuexit __cpu_disable(void)
 500 {
 501         unsigned int cpu = smp_processor_id();
 502
 503         if (cpu == 0)
 504                 return -EPERM;
 505
 506         set_cpu_online(cpu, false);
 507         return 0;
 508 }
 509
 510 static DECLARE_COMPLETION(cpu_killed);
 511
 512 int __cpuexit __cpu_die(unsigned int cpu)
 513 {
 514         return wait_for_completion_timeout(&cpu_killed, 5000);
 515 }
 516
 517 void cpu_die(void)
 518 {
 519         complete(&cpu_killed);
 520
 521         atomic_dec(&init_mm.mm_users);
 522         atomic_dec(&init_mm.mm_count);
 523
 524         local_irq_disable();
 525         platform_cpu_die();
 526 }
 527 #endif