src/cpu/x86/lapic/lapic_cpu_init.c

   1 /*
   2  * This file is part of the coreboot project.
   3  *
   4  * Copyright (C) 2001 Eric Biederman
   5  * Copyright (C) 2001 Ronald G. Minnich
   6  * Copyright (C) 2005 Yinghai Lu
   7  * Copyright (C) 2008 coresystems GmbH
   8  * Copyright (C) 2015 Timothy Pearson <tpearson@raptorengineeringinc.com>, Raptor Engineering
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; version 2 of the License.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  */
  19
  20 #include <cpu/x86/cr.h>
  21 #include <cpu/x86/gdt.h>
  22 #include <cpu/x86/lapic.h>
  23 #include <arch/acpi.h>
  24 #include <delay.h>
  25 #include <halt.h>
  26 #include <lib.h>
  27 #include <string.h>
  28 #include <symbols.h>
  29 #include <console/console.h>
  30 #include <device/device.h>
  31 #include <device/path.h>
  32 #include <smp/atomic.h>
  33 #include <smp/spinlock.h>
  34 #include <cpu/cpu.h>
  35 #include <cpu/intel/speedstep.h>
  36 #include <thread.h>
  37
  38 #if CONFIG_SMP && CONFIG_MAX_CPUS > 1
  39 /* This is a lot more paranoid now, since Linux can NOT handle
  40  * being told there is a CPU when none exists. So any errors
  41  * will return 0, meaning no CPU.
  42  *
  43  * We actually handling that case by noting which cpus startup
  44  * and not telling anyone about the ones that don't.
  45  */
  46
  47 /* Start-UP IPI vector must be 4kB aligned and below 1MB. */
  48 #define AP_SIPI_VECTOR 0x1000
  49
  50 static char *lowmem_backup;
  51 static char *lowmem_backup_ptr;
  52 static int  lowmem_backup_size;
  53
  54 static inline void setup_secondary_gdt(void)
  55 {
  56         u16 *gdt_limit;
  57 #ifdef __x86_64__
  58         u64 *gdt_base;
  59 #else
  60         u32 *gdt_base;
  61 #endif
  62
  63         gdt_limit = (void *)&_secondary_gdt_addr;
  64         gdt_base = (void *)&gdt_limit[1];
  65
  66         *gdt_limit = (uintptr_t)&gdt_end - (uintptr_t)&gdt - 1;
  67         *gdt_base = (uintptr_t)&gdt;
  68 }
  69
  70 static void copy_secondary_start_to_lowest_1M(void)
  71 {
  72         unsigned long code_size;
  73
  74         /* Fill in secondary_start's local gdt. */
  75         setup_secondary_gdt();
  76
  77         code_size = (unsigned long)_secondary_start_end - (unsigned long)_secondary_start;
  78
  79         if (acpi_is_wakeup_s3()) {
  80                 /* need to save it for RAM resume */
  81                 lowmem_backup_size = code_size;
  82                 lowmem_backup = malloc(code_size);
  83                 lowmem_backup_ptr = (char *)AP_SIPI_VECTOR;
  84
  85                 if (lowmem_backup == NULL)
  86                         die("Out of backup memory\n");
  87
  88                 memcpy(lowmem_backup, lowmem_backup_ptr, lowmem_backup_size);
  89         }
  90
  91         /* copy the _secondary_start to the ram below 1M*/
  92         memcpy((unsigned char *)AP_SIPI_VECTOR, (unsigned char *)_secondary_start, code_size);
  93
  94         printk(BIOS_DEBUG, "start_eip=0x%08lx, code_size=0x%08lx\n",
  95                 (long unsigned int)AP_SIPI_VECTOR, code_size);
  96 }
  97
  98 static void recover_lowest_1M(void)
  99 {
 100         if (acpi_is_wakeup_s3())
 101                 memcpy(lowmem_backup_ptr, lowmem_backup, lowmem_backup_size);
 102 }
 103
 104 static int lapic_start_cpu(unsigned long apicid)
 105 {
 106         int timeout;
 107         unsigned long send_status, accept_status;
 108         int j, maxlvt;
 109
 110         /*
 111          * Starting actual IPI sequence...
 112          */
 113
 114         printk(BIOS_SPEW, "Asserting INIT.\n");
 115
 116         /*
 117          * Turn INIT on target chip
 118          */
 119         lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(apicid));
 120
 121         /*
 122          * Send IPI
 123          */
 124
 125         lapic_write_around(LAPIC_ICR, LAPIC_INT_LEVELTRIG | LAPIC_INT_ASSERT
 126                                 | LAPIC_DM_INIT);
 127
 128         printk(BIOS_SPEW, "Waiting for send to finish...\n");
 129         timeout = 0;
 130         do {
 131                 printk(BIOS_SPEW, "+");
 132                 udelay(100);
 133                 send_status = lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY;
 134         } while (send_status && (timeout++ < 1000));
 135         if (timeout >= 1000) {
 136                 printk(BIOS_ERR, "CPU %ld: First APIC write timed out. "
 137                         "Disabling\n", apicid);
 138                 // too bad.
 139                 printk(BIOS_ERR, "ESR is 0x%lx\n", lapic_read(LAPIC_ESR));
 140                 if (lapic_read(LAPIC_ESR)) {
 141                         printk(BIOS_ERR, "Try to reset ESR\n");
 142                         lapic_write_around(LAPIC_ESR, 0);
 143                         printk(BIOS_ERR, "ESR is 0x%lx\n",
 144                                 lapic_read(LAPIC_ESR));
 145                 }
 146                 return 0;
 147         }
 148 #if !CONFIG_CPU_AMD_MODEL_10XXX && !CONFIG_CPU_INTEL_MODEL_206AX && !CONFIG_CPU_INTEL_MODEL_2065X
 149         mdelay(10);
 150 #endif
 151
 152         printk(BIOS_SPEW, "Deasserting INIT.\n");
 153
 154         /* Target chip */
 155         lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(apicid));
 156
 157         /* Send IPI */
 158         lapic_write_around(LAPIC_ICR, LAPIC_INT_LEVELTRIG | LAPIC_DM_INIT);
 159
 160         printk(BIOS_SPEW, "Waiting for send to finish...\n");
 161         timeout = 0;
 162         do {
 163                 printk(BIOS_SPEW, "+");
 164                 udelay(100);
 165                 send_status = lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY;
 166         } while (send_status && (timeout++ < 1000));
 167         if (timeout >= 1000) {
 168                 printk(BIOS_ERR, "CPU %ld: Second apic write timed out. "
 169                         "Disabling\n", apicid);
 170                 // too bad.
 171                 return 0;
 172         }
 173
 174         /*
 175          * Run STARTUP IPI loop.
 176          */
 177         printk(BIOS_SPEW, "#startup loops: %d.\n", CONFIG_NUM_IPI_STARTS);
 178
 179         maxlvt = 4;
 180
 181         for (j = 1; j <= CONFIG_NUM_IPI_STARTS; j++) {
 182                 printk(BIOS_SPEW, "Sending STARTUP #%d to %lu.\n", j, apicid);
 183                 lapic_read_around(LAPIC_SPIV);
 184                 lapic_write(LAPIC_ESR, 0);
 185                 lapic_read(LAPIC_ESR);
 186                 printk(BIOS_SPEW, "After apic_write.\n");
 187
 188                 /*
 189                  * STARTUP IPI
 190                  */
 191
 192                 /* Target chip */
 193                 lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(apicid));
 194
 195                 /* Boot on the stack */
 196                 /* Kick the second */
 197                 lapic_write_around(LAPIC_ICR, LAPIC_DM_STARTUP
 198                                         | (AP_SIPI_VECTOR >> 12));
 199
 200                 /*
 201                  * Give the other CPU some time to accept the IPI.
 202                  */
 203                 udelay(300);
 204
 205                 printk(BIOS_SPEW, "Startup point 1.\n");
 206
 207                 printk(BIOS_SPEW, "Waiting for send to finish...\n");
 208                 timeout = 0;
 209                 do {
 210                         printk(BIOS_SPEW, "+");
 211                         udelay(100);
 212                         send_status = lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY;
 213                 } while (send_status && (timeout++ < 1000));
 214
 215                 /*
 216                  * Give the other CPU some time to accept the IPI.
 217                  */
 218                 udelay(200);
 219                 /*
 220                  * Due to the Pentium erratum 3AP.
 221                  */
 222                 if (maxlvt > 3) {
 223                         lapic_read_around(LAPIC_SPIV);
 224                         lapic_write(LAPIC_ESR, 0);
 225                 }
 226                 accept_status = (lapic_read(LAPIC_ESR) & 0xEF);
 227                 if (send_status || accept_status)
 228                         break;
 229         }
 230         printk(BIOS_SPEW, "After Startup.\n");
 231         if (send_status)
 232                 printk(BIOS_WARNING, "APIC never delivered???\n");
 233         if (accept_status)
 234                 printk(BIOS_WARNING, "APIC delivery error (%lx).\n",
 235                         accept_status);
 236         if (send_status || accept_status)
 237                 return 0;
 238         return 1;
 239 }
 240
 241 /* Number of cpus that are currently running in coreboot */
 242 static atomic_t active_cpus = ATOMIC_INIT(1);
 243
 244 /* start_cpu_lock covers last_cpu_index and secondary_stack.
 245  * Only starting one cpu at a time let's me remove the logic
 246  * for select the stack from assembly language.
 247  *
 248  * In addition communicating by variables to the cpu I
 249  * am starting allows me to verify it has started before
 250  * start_cpu returns.
 251  */
 252
 253 static spinlock_t start_cpu_lock = SPIN_LOCK_UNLOCKED;
 254 static unsigned int last_cpu_index = 0;
 255 static void *stacks[CONFIG_MAX_CPUS];
 256 volatile unsigned long secondary_stack;
 257 volatile unsigned int secondary_cpu_index;
 258
 259 int start_cpu(struct device *cpu)
 260 {
 261         struct cpu_info *info;
 262         unsigned long stack_end;
 263         unsigned long stack_base;
 264         unsigned long *stack;
 265         unsigned long apicid;
 266         unsigned int index;
 267         unsigned long count;
 268         int i;
 269         int result;
 270
 271         spin_lock(&start_cpu_lock);
 272
 273         /* Get the CPU's apicid */
 274         apicid = cpu->path.apic.apic_id;
 275
 276         /* Get an index for the new processor */
 277         index = ++last_cpu_index;
 278
 279         /* Find end of the new processor's stack */
 280         stack_end = ((unsigned long)_estack) - (CONFIG_STACK_SIZE*index) -
 281                         sizeof(struct cpu_info);
 282
 283         stack_base = ((unsigned long)_estack) - (CONFIG_STACK_SIZE*(index+1));
 284         printk(BIOS_SPEW, "CPU%d: stack_base %p, stack_end %p\n", index,
 285                 (void *)stack_base, (void *)stack_end);
 286         /* poison the stack */
 287         for(stack = (void *)stack_base, i = 0; i < CONFIG_STACK_SIZE; i++)
 288                 stack[i/sizeof(*stack)] = 0xDEADBEEF;
 289         stacks[index] = stack;
 290         /* Record the index and which CPU structure we are using */
 291         info = (struct cpu_info *)stack_end;
 292         info->index = index;
 293         info->cpu   = cpu;
 294         thread_init_cpu_info_non_bsp(info);
 295
 296         /* Advertise the new stack and index to start_cpu */
 297         secondary_stack = stack_end;
 298         secondary_cpu_index = index;
 299
 300         /* Until the CPU starts up report the CPU is not enabled */
 301         cpu->enabled = 0;
 302         cpu->initialized = 0;
 303
 304         /* Start the cpu */
 305         result = lapic_start_cpu(apicid);
 306
 307         if (result) {
 308                 result = 0;
 309                 /* Wait 1s or until the new cpu calls in */
 310                 for(count = 0; count < 100000 ; count++) {
 311                         if (secondary_stack == 0) {
 312                                 result = 1;
 313                                 break;
 314                         }
 315                         udelay(10);
 316                 }
 317         }
 318         secondary_stack = 0;
 319         spin_unlock(&start_cpu_lock);
 320         return result;
 321 }
 322
 323 #if CONFIG_AP_IN_SIPI_WAIT
 324
 325 /**
 326  * Sending INIT IPI to self is equivalent of asserting #INIT with a bit of
 327  * delay.
 328  * An undefined number of instruction cycles will complete. All global locks
 329  * must be released before INIT IPI and no printk is allowed after this.
 330  * De-asserting INIT IPI is a no-op on later Intel CPUs.
 331  *
 332  * If you set DEBUG_HALT_SELF to 1, printk's after INIT IPI are enabled
 333  * but running thread may halt without releasing the lock and effectively
 334  * deadlock other CPUs.
 335  */
 336 #define DEBUG_HALT_SELF 0
 337
 338 /**
 339  * Normally this function is defined in lapic.h as an always inline function
 340  * that just keeps the CPU in a hlt() loop. This does not work on all CPUs.
 341  * I think all hyperthreading CPUs might need this version, but I could only
 342  * verify this on the Intel Core Duo
 343  */
 344 void stop_this_cpu(void)
 345 {
 346         int timeout;
 347         unsigned long send_status;
 348         unsigned long id;
 349
 350         id = lapic_read(LAPIC_ID) >> 24;
 351
 352         printk(BIOS_DEBUG, "CPU %ld going down...\n", id);
 353
 354         /* send an LAPIC INIT to myself */
 355         lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(id));
 356         lapic_write_around(LAPIC_ICR, LAPIC_INT_LEVELTRIG |
 357                                 LAPIC_INT_ASSERT | LAPIC_DM_INIT);
 358
 359         /* wait for the ipi send to finish */
 360 #if DEBUG_HALT_SELF
 361         printk(BIOS_SPEW, "Waiting for send to finish...\n");
 362 #endif
 363         timeout = 0;
 364         do {
 365 #if DEBUG_HALT_SELF
 366                 printk(BIOS_SPEW, "+");
 367 #endif
 368                 udelay(100);
 369                 send_status = lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY;
 370         } while (send_status && (timeout++ < 1000));
 371         if (timeout >= 1000) {
 372 #if DEBUG_HALT_SELF
 373                 printk(BIOS_ERR, "timed out\n");
 374 #endif
 375         }
 376         mdelay(10);
 377
 378 #if DEBUG_HALT_SELF
 379         printk(BIOS_SPEW, "Deasserting INIT.\n");
 380 #endif
 381         /* Deassert the LAPIC INIT */
 382         lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(id));
 383         lapic_write_around(LAPIC_ICR, LAPIC_INT_LEVELTRIG | LAPIC_DM_INIT);
 384
 385 #if DEBUG_HALT_SELF
 386         printk(BIOS_SPEW, "Waiting for send to finish...\n");
 387 #endif
 388         timeout = 0;
 389         do {
 390 #if DEBUG_HALT_SELF
 391                 printk(BIOS_SPEW, "+");
 392 #endif
 393                 udelay(100);
 394                 send_status = lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY;
 395         } while (send_status && (timeout++ < 1000));
 396         if (timeout >= 1000) {
 397 #if DEBUG_HALT_SELF
 398                 printk(BIOS_ERR, "timed out\n");
 399 #endif
 400         }
 401
 402         halt();
 403 }
 404 #endif
 405
 406 /* C entry point of secondary cpus */
 407 void asmlinkage secondary_cpu_init(unsigned int index)
 408 {
 409         atomic_inc(&active_cpus);
 410
 411         if (!IS_ENABLED(CONFIG_PARALLEL_CPU_INIT))
 412                 spin_lock(&start_cpu_lock);
 413
 414 #ifdef __SSE3__
 415         /*
 416          * Seems that CR4 was cleared when AP start via lapic_start_cpu()
 417          * Turn on CR4.OSFXSR and CR4.OSXMMEXCPT when SSE options enabled
 418          */
 419         u32 cr4_val;
 420         cr4_val = read_cr4();
 421         cr4_val |= (CR4_OSFXSR | CR4_OSXMMEXCPT);
 422         write_cr4(cr4_val);
 423 #endif
 424         cpu_initialize(index);
 425
 426         if (!IS_ENABLED(CONFIG_PARALLEL_CPU_INIT))
 427                 spin_unlock(&start_cpu_lock);
 428
 429         atomic_dec(&active_cpus);
 430
 431         stop_this_cpu();
 432 }
 433
 434 static void start_other_cpus(struct bus *cpu_bus, struct device *bsp_cpu)
 435 {
 436         struct device *cpu;
 437         /* Loop through the cpus once getting them started */
 438
 439         for(cpu = cpu_bus->children; cpu ; cpu = cpu->sibling) {
 440                 if (cpu->path.type != DEVICE_PATH_APIC) {
 441                         continue;
 442                 }
 443
 444                 if (IS_ENABLED(CONFIG_PARALLEL_CPU_INIT) && (cpu==bsp_cpu))
 445                         continue;
 446
 447                 if (!cpu->enabled) {
 448                         continue;
 449                 }
 450
 451                 if (cpu->initialized) {
 452                         continue;
 453                 }
 454
 455                 if (!start_cpu(cpu)) {
 456                         /* Record the error in cpu? */
 457                         printk(BIOS_ERR, "CPU 0x%02x would not start!\n",
 458                                 cpu->path.apic.apic_id);
 459                 }
 460
 461                 if (!IS_ENABLED(CONFIG_PARALLEL_CPU_INIT))
 462                         udelay(10);
 463         }
 464
 465 }
 466
 467 static void smm_other_cpus(struct bus *cpu_bus, device_t bsp_cpu)
 468 {
 469         device_t cpu;
 470         int pre_count = atomic_read(&active_cpus);
 471
 472         /* Loop through the cpus once to let them run through SMM relocator */
 473
 474         for(cpu = cpu_bus->children; cpu ; cpu = cpu->sibling) {
 475                 if (cpu->path.type != DEVICE_PATH_APIC) {
 476                         continue;
 477                 }
 478
 479                 printk(BIOS_ERR, "considering CPU 0x%02x for SMM init\n",
 480                         cpu->path.apic.apic_id);
 481
 482                 if (cpu == bsp_cpu)
 483                         continue;
 484
 485                 if (!cpu->enabled) {
 486                         continue;
 487                 }
 488
 489                 if (!start_cpu(cpu)) {
 490                         /* Record the error in cpu? */
 491                         printk(BIOS_ERR, "CPU 0x%02x would not start!\n",
 492                                 cpu->path.apic.apic_id);
 493                 }
 494
 495                 /* FIXME: endless loop */
 496                 while (atomic_read(&active_cpus) != pre_count) ;
 497         }
 498 }
 499
 500 static void wait_other_cpus_stop(struct bus *cpu_bus)
 501 {
 502         struct device *cpu;
 503         int old_active_count, active_count;
 504         long loopcount = 0;
 505         int i;
 506
 507         /* Now loop until the other cpus have finished initializing */
 508         old_active_count = 1;
 509         active_count = atomic_read(&active_cpus);
 510         while(active_count > 1) {
 511                 if (active_count != old_active_count) {
 512                         printk(BIOS_INFO, "Waiting for %d CPUS to stop\n",
 513                                 active_count - 1);
 514                         old_active_count = active_count;
 515                 }
 516                 udelay(10);
 517                 active_count = atomic_read(&active_cpus);
 518                 loopcount++;
 519         }
 520         for(cpu = cpu_bus->children; cpu; cpu = cpu->sibling) {
 521                 if (cpu->path.type != DEVICE_PATH_APIC) {
 522                         continue;
 523                 }
 524                 if (cpu->path.apic.apic_id == SPEEDSTEP_APIC_MAGIC) {
 525                         continue;
 526                 }
 527                 if (!cpu->initialized) {
 528                         printk(BIOS_ERR, "CPU 0x%02x did not initialize!\n",
 529                                 cpu->path.apic.apic_id);
 530                 }
 531         }
 532         printk(BIOS_DEBUG, "All AP CPUs stopped (%ld loops)\n", loopcount);
 533         checkstack(_estack, 0);
 534         for(i = 1; i <= last_cpu_index; i++)
 535                 checkstack((void *)stacks[i] + CONFIG_STACK_SIZE, i);
 536 }
 537
 538 #endif /* CONFIG_SMP */
 539
 540 void initialize_cpus(struct bus *cpu_bus)
 541 {
 542         struct device_path cpu_path;
 543         struct cpu_info *info;
 544
 545         /* Find the info struct for this cpu */
 546         info = cpu_info();
 547
 548 #if NEED_LAPIC == 1
 549         /* Ensure the local apic is enabled */
 550         enable_lapic();
 551
 552         /* Get the device path of the boot cpu */
 553         cpu_path.type           = DEVICE_PATH_APIC;
 554         cpu_path.apic.apic_id = lapicid();
 555 #else
 556         /* Get the device path of the boot cpu */
 557         cpu_path.type           = DEVICE_PATH_CPU;
 558         cpu_path.cpu.id       = 0;
 559 #endif
 560
 561         /* Find the device structure for the boot cpu */
 562         info->cpu = alloc_find_dev(cpu_bus, &cpu_path);
 563
 564 #if CONFIG_SMP && CONFIG_MAX_CPUS > 1
 565         // why here? In case some day we can start core1 in amd_sibling_init
 566         copy_secondary_start_to_lowest_1M();
 567 #endif
 568
 569 #if CONFIG_HAVE_SMI_HANDLER
 570         if (!IS_ENABLED(CONFIG_SERIALIZED_SMM_INITIALIZATION))
 571                 smm_init();
 572 #endif
 573
 574 #if CONFIG_SMP && CONFIG_MAX_CPUS > 1
 575         /* start all aps at first, so we can init ECC all together */
 576         if (IS_ENABLED(CONFIG_PARALLEL_CPU_INIT))
 577                 start_other_cpus(cpu_bus, info->cpu);
 578 #endif
 579
 580         /* Initialize the bootstrap processor */
 581         cpu_initialize(0);
 582
 583 #if CONFIG_SMP && CONFIG_MAX_CPUS > 1
 584         if (!IS_ENABLED(CONFIG_PARALLEL_CPU_INIT))
 585                 start_other_cpus(cpu_bus, info->cpu);
 586
 587         /* Now wait the rest of the cpus stop*/
 588         wait_other_cpus_stop(cpu_bus);
 589 #endif
 590
 591         if (IS_ENABLED(CONFIG_SERIALIZED_SMM_INITIALIZATION)) {
 592                 /* At this point, all APs are sleeping:
 593                  * smm_init() will queue a pending SMI on all cpus
 594                  * and smm_other_cpus() will start them one by one */
 595                 smm_init();
 596 #if CONFIG_SMP && CONFIG_MAX_CPUS > 1
 597                 last_cpu_index = 0;
 598                 smm_other_cpus(cpu_bus, info->cpu);
 599 #endif
 600         }
 601
 602 #if CONFIG_SMP && CONFIG_MAX_CPUS > 1
 603         recover_lowest_1M();
 604 #endif
 605 }