arch/all-pc/kernel/apic_ia32.c

   1 /*
   2     Copyright © 1995-2017, The AROS Development Team. All rights reserved.
   3     $Id$
   4
   5     Desc: Intel IA-32 APIC driver.
   6 */
   7
   8 #include <aros/macros.h>
   9 #include <asm/cpu.h>
  10 #include <asm/io.h>
  11 #include <exec/types.h>
  12
  13 #define __KERNEL_NOLIBBASE__
  14 #include <proto/kernel.h>
  15
  16 #include <proto/exec.h>
  17
  18 #include <inttypes.h>
  19
  20 #include "kernel_base.h"
  21 #include "kernel_intern.h"
  22 #include "kernel_objects.h"
  23 #include "kernel_debug.h"
  24 #include "kernel_syscall.h"
  25 #include "kernel_timer.h"
  26
  27 #include "kernel_interrupts.h"
  28
  29 #include "apic_ia32.h"
  30
  31 #define D(x)
  32 #define DINT(x)
  33 #define DWAKE(x)        /* Badly interferes with AP startup */
  34 #define DID(x)          /* Badly interferes with everything */
  35 /* #define DEBUG_WAIT */
  36
  37 /*
  38  * On i386 platform we need to support various quirks of old APICs.
  39  * x86-64 is free of that crap.
  40  */
  41 #ifdef __i386__
  42 #define CONFIG_LEGACY
  43 #endif
  44
  45 extern void core_APICErrorHandle(struct ExceptionContext *, void *, void *);
  46 AROS_INTP(APICHeartbeatServer);
  47
  48 /* APIC Interrupt Controller Functions ... ***************************/
  49
  50 struct APICInt_Private
  51 {
  52
  53 };
  54
  55 icid_t APICInt_Register(struct KernelBase *KernelBase)
  56 {
  57     DINT(bug("[Kernel:APIC-IA32] %s()\n", __func__));
  58
  59     return (icid_t)APICInt_IntrController.ic_Node.ln_Type;
  60 }
  61
  62 BOOL APICInt_Init(struct KernelBase *KernelBase, icid_t instanceCount)
  63 {
  64     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
  65     struct APICData *apicPrivate = kernPlatD->kb_APIC;
  66     APTR ssp;
  67     int irq, count = 0;
  68
  69     DINT(bug("[Kernel:APIC-IA32] %s(%d)\n", __func__, instanceCount));
  70
  71     /* its not fatal to fail on these irqs... */
  72     if ((ssp = SuperState()) != NULL)
  73     {
  74         /* Setup the APIC IRQs for CPU #0*/
  75         for (irq = (APIC_IRQ_BASE - X86_CPU_EXCEPT_COUNT); irq < ((APIC_IRQ_BASE - X86_CPU_EXCEPT_COUNT) + APIC_IRQ_COUNT); irq++)
  76         {
  77             if (((APIC_IRQ_HEARTBEAT - HW_IRQ_BASE) == irq) && (apicPrivate->flags & APF_TIMER))
  78             {
  79                 /* if we have the heartbeat timer, its time to enable it for the boot processor... */
  80                 ictl_enable_irq((APIC_IRQ_HEARTBEAT - HW_IRQ_BASE), KernelBase);
  81             }
  82             else
  83             {
  84                 if (!krnInitInterrupt(KernelBase, irq, APICInt_IntrController.ic_Node.ln_Type, 0))
  85                 {
  86                     D(bug("[Kernel:APIC-IA32] %s: failed to obtain IRQ %d\n", __func__, irq);)
  87                 }
  88                 else
  89                 {
  90                     /* dont enable the vector yet...*/
  91                     if (!core_SetIDTGate((apicidt_t *)apicPrivate->cores[0].cpu_IDT, HW_IRQ_BASE + irq, (uintptr_t)IntrDefaultGates[HW_IRQ_BASE + irq], FALSE))
  92                     {
  93                         bug("[Kernel:APIC-IA32] %s: failed to set IRQ %d's Vector gate\n", __func__, irq);
  94                     }
  95                     else
  96                         count++;
  97                 }
  98             }
  99         }
 100         UserState(ssp);
 101     }
 102
 103     /*
 104      * If we have atleast 10 APIC interrupts available,
 105      * then report that we can use MSI
 106      */
 107     if ((count > 10) && (!(kernPlatD->kb_PDFlags & PLATFORMF_HAVEMSI)))
 108         kernPlatD->kb_PDFlags |= PLATFORMF_HAVEMSI;
 109
 110     return TRUE;
 111 }
 112
 113 BOOL APICInt_DisableIRQ(APTR icPrivate, icid_t icInstance, icid_t intNum)
 114 {
 115     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
 116     struct APICData *apicPrivate = kernPlatD->kb_APIC;
 117     apicid_t cpuNum = KrnGetCPUNumber();
 118     apicidt_t *IGATES;
 119     APTR ssp = NULL;
 120     BOOL retVal = FALSE;
 121
 122     DINT(bug("[Kernel:APIC-IA32.%03u] %s(#$%02X)\n", cpuNum, __func__, intNum));
 123
 124     IGATES = (apicidt_t *)apicPrivate->cores[cpuNum].cpu_IDT;
 125
 126     if ((KrnIsSuper()) || ((ssp = SuperState()) != NULL))
 127     {
 128         IGATES[HW_IRQ_BASE + intNum].p = 0;
 129         retVal = TRUE;
 130
 131         if (ssp)
 132             UserState(ssp);
 133     }
 134
 135     return retVal;
 136 }
 137
 138 BOOL APICInt_EnableIRQ(APTR icPrivate, icid_t icInstance, icid_t intNum)
 139 {
 140     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
 141     struct APICData *apicPrivate = kernPlatD->kb_APIC;
 142     apicid_t cpuNum = KrnGetCPUNumber();
 143     apicidt_t *IGATES;
 144     APTR ssp = NULL;
 145     BOOL retVal = FALSE;
 146
 147     DINT(bug("[Kernel:APIC-IA32.%03u] %s(#$%02X)\n", cpuNum, __func__, intNum));
 148
 149     IGATES = (apicidt_t *)apicPrivate->cores[cpuNum].cpu_IDT;
 150
 151     if ((KrnIsSuper()) || ((ssp = SuperState()) != NULL))
 152     {
 153         IGATES[HW_IRQ_BASE + intNum].p = 1;
 154         retVal = TRUE;
 155
 156         if (ssp)
 157             UserState(ssp);
 158     }
 159
 160     return retVal;
 161 }
 162
 163 BOOL APICInt_AckIntr(APTR icPrivate, icid_t icInstance, icid_t intNum)
 164 {
 165     IPTR apic_base;
 166
 167     DINT(bug("[Kernel:APIC-IA32] %s(%03u #$%02X)\n", __func__, icInstance, intNum));
 168
 169     /* Write zero to EOI of APIC */
 170     apic_base = core_APIC_GetBase();
 171
 172     APIC_REG(apic_base, APIC_EOI) = 0;
 173
 174     return TRUE;
 175 }
 176
 177 struct IntrController APICInt_IntrController =
 178 {
 179     {
 180         .ln_Name = "x86 Local APIC",
 181         .ln_Pri = -50
 182     },
 183     0,
 184     AROS_MAKE_ID('A','P','I','C'),
 185     0,
 186     NULL,
 187     APICInt_Register,
 188     APICInt_Init,
 189     APICInt_EnableIRQ,
 190     APICInt_DisableIRQ,
 191     APICInt_AckIntr
 192 };
 193
 194 /* APIC IPI Related Functions ... ***************************/
 195
 196 static ULONG DoIPI(IPTR __APICBase, ULONG target, ULONG cmd)
 197 {
 198     ULONG ipisend_timeout, status_ipisend;
 199
 200     D(
 201         apicid_t cpuNum = KrnGetCPUNumber();
 202         bug("[Kernel:APIC-IA32.%03u] %s: Command 0x%08X to target %03u\n", cpuNum, __func__, cmd, target);
 203     )
 204
 205     /*
 206      * Send the IPI.
 207      * First we write target APIC ID into high command register.
 208      * Writing to the low register triggers the IPI itself.
 209      */
 210     APIC_REG(__APICBase, APIC_ICRH) = target << 24;
 211     APIC_REG(__APICBase, APIC_ICRL) = cmd;
 212
 213     D(bug("[Kernel:APIC-IA32.%03u] %s: Waiting for IPI to complete ", cpuNum, __func__));
 214
 215     for (ipisend_timeout = 1000; ipisend_timeout > 0; ipisend_timeout--)
 216     {
 217         pit_udelay(100);
 218 #ifdef DEBUG_WAIT
 219         if ((ipisend_timeout % 100) == 0)
 220         {
 221             bug(".");
 222         }
 223 #endif
 224         status_ipisend = APIC_REG(__APICBase, APIC_ICRL) & ICR_DS;
 225         /* Delivery status resets to 0 when delivery is done */
 226         if (status_ipisend == 0)
 227             break;
 228     }
 229     D(bug("\n"));
 230     D(bug("[Kernel:APIC-IA32.%03u] %s: ... left wait loop (status = 0x%08X)\n", cpuNum, __func__, status_ipisend));
 231
 232     return status_ipisend;
 233 }
 234
 235 /**********************************************************
 236                         Driver functions
 237  **********************************************************/
 238
 239 void core_APIC_Init(struct APICData *apic, apicid_t cpuNum)
 240 {
 241     IPTR __APICBase = apic->lapicBase;
 242     ULONG apic_ver = APIC_REG(__APICBase, APIC_VERSION);
 243     ULONG maxlvt = APIC_LVT(apic_ver), calibrated = 0;
 244     LONG lapic_initial, lapic_final;
 245     UQUAD tsc_initial, tsc_final;
 246     UQUAD calibrated_tsc = 0;
 247     WORD pit_final;
 248     icintrid_t coreICInstID;
 249
 250 #ifdef CONFIG_LEGACY
 251     /* 82489DX doesnt report no. of LVT entries. */
 252     if (!APIC_INTEGRATED(apic_ver))
 253         maxlvt = 2;
 254 #endif
 255
 256     if ((coreICInstID = krnAddInterruptController(KernelBase, &APICInt_IntrController)) != (icintrid_t)-1)
 257     {
 258         APTR ssp = NULL;
 259         int i;
 260
 261         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC IC ID #%d:%d\n", cpuNum, __func__, ICINTR_ICID(coreICInstID), ICINTR_INST(coreICInstID)));
 262
 263         /*
 264          * NB: - BSP calls us in user mode, but AP's call us from supervisor
 265          */
 266         if ((KrnIsSuper()) || ((ssp = SuperState()) != NULL))
 267         {
 268             /* Obtain/set the critical IRQs and Vectors */
 269             for (i = 0; i < X86_CPU_EXCEPT_COUNT; i++)
 270             {
 271                 if ((HW_IRQ_BASE < i) && (cpuNum == 0))
 272                 {
 273                     if (!krnInitInterrupt(KernelBase, (i - HW_IRQ_BASE), APICInt_IntrController.ic_Node.ln_Type, 0))
 274                     {
 275                         krnPanic(NULL, "Failed to obtain APIC Exception IRQ\n"
 276                                        "IRQ #$%02X\n", (i - HW_IRQ_BASE));
 277                     }
 278                     if (!core_SetIRQGate(apic->cores[cpuNum].cpu_IDT, (i - HW_IRQ_BASE), (uintptr_t)IntrDefaultGates[i]))
 279                     {
 280                         krnPanic(NULL, "Failed to set APIC Exception IRQ Vector\n"
 281                                        "IRQ #$%02X, Vector #$%02X\n", (i - HW_IRQ_BASE), i);
 282                     }
 283                 }
 284                 else if (!core_SetIDTGate((apicidt_t *)apic->cores[cpuNum].cpu_IDT, i, (uintptr_t)IntrDefaultGates[i], TRUE))
 285                 {
 286                     krnPanic(NULL, "Failed to set APIC Exception Vector\n"
 287                                    "Vector #$%02X\n", i);
 288                 }
 289             }
 290             D(bug("[Kernel:APIC-IA32.%03u] %s: APIC Exception Vectors configured\n", cpuNum, __func__));
 291
 292             if ((APIC_IRQ_ERROR < HW_IRQ_COUNT) && (cpuNum == 0))
 293             {
 294                 if (!krnInitInterrupt(KernelBase, (APIC_IRQ_ERROR - HW_IRQ_BASE), APICInt_IntrController.ic_Node.ln_Type, 0))
 295                 {
 296                     krnPanic(NULL, "Failed to obtain APIC Error IRQ\n"
 297                                    "IRQ #$%02X\n", (APIC_IRQ_ERROR - HW_IRQ_BASE));
 298                 }
 299                 if (!core_SetIRQGate(apic->cores[cpuNum].cpu_IDT, (APIC_IRQ_ERROR - HW_IRQ_BASE), (uintptr_t)IntrDefaultGates[APIC_IRQ_ERROR]))
 300                 {
 301                     krnPanic(NULL, "Failed to set APIC Error IRQ Vector\n"
 302                                    "IRQ #$%02X, Vector #$%02X\n", (APIC_IRQ_ERROR - HW_IRQ_BASE), APIC_IRQ_ERROR);
 303                 }
 304             }
 305             else if (!core_SetIDTGate((apicidt_t *)apic->cores[cpuNum].cpu_IDT, APIC_IRQ_ERROR, (uintptr_t)IntrDefaultGates[APIC_IRQ_ERROR], TRUE))
 306             {
 307                 krnPanic(NULL, "Failed to set APIC Error Vector\n"
 308                                "Vector #$%02X\n", APIC_IRQ_ERROR);
 309             }
 310             else if (cpuNum == 0)
 311                 KrnAddExceptionHandler((APIC_IRQ_ERROR - APIC_IRQ_COUNT), core_APICErrorHandle, NULL, NULL);
 312
 313             D(bug("[Kernel:APIC-IA32.%03u] %s: APIC Error Vector #$%02X configured\n", cpuNum, __func__, APIC_IRQ_ERROR));
 314
 315             for (i = APIC_IRQ_IPI_START; i <= APIC_IRQ_IPI_END; i++)
 316             {
 317                 if (!core_SetIDTGate((apicidt_t *)apic->cores[cpuNum].cpu_IDT, i, (uintptr_t)IntrDefaultGates[i], TRUE))
 318                 {
 319                     krnPanic(NULL, "Failed to set APIC IPI Vector\n"
 320                                 "Vector #$%02X\n", i);
 321                 }
 322             }
 323             D(bug("[Kernel::APIC-IA32.%03u] %s: APIC IPI Vectors configured\n", cpuNum, __func__));
 324
 325             if (ssp)
 326                 UserState(ssp);
 327         }
 328         else
 329         {
 330             krnPanic(NULL, "Failed to configure APIC\n"
 331                                "APIC #%03e ID %03u\n", cpuNum, apic->cores[cpuNum].cpu_LocalID);
 332         }
 333
 334         /* Use flat interrupt model with logical destination ID = 1 */
 335         APIC_REG(__APICBase, APIC_DFR) = DFR_FLAT;
 336         APIC_REG(__APICBase, APIC_LDR) = 1 << LDR_ID_SHIFT;
 337
 338         /* Set Task Priority to 'accept all interrupts' */
 339         APIC_REG(__APICBase, APIC_TPR) = 0;
 340
 341         /* Set spurious IRQ vector to 0xFF. APIC enabled, focus check disabled. */
 342         APIC_REG(__APICBase, APIC_SVR) = SVR_ASE|SVR_FCC|0xFF;
 343
 344         /*
 345          * Set LINT0 to external and LINT1 to NMI.
 346          * These are common defaults and they are going to be overriden by ACPI tables.
 347          */
 348         if (cpuNum == 0)
 349             APIC_REG(__APICBase, APIC_LINT0_VEC) = LVT_MT_EXT;
 350         else
 351             APIC_REG(__APICBase, APIC_LINT0_VEC) = LVT_MASK;
 352
 353         APIC_REG(__APICBase, APIC_LINT1_VEC) = LVT_MT_NMI;
 354
 355 #ifdef CONFIG_LEGACY
 356         /* Due to the Pentium erratum 3AP. */
 357         if (maxlvt > 3)
 358              APIC_REG(__APICBase, APIC_ESR) = 0;
 359 #endif
 360
 361         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC ESR before enabling vector: %08x\n", cpuNum, __func__, APIC_REG(__APICBase, APIC_ESR)));
 362
 363         /* Set APIC error interrupt to fixed vector interrupt "APIC_IRQ_ERROR",  on APIC error */
 364         APIC_REG(__APICBase, APIC_ERROR_VEC) = APIC_IRQ_ERROR;
 365
 366         /* spec says clear errors after enabling vector. */
 367         if (maxlvt > 3)
 368              APIC_REG(__APICBase, APIC_ESR) = 0;
 369
 370         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC ESR after enabling vector: %08x\n", cpuNum, __func__, APIC_REG(__APICBase, APIC_ESR)));
 371
 372         /*
 373          * Now the tricky thing - calibrate LAPIC timer frequency.
 374          * In fact we could simply query CPU's clock frequency, but... x86 sucks. There's no
 375          * unified way to get it on whatever CPU. Intel has own way, AMD has own way... Etc... Which, additionally,
 376          * varies between CPU generations.
 377          *
 378          * The idea behing the calibration is to run the timer once, and see how much ticks passes in some defined
 379          * period of time. Then calculate a proportion.
 380          * We use 8253 PIT as our reference.
 381          * This calibrarion algorighm is based on NetBSD one.
 382          */
 383
 384         /* Set the timer to one-shot mode, no interrupt, 1:1 divisor */
 385         APIC_REG(__APICBase, APIC_TIMER_VEC) = LVT_MASK;
 386         APIC_REG(__APICBase, APIC_TIMER_DIV) = TIMER_DIV_1;
 387         APIC_REG(__APICBase, APIC_TIMER_ICR) = 0x80000000;      /* Just some very large value */
 388
 389         /*
 390          * Now wait for 11931 PIT ticks, which is equal to 10 milliseconds.
 391          * We don't use pit_udelay() here, because for improved accuracy we need to sample LAPIC timer counter twice,
 392          * before and after our actual delay (PIT setup also takes up some time, so LAPIC will count away from its
 393          * initial value).  We run it 10 times to make up for cache setup discrepancies.
 394          */
 395         for (i = 0; i < 10; i ++)
 396         {
 397             pit_start(11931);
 398             lapic_initial = (LONG)APIC_REG(__APICBase, APIC_TIMER_CCR);
 399             tsc_initial = RDTSC();
 400
 401             pit_final   = pit_wait(11931);
 402
 403             tsc_final = RDTSC();
 404             lapic_final = (LONG)APIC_REG(__APICBase, APIC_TIMER_CCR);
 405
 406             calibrated += (((QUAD)(lapic_initial - lapic_final) * 11931LL)/(11931LL - (QUAD)pit_final)) ;
 407             calibrated_tsc += ((tsc_final - tsc_initial) * 11931LL) / (11931LL - (QUAD)pit_final);
 408         }
 409         apic->cores[cpuNum].cpu_TimerFreq = 10 * calibrated;
 410         apic->cores[cpuNum].cpu_TSCFreq = 10 * calibrated_tsc;
 411         D(bug("[Kernel:APIC-IA32.%03u] %s: LAPIC frequency should be %u Hz (%u MHz)\n", cpuNum, __func__, apic->cores[cpuNum].cpu_TimerFreq, (apic->cores[cpuNum].cpu_TimerFreq + 500000) / 1000000));
 412         D(bug("[Kernel:APIC-IA32.%03u] %s: TSC frequency should be %u kHz (%u MHz)\n", cpuNum, __func__, (ULONG)((apic->cores[cpuNum].cpu_TSCFreq + 500)/1000), (ULONG)((apic->cores[cpuNum].cpu_TSCFreq + 500000) / 1000000)));
 413         /*
 414          * Once APIC timer has been calibrated -:
 415          * # Set it to run at it's full frequency.
 416          * # Enable the heartbeat vector and use a suitable rate,
 417          *   otherwise set to reload every second and disable it.
 418          */
 419         if (cpuNum == 0)
 420         {
 421             if (krnInitInterrupt(KernelBase, (APIC_IRQ_HEARTBEAT - HW_IRQ_BASE), APICInt_IntrController.ic_Node.ln_Type, 0))
 422             {
 423                 struct IntrNode *hbHandle;
 424
 425                 hbHandle = krnAllocIntrNode();
 426                 D(bug("[Kernel:APIC-IA32.%03u] %s: heartbeat IRQ #$%02X (%d) handler @ 0x%p\n", cpuNum, __func__, (APIC_IRQ_HEARTBEAT - HW_IRQ_BASE), (APIC_IRQ_HEARTBEAT - HW_IRQ_BASE), hbHandle);)
 427
 428                 if (hbHandle)
 429                 {
 430                     hbHandle->in_Handler = APICHeartbeatServer;
 431                     hbHandle->in_HandlerData = SysBase;
 432                     hbHandle->in_HandlerData2 = NULL;
 433                     hbHandle->in_type = it_interrupt;
 434                     hbHandle->in_nr = APIC_IRQ_HEARTBEAT - HW_IRQ_BASE;
 435
 436                     Disable();
 437                     ADDHEAD(&KERNELIRQ_LIST(hbHandle->in_nr), &hbHandle->in_Node);
 438                     Enable();
 439
 440                     apic->flags |= APF_TIMER;
 441                 }
 442                 else
 443                 {
 444                     D(bug("[Kernel:APIC-IA32.%03u] %s: failed to allocate HeartBeat handler\n", cpuNum, __func__);)
 445                 }
 446             }
 447             else
 448             {
 449                 D(bug("[Kernel:APIC-IA32.%03u] %s: failed to obtain HeartBeat IRQ %d\n", cpuNum, __func__, (APIC_IRQ_HEARTBEAT - HW_IRQ_BASE));)
 450             }
 451         }
 452
 453         APIC_REG(__APICBase, APIC_TIMER_DIV) = TIMER_DIV_1;
 454
 455         if ((apic->flags & APF_TIMER) &&
 456             ((KrnIsSuper()) || ((ssp = SuperState()) != NULL)))
 457         {
 458 #if defined(__AROSEXEC_SMP__)
 459             tls_t *apicTLS = apic->cores[cpuNum].cpu_TLS;
 460             struct X86SchedulerPrivate *schedData = apicTLS->ScheduleData;
 461             D(bug("[Kernel:APIC-IA32.%03u] %s: tls @ 0x%p, scheduling data @ 0x%p\n", cpuNum, __func__, apicTLS, schedData);)
 462 #endif
 463
 464             if (!core_SetIDTGate(apic->cores[cpuNum].cpu_IDT, APIC_IRQ_HEARTBEAT, (uintptr_t)IntrDefaultGates[APIC_IRQ_HEARTBEAT], TRUE))
 465             {
 466                 krnPanic(NULL, "Failed to set APIC HeartBeat IRQ Vector\n"
 467                                "IRQ #$%02X, Vector #$%02X\n", (APIC_IRQ_HEARTBEAT - HW_IRQ_BASE), APIC_IRQ_HEARTBEAT);
 468             }
 469
 470             apic->cores[cpuNum].cpu_LAPICTick = 0;
 471             D(bug("[Kernel:APIC-IA32.%03u] %s: heartbeat IRQ Vector #$%02X (%d) set\n", cpuNum, __func__, APIC_IRQ_HEARTBEAT, APIC_IRQ_HEARTBEAT);)
 472
 473             if (ssp)
 474                 UserState(ssp);
 475
 476 #if defined(__AROSEXEC_SMP__)
 477             // TODO: Adjust based on the amount of work the APIC can do at its given frequency.
 478             schedData->Granularity = 1;
 479             schedData->Quantum = 5;
 480             APIC_REG(__APICBase, APIC_TIMER_ICR) = (apic->cores[cpuNum].cpu_TimerFreq);
 481 #else
 482             APIC_REG(__APICBase, APIC_TIMER_ICR) = (apic->cores[cpuNum].cpu_TimerFreq + 25) / 50;
 483 #endif
 484             APIC_REG(__APICBase, APIC_TIMER_VEC) = APIC_IRQ_HEARTBEAT; // | LVT_TMM_PERIOD;
 485
 486         }
 487         else
 488         {
 489             APIC_REG(__APICBase, APIC_TIMER_ICR) = apic->cores[cpuNum].cpu_TimerFreq;
 490             APIC_REG(__APICBase, APIC_TIMER_VEC) = LVT_MASK | LVT_TMM_PERIOD;
 491         }
 492     }
 493 }
 494
 495 apicid_t core_APIC_GetID(IPTR _APICBase)
 496 {
 497     apicid_t _apic_id;
 498
 499     /* The actual ID is in 8 most significant bits */
 500     _apic_id = APIC_REG(_APICBase, APIC_ID) >> APIC_ID_SHIFT;
 501     DID(bug("[Kernel:APIC-IA32] %s: %03u\n", __func__, _apic_id));
 502
 503     return _apic_id;
 504 }
 505
 506 ULONG core_APIC_Wake(APTR wake_apicstartrip, apicid_t wake_apicid, IPTR __APICBase)
 507 {
 508     ULONG status_ipisend, status_ipirecv;
 509     ULONG start_count;
 510 #ifdef CONFIG_LEGACY
 511     ULONG apic_ver = APIC_REG(__APICBase, APIC_VERSION);
 512 #endif
 513     D(
 514         apicid_t cpuNo = KrnGetCPUNumber();
 515
 516         bug("[Kernel:APIC-IA32.%03u] %s(%03u @ %p)\n", cpuNo, __func__, wake_apicid, wake_apicstartrip);
 517         bug("[Kernel:APIC-IA32.%03u] %s: Base @ %p\n", cpuNo, __func__, __APICBase);
 518     )
 519 #ifdef CONFIG_LEGACY
 520     /*
 521      * Check if we have old 82489DX discrete APIC (version & 0xF0 == 0).
 522      * This APIC needs different startup procedure. It doesn't support STARTUP IPI
 523      * because old CPUs didn't have INIT signal. They jump to BIOS ROM boot code
 524      * immediately after INIT IPI. In order to run the bootstrap, a BIOS warm reset
 525      * magic has to be used there.
 526      */
 527     if (!APIC_INTEGRATED(apic_ver))
 528     {
 529         /*
 530          * BIOS warm reset magic, part one.
 531          * Write real-mode bootstrap routine address to 40:67 (real-mode address) location.
 532          * This is standard feature of IBM PC AT BIOS. If a warm reset condition is detected,
 533          * the BIOS jumps to the given address.
 534          */
 535         D(bug("[Kernel:APIC-IA32.%03u] %s: Setting BIOS vector for trampoline @ %p ..\n", cpuNo, __func__, wake_apicstartrip));
 536         *((volatile unsigned short *)0x469) = (IPTR)wake_apicstartrip >> 4;
 537         *((volatile unsigned short *)0x467) = 0; /* Actually wake_apicstartrip & 0x0F, it's page-aligned. */
 538
 539         /*
 540          * BIOS warm reset magic, part two.
 541          * This writes 0x0A into CMOS RAM, location 0x0F. This signals a warm reset condition to BIOS,
 542          * making part one work.
 543          */
 544         D(bug("[Kernel:APIC-IA32.%03u] %s: Setting warm reset code ..\n", cpuNo, __func__));
 545         outb(0xf, 0x70);
 546         outb(0xa, 0x71);
 547     }
 548 #endif
 549
 550     /* Flush TLB (we are supervisor here) */
 551     wrcr(cr3, rdcr(cr3));
 552
 553     /* First we send the INIT command (reset the core). Vector must be zero for this. */
 554     status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_INT_LEVELTRIG | ICR_INT_ASSERT | ICR_DM_INIT);
 555     if (status_ipisend)
 556     {
 557         D(bug("[Kernel:APIC-IA32.%03u] %s: Error asserting INIT\n", cpuNo, __func__));
 558         return status_ipisend;
 559     }
 560
 561     /* Deassert INIT after a small delay */
 562     pit_udelay(10 * 1000);
 563
 564     /* Deassert INIT */
 565     status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_INT_LEVELTRIG | ICR_DM_INIT);
 566     if (status_ipisend)
 567     {
 568         D(bug("[Kernel:APIC-IA32.%03u] %s: Error deasserting INIT\n", cpuNo, __func__));
 569         return status_ipisend;
 570     }
 571
 572     /* memory barrier */
 573     asm volatile("mfence":::"memory");
 574
 575 #ifdef CONFIG_LEGACY
 576     /* If it's 82489DX, we are done. */
 577     if (!APIC_INTEGRATED(apic_ver))
 578     {
 579         DWAKE(bug("[Kernel:APIC-IA32.%03u] %s: 82489DX detected, wakeup done\n", cpuNo, __func__));
 580         return 0;
 581     }
 582 #endif
 583
 584     /*
 585      * Perform IPI STARTUP loop.
 586      * According to official Intel specification, this must be done twice.
 587      * It's not explained why. ;-)
 588      */
 589     for (start_count = 1; start_count <= 2; start_count++)
 590     {
 591         D(bug("[Kernel:APIC-IA32.%03u] %s: Attempting STARTUP .. %u\n", cpuNo, __func__, start_count));
 592
 593         /* Clear any pending error condition */
 594         APIC_REG(__APICBase, APIC_ESR) = 0;
 595
 596         /*
 597          * Send STARTUP IPI.
 598          * The processor starts up at CS = (vector << 16) and IP = 0.
 599          */
 600         status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_DM_STARTUP | ((IPTR)wake_apicstartrip >> 12));
 601
 602         /* Allow the target APIC to accept the IPI */
 603         pit_udelay(200);
 604
 605 #ifdef CONFIG_LEGACY
 606         /* Pentium erratum 3AP quirk */
 607         if (APIC_LVT(apic_ver) > 3)
 608             APIC_REG(__APICBase, APIC_ESR) = 0;
 609 #endif
 610
 611         status_ipirecv = APIC_REG(__APICBase, APIC_ESR) & 0xEF;
 612
 613         /*
 614          * EXPERIMENTAL:
 615          * On my machine (macmini 3,1, as OS X system profiler says), the core starts up from first
 616          * attempt. The second attempt ends up in error (according to the documentation, the STARTUP
 617          * can be accepted only once, while the core in RESET or INIT state, and first STARTUP, if
 618          * successful, brings the core out of this state).
 619          * Here we try to detect this condition. If the core accepted STARTUP, we suggest that it has
 620          * started up, and break the loop.
 621          * A topic at osdev.org forum (http://forum.osdev.org/viewtopic.php?f=1&t=23018)
 622          * also tells about some problems with double STARTUP. According to it, the second STARTUP can
 623          * manage to re-run the core from the given address, leaving it in 64-bit mode, causing it to crash.
 624          *
 625          * If startup problems pops up (the core doesn't respond and AROS halts at "Launching APIC no X" stage),
 626          * the following two variations of this algorithm can be tried:
 627          * a) Always send STARTUP twice, but signal error condition only if both attempts failed.
 628          * b) Send first STARTUP, abort on error. Allow second attempt to fail and ignore its result.
 629          *
 630          *                                                              Sonic <pavel_fedin@mail.ru>
 631          */
 632         if (!status_ipisend && !status_ipirecv)
 633             break;
 634     }
 635
 636     DWAKE(bug("[Kernel:APIC-IA32.%03u] %s: STARTUP run status 0x%08X, error 0x%08X\n", cpuNo, __func__, status_ipisend, status_ipirecv));
 637
 638     /*
 639      * We return nonzero on error.
 640      * Actually least significant byte of this value holds ESR value, and 12th bit
 641      * holds delivery status flag from DoIPI() routine. It will be '1' if we got
 642      * stuck at sending phase.
 643      */
 644     return (status_ipisend | status_ipirecv);
 645 }