arch/all-pc/kernel/apic_ia32.c

   1 /*
   2     Copyright � 1995-2017, The AROS Development Team. All rights reserved.
   3     $Id$
   4
   5     Desc: Intel IA-32 APIC driver.
   6 */
   7
   8 #include <aros/macros.h>
   9 #include <asm/cpu.h>
  10 #include <asm/io.h>
  11 #include <exec/types.h>
  12
  13 #define __KERNEL_NOLIBBASE__
  14 #include <proto/kernel.h>
  15
  16 #include <proto/exec.h>
  17
  18 #include <inttypes.h>
  19
  20 #include "kernel_base.h"
  21 #include "kernel_intern.h"
  22 #include "kernel_debug.h"
  23 #include "kernel_syscall.h"
  24 #include "kernel_timer.h"
  25
  26 #include "kernel_interrupts.h"
  27
  28 #include "apic_ia32.h"
  29
  30 #define D(x)
  31 #define DINT(x)
  32 #define DWAKE(x)        /* Badly interferes with AP startup */
  33 #define DID(x)          /* Badly interferes with everything */
  34 /* #define DEBUG_WAIT */
  35
  36 /*
  37  * On i386 platform we need to support various quirks of old APICs.
  38  * x86-64 is free of that crap.
  39  */
  40 #ifdef __i386__
  41 #define CONFIG_LEGACY
  42 #endif
  43
  44 extern void core_APICErrorHandle(struct ExceptionContext *, void *, void *);
  45
  46 /* APIC Interrupt Controller Functions ... ***************************/
  47
  48 struct APICInt_Private
  49 {
  50
  51 };
  52
  53 icid_t APICInt_Register(struct KernelBase *KernelBase)
  54 {
  55     DINT(bug("[Kernel:APIC-IA32] %s()\n", __func__));
  56
  57     return (icid_t)APICInt_IntrController.ic_Node.ln_Type;
  58 }
  59
  60 BOOL APICInt_Init(struct KernelBase *KernelBase, icid_t instanceCount)
  61 {
  62     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
  63     struct APICData *apicPrivate = kernPlatD->kb_APIC;
  64     APTR ssp;
  65     int irq;
  66
  67     DINT(bug("[Kernel:APIC-IA32] %s(%d)\n", __func__, instanceCount));
  68
  69     /* its not fatal to fail on these irqs... */
  70     if ((ssp = SuperState()) != NULL)
  71     {
  72         /* Setup the APIC IRQs for CPU #0*/
  73         for (irq = (APIC_IRQ_BASE - X86_CPU_EXCEPT_COUNT); irq < ((APIC_IRQ_BASE - X86_CPU_EXCEPT_COUNT) + APIC_IRQ_COUNT); irq++)
  74         {
  75             if (!krnInitInterrupt(KernelBase, irq, APICInt_IntrController.ic_Node.ln_Type, 0))
  76             {
  77                 D(bug("[Kernel:APIC-IA32] %s: failed to obtain IRQ %d\n", __func__, irq);)
  78             }
  79             else
  80             {
  81                 /* dont enable the vector yet...*/
  82                 if (!core_SetIDTGate((apicidt_t *)apicPrivate->cores[0].cpu_IDT, HW_IRQ_BASE + irq, (uintptr_t)IntrDefaultGates[HW_IRQ_BASE + irq], FALSE))
  83                 {
  84                     bug("[Kernel:APIC-IA32] %s: failed to set IRQ %d's Vector gate\n", __func__, irq);
  85                 }
  86             }
  87         }
  88         UserState(ssp);
  89     }
  90
  91     return TRUE;
  92 }
  93
  94 BOOL APICInt_DisableIRQ(APTR icPrivate, icid_t icInstance, icid_t intNum)
  95 {
  96     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
  97     struct APICData *apicPrivate = kernPlatD->kb_APIC;
  98     apicidt_t *IGATES;
  99     APTR ssp;
 100
 101     DINT(bug("[Kernel:APIC-IA32] %s(%03u #$%02X)\n", __func__, icInstance, intNum));
 102
 103     IGATES = (apicidt_t *)apicPrivate->cores[icInstance].cpu_IDT;
 104
 105     if ((ssp = SuperState()) != NULL)
 106     {
 107         IGATES[HW_IRQ_BASE + intNum].p = 0;
 108         UserState(ssp);
 109     }
 110
 111     return TRUE;
 112 }
 113
 114 BOOL APICInt_EnableIRQ(APTR icPrivate, icid_t icInstance, icid_t intNum)
 115 {
 116     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
 117     struct APICData *apicPrivate = kernPlatD->kb_APIC;
 118     apicidt_t *IGATES;
 119     APTR ssp;
 120
 121     DINT(bug("[Kernel:APIC-IA32] %s(%03u #$%02X)\n", __func__, icInstance, intNum));
 122
 123     IGATES = (apicidt_t *)apicPrivate->cores[icInstance].cpu_IDT;
 124
 125     if ((ssp = SuperState()) != NULL)
 126     {
 127         IGATES[HW_IRQ_BASE + intNum].p = 1;
 128         UserState(ssp);
 129     }
 130
 131     return TRUE;
 132 }
 133
 134 BOOL APICInt_AckIntr(APTR icPrivate, icid_t icInstance, icid_t intNum)
 135 {
 136     IPTR apic_base;
 137
 138     DINT(bug("[Kernel:APIC-IA32] %s(%03u #$%02X)\n", __func__, icInstance, intNum));
 139
 140     /* Write zero to EOI of APIC */
 141     apic_base = core_APIC_GetBase();
 142
 143     APIC_REG(apic_base, APIC_EOI) = 0;
 144
 145     return TRUE;
 146 }
 147
 148 struct IntrController APICInt_IntrController =
 149 {
 150     {
 151         .ln_Name = "x86 Local APIC",
 152         .ln_Pri = -50
 153     },
 154     0,
 155     AROS_MAKE_ID('A','P','I','C'),
 156     0,
 157     NULL,
 158     APICInt_Register,
 159     APICInt_Init,
 160     APICInt_EnableIRQ,
 161     APICInt_DisableIRQ,
 162     APICInt_AckIntr
 163 };
 164
 165 /* APIC IPI Related Functions ... ***************************/
 166
 167 static ULONG DoIPI(IPTR __APICBase, ULONG target, ULONG cmd)
 168 {
 169     ULONG ipisend_timeout, status_ipisend;
 170
 171     D(
 172         apicid_t cpuNum = KrnGetCPUNumber();
 173         bug("[Kernel:APIC-IA32.%03u] %s: Command 0x%08X to target %03u\n", cpuNum, __func__, cmd, target);
 174     )
 175
 176     /*
 177      * Send the IPI.
 178      * First we write target APIC ID into high command register.
 179      * Writing to the low register triggers the IPI itself.
 180      */
 181     APIC_REG(__APICBase, APIC_ICRH) = target << 24;
 182     APIC_REG(__APICBase, APIC_ICRL) = cmd;
 183
 184     D(bug("[Kernel:APIC-IA32.%03u] %s: Waiting for IPI to complete ", cpuNum, __func__));
 185
 186     for (ipisend_timeout = 1000; ipisend_timeout > 0; ipisend_timeout--)
 187     {
 188         pit_udelay(100);
 189 #ifdef DEBUG_WAIT
 190         if ((ipisend_timeout % 100) == 0)
 191         {
 192             bug(".");
 193         }
 194 #endif
 195         status_ipisend = APIC_REG(__APICBase, APIC_ICRL) & ICR_DS;
 196         /* Delivery status resets to 0 when delivery is done */
 197         if (status_ipisend == 0)
 198             break;
 199     }
 200     D(bug("\n"));
 201     D(bug("[Kernel:APIC-IA32.%03u] %s: ... left wait loop (status = 0x%08X)\n", cpuNum, __func__, status_ipisend));
 202
 203     return status_ipisend;
 204 }
 205
 206 /**********************************************************
 207                         Driver functions
 208  **********************************************************/
 209
 210 void core_APIC_Init(struct APICData *apic, apicid_t cpuNum)
 211 {
 212     IPTR __APICBase = apic->lapicBase;
 213     ULONG apic_ver = APIC_REG(__APICBase, APIC_VERSION);
 214     ULONG maxlvt = APIC_LVT(apic_ver), calibrated = 0;
 215     LONG lapic_initial, lapic_final;
 216     WORD pit_final;
 217     icintrid_t coreICInstID;
 218
 219 #ifdef CONFIG_LEGACY
 220     /* 82489DX doesnt report no. of LVT entries. */
 221     if (!APIC_INTEGRATED(apic_ver))
 222         maxlvt = 2;
 223 #endif
 224
 225     if ((coreICInstID = krnAddInterruptController(KernelBase, &APICInt_IntrController)) != (icintrid_t)-1)
 226     {
 227         APTR ssp = NULL;
 228         int i;
 229
 230         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC IC ID #%d:%d\n", cpuNum, __func__, ICINTR_ICID(coreICInstID), ICINTR_INST(coreICInstID)));
 231
 232         /*
 233          * NB: - BSP calls us in user mode, but AP's call us from supervisor
 234          */
 235         if ((KrnIsSuper()) || ((ssp = SuperState()) != NULL))
 236         {
 237             /* Obtain/set the critical IRQs and Vectors */
 238             for (i = 0; i < X86_CPU_EXCEPT_COUNT; i++)
 239             {
 240                 if ((HW_IRQ_BASE < i) && (cpuNum == 0))
 241                 {
 242                     if (!krnInitInterrupt(KernelBase, (i - HW_IRQ_BASE), APICInt_IntrController.ic_Node.ln_Type, 0))
 243                     {
 244                         krnPanic(NULL, "Failed to obtain APIC Exception IRQ\n"
 245                                        "IRQ #$%02X\n", (i - HW_IRQ_BASE));
 246                     }
 247                     if (!core_SetIRQGate(apic->cores[cpuNum].cpu_IDT, (i - HW_IRQ_BASE), (uintptr_t)IntrDefaultGates[i]))
 248                     {
 249                         krnPanic(NULL, "Failed to set APIC Exception IRQ Vector\n"
 250                                        "IRQ #$%02X, Vector #$02X\n", (i - HW_IRQ_BASE), i);
 251                     }
 252                 }
 253                 else if (!core_SetIDTGate((apicidt_t *)apic->cores[cpuNum].cpu_IDT, i, (uintptr_t)IntrDefaultGates[i], TRUE))
 254                 {
 255                     krnPanic(NULL, "Failed to set APIC Exception Vector\n"
 256                                    "Vector #$%02X\n", i);
 257                 }
 258             }
 259             D(bug("[Kernel:APIC-IA32.%03u] %s: APIC Exception Vectors configured\n", cpuNum, __func__));
 260
 261             if ((APIC_IRQ_ERROR < HW_IRQ_COUNT) && (cpuNum == 0))
 262             {
 263                 if (!krnInitInterrupt(KernelBase, (APIC_IRQ_ERROR - HW_IRQ_BASE), APICInt_IntrController.ic_Node.ln_Type, 0))
 264                 {
 265                     krnPanic(NULL, "Failed to obtain APIC Error IRQ\n"
 266                                    "IRQ #$%02X\n", (APIC_IRQ_ERROR - HW_IRQ_BASE));
 267                 }
 268                 if (!core_SetIRQGate(apic->cores[cpuNum].cpu_IDT, (APIC_IRQ_ERROR - HW_IRQ_BASE), (uintptr_t)IntrDefaultGates[APIC_IRQ_ERROR]))
 269                 {
 270                     krnPanic(NULL, "Failed to set APIC Error IRQ Vector\n"
 271                                    "IRQ #$%02X, Vector #$02X\n", (APIC_IRQ_ERROR - HW_IRQ_BASE), APIC_IRQ_ERROR);
 272                 }
 273             }
 274             else if (!core_SetIDTGate((apicidt_t *)apic->cores[cpuNum].cpu_IDT, APIC_IRQ_ERROR, (uintptr_t)IntrDefaultGates[APIC_IRQ_ERROR], TRUE))
 275             {
 276                 krnPanic(NULL, "Failed to set APIC Error Vector\n"
 277                                "Vector #$%02X\n", APIC_IRQ_ERROR);
 278             }
 279             else if (cpuNum == 0)
 280                 KrnAddExceptionHandler((APIC_IRQ_ERROR - APIC_IRQ_COUNT), core_APICErrorHandle, NULL, NULL);
 281
 282             D(bug("[Kernel:APIC-IA32.%03u] %s: APIC Error Vector #$%02X configured\n", cpuNum, __func__, APIC_IRQ_ERROR));
 283
 284             if (ssp)
 285                 UserState(ssp);
 286         }
 287         else
 288         {
 289             krnPanic(NULL, "Failed to configure APIC\n"
 290                                "APIC #%03e ID %03u\n", cpuNum, apic->cores[cpuNum].cpu_LocalID);
 291         }
 292
 293         /* Use flat interrupt model with logical destination ID = 1 */
 294         APIC_REG(__APICBase, APIC_DFR) = DFR_FLAT;
 295         APIC_REG(__APICBase, APIC_LDR) = 1 << LDR_ID_SHIFT;
 296
 297         /* Set Task Priority to 'accept all interrupts' */
 298         APIC_REG(__APICBase, APIC_TPR) = 0;
 299
 300         /* Set spurious IRQ vector to 0xFF. APIC enabled, focus check disabled. */
 301         APIC_REG(__APICBase, APIC_SVR) = SVR_ASE|SVR_FCC|0xFF;
 302
 303         /*
 304          * Set LINT0 to external and LINT1 to NMI.
 305          * These are common defaults and they are going to be overriden by ACPI tables.
 306          */
 307         APIC_REG(__APICBase, APIC_LINT0_VEC) = LVT_MT_EXT;
 308         APIC_REG(__APICBase, APIC_LINT1_VEC) = LVT_MT_NMI;
 309
 310 #ifdef CONFIG_LEGACY
 311         /* Due to the Pentium erratum 3AP. */
 312         if (maxlvt > 3)
 313              APIC_REG(__APICBase, APIC_ESR) = 0;
 314 #endif
 315
 316         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC ESR before enabling vector: %08x\n", cpuNum, __func__, APIC_REG(__APICBase, APIC_ESR)));
 317
 318         /* Set APIC error interrupt to fixed vector interrupt "APIC_IRQ_ERROR",  on APIC error */
 319         APIC_REG(__APICBase, APIC_ERROR_VEC) = APIC_IRQ_ERROR;
 320
 321         /* spec says clear errors after enabling vector. */
 322         if (maxlvt > 3)
 323              APIC_REG(__APICBase, APIC_ESR) = 0;
 324
 325         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC ESR after enabling vector: %08x\n", cpuNum, __func__, APIC_REG(__APICBase, APIC_ESR)));
 326
 327         /*
 328          * Now the tricky thing - calibrate LAPIC timer frequency.
 329          * In fact we could simply query CPU's clock frequency, but... x86 sucks. There's no
 330          * unified way to get it on whatever CPU. Intel has own way, AMD has own way... Etc... Which, additionally,
 331          * varies between CPU generations.
 332          *
 333          * The idea behing the calibration is to run the timer once, and see how much ticks passes in some defined
 334          * period of time. Then calculate a proportion.
 335          * We use 8253 PIT as our reference.
 336          * This calibrarion algorighm is based on NetBSD one.
 337          */
 338
 339         /* Set the timer to one-shot mode, no interrupt, 1:1 divisor */
 340         APIC_REG(__APICBase, APIC_TIMER_VEC) = LVT_MASK;
 341         APIC_REG(__APICBase, APIC_TIMER_DIV) = TIMER_DIV_1;
 342         APIC_REG(__APICBase, APIC_TIMER_ICR) = 0x80000000;      /* Just some very large value */
 343
 344         /*
 345          * Now wait for 11931 PIT ticks, which is equal to 10 milliseconds.
 346          * We don't use pit_udelay() here, because for improved accuracy we need to sample LAPIC timer counter twice,
 347          * before and after our actual delay (PIT setup also takes up some time, so LAPIC will count away from its
 348          * initial value).  We run it 5 times to make up for cache setup discrepancies.
 349          */
 350         for (i = 0; i < 5; i ++)
 351         {
 352             pit_start(11931);
 353             lapic_initial = (LONG)APIC_REG(__APICBase, APIC_TIMER_CCR);
 354
 355             pit_final   = pit_wait(11931);
 356             lapic_final = (LONG)APIC_REG(__APICBase, APIC_TIMER_CCR);
 357             calibrated += (((QUAD)(lapic_initial - lapic_final) * 11931LL)/(11931LL - (QUAD)pit_final)) ;
 358         }
 359         apic->cores[cpuNum].cpu_TimerFreq = 20 * calibrated;
 360         D(bug("[Kernel:APIC-IA32.%03u] %s: LAPIC frequency should be %u Hz (%u MHz)\n", cpuNum, __func__, apic->cores[cpuNum].cpu_TimerFreq, (apic->cores[cpuNum].cpu_TimerFreq + 500000) / 1000000));
 361
 362         /*
 363             Once APIC timer has been calibrated, set it to run at it's full speed and reload every second. Mask interrupts
 364         */
 365         APIC_REG(__APICBase, APIC_TIMER_VEC) = LVT_MASK | LVT_TMM_PERIOD;
 366         APIC_REG(__APICBase, APIC_TIMER_DIV) = TIMER_DIV_1;
 367         APIC_REG(__APICBase, APIC_TIMER_CCR) = apic->cores[cpuNum].cpu_TimerFreq;
 368         APIC_REG(__APICBase, APIC_TIMER_ICR) = apic->cores[cpuNum].cpu_TimerFreq;
 369
 370     }
 371 }
 372
 373 apicid_t core_APIC_GetID(IPTR _APICBase)
 374 {
 375     apicid_t _apic_id;
 376
 377     /* The actual ID is in 8 most significant bits */
 378     _apic_id = APIC_REG(_APICBase, APIC_ID) >> APIC_ID_SHIFT;
 379     DID(bug("[Kernel:APIC-IA32] %s: %03u\n", __func__, _apic_id));
 380
 381     return _apic_id;
 382 }
 383
 384 ULONG core_APIC_Wake(APTR wake_apicstartrip, apicid_t wake_apicid, IPTR __APICBase)
 385 {
 386     ULONG status_ipisend, status_ipirecv;
 387     ULONG start_count;
 388 #ifdef CONFIG_LEGACY
 389     ULONG apic_ver = APIC_REG(__APICBase, APIC_VERSION);
 390 #endif
 391     D(
 392         apicid_t cpuNo = KrnGetCPUNumber();
 393
 394         bug("[Kernel:APIC-IA32.%03u] %s(%03u @ %p)\n", cpuNo, __func__, wake_apicid, wake_apicstartrip);
 395         bug("[Kernel:APIC-IA32.%03u] %s: Base @ %p\n", cpuNo, __func__, __APICBase);
 396     )
 397 #ifdef CONFIG_LEGACY
 398     /*
 399      * Check if we have old 82489DX discrete APIC (version & 0xF0 == 0).
 400      * This APIC needs different startup procedure. It doesn't support STARTUP IPI
 401      * because old CPUs didn't have INIT signal. They jump to BIOS ROM boot code
 402      * immediately after INIT IPI. In order to run the bootstrap, a BIOS warm reset
 403      * magic has to be used there.
 404      */
 405     if (!APIC_INTEGRATED(apic_ver))
 406     {
 407         /*
 408          * BIOS warm reset magic, part one.
 409          * Write real-mode bootstrap routine address to 40:67 (real-mode address) location.
 410          * This is standard feature of IBM PC AT BIOS. If a warm reset condition is detected,
 411          * the BIOS jumps to the given address.
 412          */
 413         D(bug("[Kernel:APIC-IA32.%03u] %s: Setting BIOS vector for trampoline @ %p ..\n", cpuNo, __func__, wake_apicstartrip));
 414         *((volatile unsigned short *)0x469) = (IPTR)wake_apicstartrip >> 4;
 415         *((volatile unsigned short *)0x467) = 0; /* Actually wake_apicstartrip & 0x0F, it's page-aligned. */
 416
 417         /*
 418          * BIOS warm reset magic, part two.
 419          * This writes 0x0A into CMOS RAM, location 0x0F. This signals a warm reset condition to BIOS,
 420          * making part one work.
 421          */
 422         D(bug("[Kernel:APIC-IA32.%03u] %s: Setting warm reset code ..\n", cpuNo, __func__));
 423         outb(0xf, 0x70);
 424         outb(0xa, 0x71);
 425     }
 426 #endif
 427
 428     /* Flush TLB (we are supervisor here) */
 429     wrcr(cr3, rdcr(cr3));
 430
 431     /* First we send the INIT command (reset the core). Vector must be zero for this. */
 432     status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_INT_LEVELTRIG | ICR_INT_ASSERT | ICR_DM_INIT);
 433     if (status_ipisend)
 434     {
 435         D(bug("[Kernel:APIC-IA32.%03u] %s: Error asserting INIT\n", cpuNo, __func__));
 436         return status_ipisend;
 437     }
 438
 439     /* Deassert INIT after a small delay */
 440     pit_udelay(10 * 1000);
 441
 442     /* Deassert INIT */
 443     status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_INT_LEVELTRIG | ICR_DM_INIT);
 444     if (status_ipisend)
 445     {
 446         D(bug("[Kernel:APIC-IA32.%03u] %s: Error deasserting INIT\n", cpuNo, __func__));
 447         return status_ipisend;
 448     }
 449
 450     /* memory barrier */
 451     asm volatile("mfence":::"memory");
 452
 453 #ifdef CONFIG_LEGACY
 454     /* If it's 82489DX, we are done. */
 455     if (!APIC_INTEGRATED(apic_ver))
 456     {
 457         DWAKE(bug("[Kernel:APIC-IA32.%03u] %s: 82489DX detected, wakeup done\n", cpuNo, __func__));
 458         return 0;
 459     }
 460 #endif
 461
 462     /*
 463      * Perform IPI STARTUP loop.
 464      * According to official Intel specification, this must be done twice.
 465      * It's not explained why. ;-)
 466      */
 467     for (start_count = 1; start_count <= 2; start_count++)
 468     {
 469         D(bug("[Kernel:APIC-IA32.%03u] %s: Attempting STARTUP .. %u\n", cpuNo, __func__, start_count));
 470
 471         /* Clear any pending error condition */
 472         APIC_REG(__APICBase, APIC_ESR) = 0;
 473
 474         /*
 475          * Send STARTUP IPI.
 476          * The processor starts up at CS = (vector << 16) and IP = 0.
 477          */
 478         status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_DM_STARTUP | ((IPTR)wake_apicstartrip >> 12));
 479
 480         /* Allow the target APIC to accept the IPI */
 481         pit_udelay(200);
 482
 483 #ifdef CONFIG_LEGACY
 484         /* Pentium erratum 3AP quirk */
 485         if (APIC_LVT(apic_ver) > 3)
 486             APIC_REG(__APICBase, APIC_ESR) = 0;
 487 #endif
 488
 489         status_ipirecv = APIC_REG(__APICBase, APIC_ESR) & 0xEF;
 490
 491         /*
 492          * EXPERIMENTAL:
 493          * On my machine (macmini 3,1, as OS X system profiler says), the core starts up from first
 494          * attempt. The second attempt ends up in error (according to the documentation, the STARTUP
 495          * can be accepted only once, while the core in RESET or INIT state, and first STARTUP, if
 496          * successful, brings the core out of this state).
 497          * Here we try to detect this condition. If the core accepted STARTUP, we suggest that it has
 498          * started up, and break the loop.
 499          * A topic at osdev.org forum (http://forum.osdev.org/viewtopic.php?f=1&t=23018)
 500          * also tells about some problems with double STARTUP. According to it, the second STARTUP can
 501          * manage to re-run the core from the given address, leaving it in 64-bit mode, causing it to crash.
 502          *
 503          * If startup problems pops up (the core doesn't respond and AROS halts at "Launching APIC no X" stage),
 504          * the following two variations of this algorithm can be tried:
 505          * a) Always send STARTUP twice, but signal error condition only if both attempts failed.
 506          * b) Send first STARTUP, abort on error. Allow second attempt to fail and ignore its result.
 507          *
 508          *                                                              Sonic <pavel_fedin@mail.ru>
 509          */
 510         if (!status_ipisend && !status_ipirecv)
 511             break;
 512     }
 513
 514     DWAKE(bug("[Kernel:APIC-IA32.%03u] %s: STARTUP run status 0x%08X, error 0x%08X\n", cpuNo, __func__, status_ipisend, status_ipirecv));
 515
 516     /*
 517      * We return nonzero on error.
 518      * Actually least significant byte of this value holds ESR value, and 12th bit
 519      * holds delivery status flag from DoIPI() routine. It will be '1' if we got
 520      * stuck at sending phase.
 521      */
 522     return (status_ipisend | status_ipirecv);
 523 }