arch/x86_64-pc/kernel/intr.c

   1 #include <asm/cpu.h>
   2 #include <asm/io.h>
   3 #include <aros/libcall.h>
   4 #include <aros/asmcall.h>
   5 #include <exec/execbase.h>
   6 #include <hardware/intbits.h>
   7 #include <proto/exec.h>
   8
   9 #include <inttypes.h>
  10
  11 #include "kernel_base.h"
  12 #include "kernel_bootmem.h"
  13 #include "kernel_debug.h"
  14 #include "kernel_globals.h"
  15 #include "kernel_interrupts.h"
  16 #include "kernel_intern.h"
  17 #include "kernel_intr.h"
  18 #include "kernel_scheduler.h"
  19 #include "kernel_syscall.h"
  20 #include "cpu_traps.h"
  21 #include "apic.h"
  22 #include "xtpic.h"
  23
  24 #define D(x)
  25 #define DSYSCALL(x)
  26 #define DTRAP(x)
  27 #define DUMP_CONTEXT
  28
  29 /*
  30  * Simulate SysBase access at address 8.
  31  * Disabled because global SysBase is moved away from zeropage.
  32  *
  33 #define EMULATE_SYSBASE 8 */
  34
  35 #define IRQ(x,y) \
  36     IRQ##x##y##_intr
  37
  38 #define IRQPROTO(x, y) \
  39     void IRQ(x, y)(void)
  40
  41 #define IRQPROTO_16(x) \
  42     IRQPROTO(x,0); IRQPROTO(x,1); IRQPROTO(x,2); IRQPROTO(x,3); \
  43     IRQPROTO(x,4); IRQPROTO(x,5); IRQPROTO(x,6); IRQPROTO(x,7); \
  44     IRQPROTO(x,8); IRQPROTO(x,9); IRQPROTO(x,a); IRQPROTO(x,b); \
  45     IRQPROTO(x,c); IRQPROTO(x,d); IRQPROTO(x,e); IRQPROTO(x,f)
  46
  47 #define IRQLIST_16(x) \
  48     IRQ(x,0), IRQ(x,1), IRQ(x,2), IRQ(x,3), \
  49     IRQ(x,4), IRQ(x,5), IRQ(x,6), IRQ(x,7), \
  50     IRQ(x,8), IRQ(x,9), IRQ(x,a), IRQ(x,b), \
  51     IRQ(x,c), IRQ(x,d), IRQ(x,e), IRQ(x,f)
  52
  53 /* This generates prototypes for entry points */
  54 IRQPROTO_16(0x0);
  55 IRQPROTO_16(0x1);
  56 IRQPROTO_16(0x2);
  57 IRQPROTO(0x8, 0);
  58 IRQPROTO(0xf, e);
  59 extern void core_DefaultIRETQ(void);
  60
  61 const void *interrupt[256] =
  62 {
  63     IRQLIST_16(0x0),
  64     IRQLIST_16(0x1),
  65     IRQLIST_16(0x2)
  66 };
  67
  68 void core_SetupIDT(struct KernBootPrivate *__KernBootPrivate)
  69 {
  70     int i;
  71     uintptr_t off;
  72     struct segment_selector IDT_sel;
  73     struct int_gate_64bit *IGATES;
  74
  75     if (!__KernBootPrivate->IDT)
  76     {
  77         __KernBootPrivate->IDT = krnAllocBootMemAligned(sizeof(struct int_gate_64bit) * 256, 256);
  78
  79         D(bug("[Kernel] Allocated IDT at 0x%p\n", __KernBootPrivate->IDT));
  80     }
  81
  82     D(bug("[Kernel] core_SetupIDT: Setting all interrupt handlers to default value\n"));
  83     IGATES = __KernBootPrivate->IDT;
  84
  85     for (i=0; i < 256; i++)
  86     {
  87         if (interrupt[i])
  88             off = (uintptr_t)interrupt[i];
  89         else if (i == 0x80)
  90             off = (uintptr_t)IRQ0x80_intr;
  91         else if (i == 0xfe)
  92             off = (uintptr_t)IRQ0xfe_intr;
  93         else
  94             off = (uintptr_t)core_DefaultIRETQ;
  95
  96         IGATES[i].offset_low = off & 0xffff;
  97         IGATES[i].offset_mid = (off >> 16) & 0xffff;
  98         IGATES[i].offset_high = (off >> 32) & 0xffffffff;
  99         IGATES[i].type = 0x0e;
 100         IGATES[i].dpl = 3;
 101         IGATES[i].p = 1;
 102         IGATES[i].selector = KERNEL_CS;
 103         IGATES[i].ist = 0;
 104     }
 105
 106     D(bug("[Kernel] core_SetupIDT: Registering interrupt handlers ..\n"));
 107
 108     IDT_sel.size = sizeof(struct int_gate_64bit) * 256 - 1;
 109     IDT_sel.base = (unsigned long)__KernBootPrivate->IDT;
 110     asm volatile ("lidt %0"::"m"(IDT_sel));
 111 }
 112
 113 /* CPU exceptions are processed here */
 114 void core_IRQHandle(struct ExceptionContext *regs, unsigned long error_code, unsigned long irq_number)
 115 {
 116     struct KernelBase *KernelBase = getKernelBase();
 117
 118 #ifdef EMULATE_SYSBASE
 119     if (irq_number == 0x0e)
 120     {
 121         uint64_t ptr = rdcr(cr2);
 122         unsigned char *ip = (unsigned char *)regs->rip;
 123
 124         D(bug("[Kernel] Page fault exception\n"));
 125
 126         if (ptr == EMULATE_SYSBASE)
 127         {
 128             D(bug("[Kernel] ** Code at 0x%p is trying to access the SysBase at 0x%p.\n", ip, ptr));
 129
 130             if ((ip[0] & 0xfb) == 0x48 &&
 131                  ip[1]         == 0x8b &&
 132                 (ip[2] & 0xc7) == 0x04 &&
 133                  ip[3]         == 0x25)
 134             {
 135                 int reg = ((ip[2] >> 3) & 0x07) | ((ip[0] & 0x04) << 1);
 136
 137                 switch(reg)
 138                 {
 139                     case 0:
 140                         regs->rax = (UQUAD)SysBase;
 141                         break;
 142                     case 1:
 143                         regs->rcx = (UQUAD)SysBase;
 144                         break;
 145                     case 2:
 146                         regs->rdx = (UQUAD)SysBase;
 147                         break;
 148                     case 3:
 149                         regs->rbx = (UQUAD)SysBase;
 150                         break;
 151 //                    case 4:   /* Cannot put SysBase into rSP register */
 152 //                        regs->rsp = (UQUAD)SysBase;
 153 //                        break;
 154                     case 5:
 155                         regs->rbp = (UQUAD)SysBase;
 156                         break;
 157                     case 6:
 158                         regs->rsi = (UQUAD)SysBase;
 159                         break;
 160                     case 7:
 161                         regs->rdi = (UQUAD)SysBase;
 162                         break;
 163                     case 8:
 164                         regs->r8 = (UQUAD)SysBase;
 165                         break;
 166                     case 9:
 167                         regs->r9 = (UQUAD)SysBase;
 168                         break;
 169                     case 10:
 170                         regs->r10 = (UQUAD)SysBase;
 171                         break;
 172                     case 11:
 173                         regs->r11 = (UQUAD)SysBase;
 174                         break;
 175                     case 12:
 176                         regs->r12 = (UQUAD)SysBase;
 177                         break;
 178                     case 13:
 179                         regs->r13 = (UQUAD)SysBase;
 180                         break;
 181                     case 14:
 182                         regs->r14 = (UQUAD)SysBase;
 183                         break;
 184                     case 15:
 185                         regs->r15 = (UQUAD)SysBase;
 186                         break;
 187                 }
 188
 189                 regs->rip += 8;
 190
 191                 core_LeaveInterrupt(regs);
 192             }
 193             else if ((ip[0] & 0xfb) == 0x48 &&
 194                       ip[1]         == 0x8b &&
 195                      (ip[2] & 0xc7) == 0x05)
 196             {
 197                 int reg = ((ip[2] >> 3) & 0x07) | ((ip[0] & 0x04) << 1);
 198
 199                 switch(reg)
 200                 {
 201                     case 0:
 202                         regs->rax = (UQUAD)SysBase;
 203                         break;
 204                     case 1:
 205                         regs->rcx = (UQUAD)SysBase;
 206                         break;
 207                     case 2:
 208                         regs->rdx = (UQUAD)SysBase;
 209                         break;
 210                     case 3:
 211                         regs->rbx = (UQUAD)SysBase;
 212                         break;
 213 //                    case 4:   /* Cannot put SysBase into rSP register */
 214 //                        regs->rsp = (UQUAD)SysBase;
 215 //                        break;
 216                     case 5:
 217                         regs->rbp = (UQUAD)SysBase;
 218                         break;
 219                     case 6:
 220                         regs->rsi = (UQUAD)SysBase;
 221                         break;
 222                     case 7:
 223                         regs->rdi = (UQUAD)SysBase;
 224                         break;
 225                     case 8:
 226                         regs->r8 = (UQUAD)SysBase;
 227                         break;
 228                     case 9:
 229                         regs->r9 = (UQUAD)SysBase;
 230                         break;
 231                     case 10:
 232                         regs->r10 = (UQUAD)SysBase;
 233                         break;
 234                     case 11:
 235                         regs->r11 = (UQUAD)SysBase;
 236                         break;
 237                     case 12:
 238                         regs->r12 = (UQUAD)SysBase;
 239                         break;
 240                     case 13:
 241                         regs->r13 = (UQUAD)SysBase;
 242                         break;
 243                     case 14:
 244                         regs->r14 = (UQUAD)SysBase;
 245                         break;
 246                     case 15:
 247                         regs->r15 = (UQUAD)SysBase;
 248                         break;
 249                 }
 250
 251                 regs->rip += 7;
 252
 253                 core_LeaveInterrupt(regs);
 254             }
 255                 D(else bug("[Kernel] Instruction not recognized\n"));
 256         }
 257
 258 #ifdef DUMP_CONTEXT
 259         unsigned int i;
 260
 261         bug("[Kernel] PAGE FAULT accessing 0x%p\n", ptr);
 262         bug("[Kernel] Insn: ");
 263         for (i = 0; i < 16; i++)
 264             bug("%02x ", ip[i]);
 265         bug("\n");
 266 #endif
 267
 268         /* The exception will now be passed on to handling code below */
 269     }
 270 #endif
 271
 272     /* These exceptions are CPU traps */
 273     if (irq_number < 0x20)
 274     {
 275         cpu_Trap(regs, error_code, irq_number);
 276     }
 277     else if (irq_number == 0x80)  /* Syscall? */
 278     {
 279         /* Syscall number is actually ULONG (we use only eax) */
 280         ULONG sc = regs->rax;
 281
 282         DSYSCALL(bug("[Kernel] Syscall %u\n", sc));
 283
 284         /* The following syscalls can be run in both supervisor and user mode */
 285         switch (sc)
 286         {
 287         case SC_REBOOT:
 288             D(bug("[Kernel] Warm restart, stack 0x%p\n", AROS_GET_SP));
 289
 290             /*
 291              * Restart the kernel with a double stack swap. This doesn't return.
 292              * Double swap guarantees that core_Kick() is called when SP is set to a
 293              * dynamically allocated emergency stack and not to boot stack.
 294              * Such situation is rare but can occur in the following situation:
 295              * 1. Boot task calls SuperState(). Privilege changed, but stack is manually reset
 296              *    back into our .bss space.
 297              * 2. Boot task crashes. Privilege doesn't change this time, RSP is not changed.
 298              * 3. If we call core_Kick() right now, we are dead (core_Kick() clears .bss).
 299              */
 300             __asm__ __volatile__(
 301                 "cli\n\t"
 302                 "cld\n\t"
 303                 "movq %0, %%rsp\n\t"
 304                 "jmp *%1\n"
 305                 ::"r"(__KernBootPrivate->SystemStack + STACK_SIZE), "r"(core_Kick), "D"(BootMsg), "S"(kernel_cstart));
 306
 307         case SC_SUPERVISOR:
 308             /* This doesn't return */
 309             core_Supervisor(regs);
 310         }
 311
 312         /*
 313          * Scheduler can be called only from within user mode.
 314          * Every task has ss register initialized to a valid segment descriptor.
 315          * The descriptor itself isn't used by x86-64, however when a privilege
 316          * level switch occurs upon an interrupt, ss is reset to zero. Old ss value
 317          * is always pushed to stack as part of interrupt context.
 318          * We rely on this in order to determine which CPL we are returning to.
 319          */
 320         if (regs->ss != 0)
 321         {
 322             DSYSCALL(bug("[Kernel] User-mode syscall\n"));
 323
 324             /* Disable interrupts for a while */
 325             __asm__ __volatile__("cli; cld;");
 326
 327             core_SysCall(sc, regs);
 328         }
 329
 330         DSYSCALL(bug("[Kernel] Returning from syscall...\n"));
 331     }
 332     else if (irq_number >= 0x20) /* Hardware IRQ */
 333     {
 334         if (KernelBase)
 335         {
 336             /* From CPU's point of view, IRQs are exceptions starting from 0x20. */
 337             irq_number -= 0x20;
 338
 339             switch (KernelBase->kb_Interrupts[irq_number].lh_Type)
 340             {
 341             case KBL_APIC:
 342                 core_APIC_AckIntr();
 343                 break;
 344
 345             case KBL_XTPIC:
 346                 XTPIC_AckIntr(irq_number, &KernelBase->kb_PlatformData->kb_XTPIC_Mask);
 347                 krnRunIRQHandlers(KernelBase, irq_number);
 348
 349                 /*
 350                  * Interrupt acknowledge on XT-PIC also disables this interrupt.
 351                  * If we still need it, we need to re-enable it.
 352                  */
 353                 if (!IsListEmpty(&KernelBase->kb_Interrupts[irq_number]))
 354                     XTPIC_EnableIRQ(irq_number, &KernelBase->kb_PlatformData->kb_XTPIC_Mask);
 355
 356                 break;
 357             }
 358         }
 359
 360         /* Upon exit from the lowest-level hardware IRQ we run the task scheduler */
 361         if (SysBase && (regs->ss != 0))
 362         {
 363             /* Disable interrupts for a while */
 364             __asm__ __volatile__("cli; cld;");
 365
 366             core_ExitInterrupt(regs);
 367         }
 368     }
 369
 370     core_LeaveInterrupt(regs);
 371 }
 372
 373 void ictl_enable_irq(unsigned char irq, struct KernelBase *KernelBase)
 374 {
 375     if (KernelBase->kb_Interrupts[irq].lh_Type == KBL_XTPIC)
 376         XTPIC_EnableIRQ(irq, &KernelBase->kb_PlatformData->kb_XTPIC_Mask);
 377 }
 378
 379 void ictl_Initialize(void)
 380 {
 381     struct KernelBase *KernelBase = getKernelBase();
 382     struct PlatformData *pdata = KernelBase->kb_PlatformData;
 383
 384     if (!pdata->kb_APIC)
 385     {
 386         /* No APIC was discovered by ACPI/whatever else. Do the probe. */
 387         pdata->kb_APIC = core_APIC_Probe();
 388     }
 389
 390     if (!pdata)
 391     {
 392         /* We are x86-64 and we always have APIC. */
 393         krnPanic(KernelBase, "Failed to allocate APIC descriptor\n.The system is low on memory.");
 394     }
 395
 396     if (pdata->kb_APIC->flags & APF_8259)
 397     {
 398         /*
 399          * Initialize legacy 8529A PIC.
 400          * TODO: We obey ACPI information about its presence, however currently we don't have
 401          * IOAPIC support. Switching to IOAPIC requires full ACPI support including AML.
 402          */
 403
 404         XTPIC_Init(&pdata->kb_XTPIC_Mask);
 405     }
 406
 407     D(bug("[Kernel] kernel_cstart: Interrupts redirected. We will go back in a minute ;)\n"));
 408 }