src/runtime/x86-64-arch.c

   1 /*
   2  * This software is part of the SBCL system. See the README file for
   3  * more information.
   4  *
   5  * This software is derived from the CMU CL system, which was
   6  * written at Carnegie Mellon University and released into the
   7  * public domain. The software is in the public domain and is
   8  * provided with absolutely no warranty. See the COPYING and CREDITS
   9  * files for more information.
  10  */
  11
  12 #include <stdio.h>
  13
  14 #include "sbcl.h"
  15 #include "runtime.h"
  16 #include "globals.h"
  17 #include "validate.h"
  18 #include "os.h"
  19 #include "sbcl.h"
  20 #include "arch.h"
  21 #include "lispregs.h"
  22 #include "signal.h"
  23 #include "alloc.h"
  24 #include "interrupt.h"
  25 #include "interr.h"
  26 #include "breakpoint.h"
  27 #include "thread.h"
  28 #include "pseudo-atomic.h"
  29
  30 #include "genesis/static-symbols.h"
  31 #include "genesis/symbol.h"
  32
  33 #define BREAKPOINT_INST 0xcc    /* INT3 */
  34 #define UD2_INST 0x0b0f         /* UD2 */
  35
  36 #ifndef LISP_FEATURE_UD2_BREAKPOINTS
  37 #define BREAKPOINT_WIDTH 1
  38 #else
  39 #define BREAKPOINT_WIDTH 2
  40 #endif
  41
  42 unsigned int cpuid_fn1_ecx;
  43 unsigned int avx_supported = 0;
  44
  45 static void cpuid(unsigned info, unsigned subinfo,
  46                   unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx)
  47 {
  48 #ifdef _MSC_VER
  49   int regs[4];
  50   __cpuid(regs, info);
  51   *eax = regs[0];
  52   *ebx = regs[1];
  53   *ecx = regs[2];
  54   *edx = regs[3];
  55 #else
  56   __asm__("cpuid;"                                            /* assembly code */
  57           :"=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) /* outputs */
  58           :"a" (info), "c" (subinfo)                          /* input: info into eax,
  59                                                                  subinfo to ecx */
  60            /* clobbers: none */
  61           );
  62 #endif
  63 }
  64
  65 static void xgetbv(unsigned *eax, unsigned *edx)
  66 {
  67   __asm__("xgetbv;"
  68           :"=a" (*eax), "=d" (*edx)
  69           : "c" (0));
  70 }
  71
  72 void arch_init(void)
  73 {
  74   unsigned int eax, ebx, ecx, edx;
  75
  76   cpuid(0, 0, &eax, &ebx, &ecx, &edx);
  77   if (eax >= 1) { // see if we can execute basic id function 1
  78       unsigned avx_mask = 0x18000000; // OXSAVE and AVX
  79       cpuid(1, 0, &eax, &ebx, &ecx, &edx);
  80       cpuid_fn1_ecx = ecx;
  81       if ((ecx & avx_mask) == avx_mask) {
  82           xgetbv(&eax, &edx);
  83           if ((eax & 0x06) == 0x06) // YMM and XMM
  84               avx_supported = 1;
  85       }
  86   }
  87 }
  88
  89 #define FILL_VECTOR_T "FILL-VECTOR/T"
  90
  91 // Poke in a byte that changes an opcode to enable faster vector fill.
  92 // Using fixed offsets and bytes is no worse than what we do elsewhere.
  93 void tune_asm_routines_for_microarch(void)
  94 {
  95     // I don't know if this works on Windows
  96 #ifndef _MSC_VER
  97     unsigned int eax, ebx, ecx, edx;
  98     cpuid(0, 0, &eax, &ebx, &ecx, &edx);
  99     if (eax >= 7) {
 100         cpuid(7, 0, &eax, &ebx, &ecx, &edx);
 101         if (ebx & (1<<9)) // Enhanced Repeat Movs/Stos
 102           asm_routine_poke(FILL_VECTOR_T, 0x12, 0x7C); // Change JMP to JL
 103     }
 104 #endif
 105 }
 106
 107 /* Undo code patches so that the core file applies to the most generic
 108    microarchitecture on startup. As it happens, FILL-VECTOR/T is fine
 109    either way, but in general this might not be true for code using
 110    instructions that don't exist on some cpu family members */
 111 void untune_asm_routines_for_microarch(void)
 112 {
 113     asm_routine_poke(FILL_VECTOR_T, 0x12, 0xEB); // Change JL to JMP
 114 }
 115
 116 #ifndef _WIN64
 117 os_vm_address_t
 118 arch_get_bad_addr(int sig, siginfo_t *code, os_context_t *context)
 119 {
 120     return (os_vm_address_t)code->si_addr;
 121 }
 122 #endif
 123
 124 \f
 125 /*
 126  * hacking signal contexts
 127  *
 128  * (This depends both on architecture, which determines what we might
 129  * want to get to, and on OS, which determines how we get to it.)
 130  */
 131
 132 os_context_register_t *
 133 context_eflags_addr(os_context_t *context)
 134 {
 135 #if defined __linux__ || defined __sun
 136     /* KLUDGE: As of kernel 2.2.14 on Red Hat 6.2, there's code in the
 137      * <sys/ucontext.h> file to define symbolic names for offsets into
 138      * gregs[], but it's conditional on __USE_GNU and not defined, so
 139      * we need to do this nasty absolute index magic number thing
 140      * instead. */
 141     return (os_context_register_t*)&context->uc_mcontext.gregs[17];
 142 #elif defined LISP_FEATURE_FREEBSD || defined(__DragonFly__)
 143     return &context->uc_mcontext.mc_rflags;
 144 #elif defined LISP_FEATURE_DARWIN
 145     return CONTEXT_ADDR_FROM_STEM(rflags);
 146 #elif defined __OpenBSD__
 147     return &context->sc_rflags;
 148 #elif defined __NetBSD__
 149     return CONTEXT_ADDR_FROM_STEM(RFLAGS);
 150 #elif defined _WIN64
 151     return (os_context_register_t*)&context->win32_context->EFlags;
 152 #else
 153 #error unsupported OS
 154 #endif
 155 }
 156 \f
 157 void arch_skip_instruction(os_context_t *context)
 158 {
 159     /* Assuming we get here via an INT3 xxx instruction, the PC now
 160      * points to the interrupt code (a Lisp value) so we just move
 161      * past it. Skip the code; after that, if the code is an
 162      * error-trap or cerror-trap then skip the data bytes that follow. */
 163
 164     int vlen;
 165     long code;
 166
 167
 168     /* Get and skip the Lisp interrupt code. */
 169     code = *(char*)(*os_context_pc_addr(context))++;
 170     switch (code)
 171         {
 172         case trap_Error:
 173         case trap_Cerror:
 174             /* Lisp error arg vector length */
 175             vlen = *(char*)(*os_context_pc_addr(context))++;
 176             /* Skip Lisp error arg data bytes. */
 177             while (vlen-- > 0) {
 178                 ++*os_context_pc_addr(context);
 179             }
 180             break;
 181
 182         case trap_Breakpoint:           /* not tested */
 183         case trap_FunEndBreakpoint: /* not tested */
 184             break;
 185
 186 #ifdef LISP_FEATURE_SB_SAFEPOINT
 187         case trap_GlobalSafepoint:
 188         case trap_CspSafepoint:
 189 #endif
 190         case trap_PendingInterrupt:
 191         case trap_Halt:
 192         case trap_SingleStepAround:
 193         case trap_SingleStepBefore:
 194         case trap_InvalidArgCount:
 195             /* only needed to skip the Code */
 196             break;
 197
 198         default:
 199             fprintf(stderr,"[arch_skip_inst invalid code %ld\n]\n",code);
 200             break;
 201         }
 202
 203     FSHOW((stderr,
 204            "/[arch_skip_inst resuming at %x]\n",
 205            *os_context_pc_addr(context)));
 206 }
 207
 208 unsigned char *
 209 arch_internal_error_arguments(os_context_t *context)
 210 {
 211     return 1 + (unsigned char *)(*os_context_pc_addr(context));
 212 }
 213
 214 boolean
 215 arch_pseudo_atomic_atomic(os_context_t *context)
 216 {
 217     return get_pseudo_atomic_atomic(arch_os_get_current_thread());
 218 }
 219
 220 void
 221 arch_set_pseudo_atomic_interrupted(os_context_t *context)
 222 {
 223     struct thread *thread = arch_os_get_current_thread();
 224     set_pseudo_atomic_interrupted(thread);
 225 }
 226
 227 void
 228 arch_clear_pseudo_atomic_interrupted(os_context_t *context)
 229 {
 230     struct thread *thread = arch_os_get_current_thread();
 231     clear_pseudo_atomic_interrupted(thread);
 232 }
 233 \f
 234 /*
 235  * This stuff seems to get called for TRACE and debug activity.
 236  */
 237
 238 unsigned int
 239 arch_install_breakpoint(void *pc)
 240 {
 241     unsigned int result = *(unsigned int*)pc;
 242
 243 #ifndef LISP_FEATURE_UD2_BREAKPOINTS
 244     *(char*)pc = BREAKPOINT_INST;               /* x86 INT3       */
 245     *((char*)pc+1) = trap_Breakpoint;           /* Lisp trap code */
 246 #else
 247     *(char*)pc = UD2_INST & 0xff;
 248     *((char*)pc+1) = UD2_INST >> 8;
 249     *((char*)pc+2) = trap_Breakpoint;
 250 #endif
 251
 252     return result;
 253 }
 254
 255 void
 256 arch_remove_breakpoint(void *pc, unsigned int orig_inst)
 257 {
 258     *((char *)pc) = orig_inst & 0xff;
 259     *((char *)pc + 1) = (orig_inst & 0xff00) >> 8;
 260 #if BREAKPOINT_WIDTH > 1
 261     *((char *)pc + 2) = (orig_inst & 0xff0000) >> 16;
 262 #endif
 263 }
 264 \f
 265 /* When single stepping, single_stepping holds the original instruction
 266  * PC location. */
 267 unsigned int *single_stepping = NULL;
 268 #ifdef CANNOT_GET_TO_SINGLE_STEP_FLAG
 269 unsigned int  single_step_save1;
 270 unsigned int  single_step_save2;
 271 unsigned int  single_step_save3;
 272 #endif
 273
 274 void
 275 arch_do_displaced_inst(os_context_t *context, unsigned int orig_inst)
 276 {
 277     unsigned int *pc = (unsigned int*)(*os_context_pc_addr(context));
 278
 279     /* Put the original instruction back. */
 280     arch_remove_breakpoint(pc, orig_inst);
 281
 282 #ifdef CANNOT_GET_TO_SINGLE_STEP_FLAG
 283     /* Install helper instructions for the single step:
 284      * pushf; or [esp],0x100; popf. */
 285     single_step_save1 = *(pc-3);
 286     single_step_save2 = *(pc-2);
 287     single_step_save3 = *(pc-1);
 288     *(pc-3) = 0x9c909090;
 289     *(pc-2) = 0x00240c81;
 290     *(pc-1) = 0x9d000001;
 291 #else
 292     *context_eflags_addr(context) |= 0x100;
 293 #endif
 294
 295     single_stepping = pc;
 296
 297 #ifdef CANNOT_GET_TO_SINGLE_STEP_FLAG
 298     *os_context_pc_addr(context) = (os_context_register_t)((char *)pc - 9);
 299 #endif
 300 }
 301
 302 void
 303 arch_handle_breakpoint(os_context_t *context)
 304 {
 305     *os_context_pc_addr(context) -= BREAKPOINT_WIDTH;
 306     handle_breakpoint(context);
 307 }
 308
 309 void
 310 arch_handle_fun_end_breakpoint(os_context_t *context)
 311 {
 312     *os_context_pc_addr(context) -= BREAKPOINT_WIDTH;
 313     *os_context_pc_addr(context) =
 314         (uword_t)handle_fun_end_breakpoint(context);
 315 }
 316
 317 void
 318 arch_handle_single_step_trap(os_context_t *context, int trap)
 319 {
 320     arch_skip_instruction(context);
 321     /* On x86-64 the fdefn / function is always in RAX, so we pass
 322      * 0 as the register_offset. */
 323     handle_single_step_trap(context, trap, 0);
 324 }
 325
 326 \f
 327 void
 328 restore_breakpoint_from_single_step(os_context_t * context)
 329 {
 330 #ifdef CANNOT_GET_TO_SINGLE_STEP_FLAG
 331     /* Un-install single step helper instructions. */
 332     *(single_stepping-3) = single_step_save1;
 333     *(single_stepping-2) = single_step_save2;
 334     *(single_stepping-1) = single_step_save3;
 335 #else
 336     *context_eflags_addr(context) &= ~0x100;
 337 #endif
 338     /* Re-install the breakpoint if possible. */
 339     if (((char *)*os_context_pc_addr(context) >
 340          (char *)single_stepping) &&
 341         ((char *)*os_context_pc_addr(context) <=
 342          (char *)single_stepping + BREAKPOINT_WIDTH)) {
 343         fprintf(stderr, "warning: couldn't reinstall breakpoint\n");
 344     } else {
 345         arch_install_breakpoint(single_stepping);
 346     }
 347
 348     single_stepping = NULL;
 349     return;
 350 }
 351
 352 void
 353 sigtrap_handler(int signal, siginfo_t *info, os_context_t *context)
 354 {
 355     unsigned int trap;
 356
 357     if (single_stepping) {
 358         restore_breakpoint_from_single_step(context);
 359         return;
 360     }
 361
 362     /* This is just for info in case the monitor wants to print an
 363      * approximation. */
 364     access_control_stack_pointer(arch_os_get_current_thread()) =
 365         (lispobj *)*os_context_sp_addr(context);
 366
 367     /* On entry %eip points just after the INT3 byte and aims at the
 368      * 'kind' value (eg trap_Cerror). For error-trap and Cerror-trap a
 369      * number of bytes will follow, the first is the length of the byte
 370      * arguments to follow. */
 371     trap = *(unsigned char *)(*os_context_pc_addr(context));
 372
 373     handle_trap(context, trap);
 374 }
 375
 376 void
 377 sigill_handler(int signal, siginfo_t *siginfo, os_context_t *context) {
 378     /* Triggering SIGTRAP using int3 is unreliable on OS X/x86, so
 379      * we need to use illegal instructions for traps.
 380      */
 381 #if defined(LISP_FEATURE_UD2_BREAKPOINTS) && !defined(LISP_FEATURE_MACH_EXCEPTION_HANDLER)
 382     if (*((unsigned short *)*os_context_pc_addr(context)) == UD2_INST) {
 383         *os_context_pc_addr(context) += 2;
 384         return sigtrap_handler(signal, siginfo, context);
 385     }
 386 #endif
 387
 388     fake_foreign_function_call(context);
 389     lose("Unhandled SIGILL at %p.", *os_context_pc_addr(context));
 390 }
 391
 392 #ifdef X86_64_SIGFPE_FIXUP
 393 #define MXCSR_IE (0x01)         /* Invalid Operation */
 394 #define MXCSR_DE (0x02)         /* Denormal */
 395 #define MXCSR_ZE (0x04)         /* Devide-by-Zero */
 396 #define MXCSR_OE (0x08)         /* Overflow */
 397 #define MXCSR_UE (0x10)         /* Underflow */
 398 #define MXCSR_PE (0x20)         /* Precision */
 399
 400 static inline int
 401 mxcsr_to_code(unsigned int mxcsr)
 402 {
 403     /* Extract unmasked exception bits. */
 404     mxcsr &= ~(mxcsr >> 7) & 0x3F;
 405
 406     /* This order is defined at "Intel 64 and IA-32 Architectures
 407      * Software Developerfs Manual" Volume 1: "Basic Architecture",
 408      * 4.9.2 "Floating-Point Exception Priority". */
 409     if (mxcsr & MXCSR_IE)
 410         return FPE_FLTINV;
 411     else if (mxcsr & MXCSR_ZE)
 412         return FPE_FLTDIV;
 413     else if (mxcsr & MXCSR_DE)
 414         return FPE_FLTUND;
 415     else if (mxcsr & MXCSR_OE)
 416         return FPE_FLTOVF;
 417     else if (mxcsr & MXCSR_UE)
 418         return FPE_FLTUND;
 419     else if (mxcsr & MXCSR_PE)
 420         return FPE_FLTRES;
 421
 422     return 0;
 423 }
 424
 425 static void
 426 sigfpe_handler(int signal, siginfo_t *siginfo, os_context_t *context)
 427 {
 428     unsigned int *mxcsr = arch_os_context_mxcsr_addr(context);
 429
 430 #ifndef LISP_FEATURE_DARWIN
 431     /* Darwin doesn't handle accrued bits right. */
 432     if (siginfo->si_code == 0)
 433 #endif
 434     { /* XMM exception */
 435         siginfo->si_code = mxcsr_to_code(*mxcsr);
 436
 437         /* Clear sticky exception flag. */
 438         *mxcsr &= ~0x3F;
 439     }
 440
 441     interrupt_handle_now(signal, siginfo, context);
 442 }
 443 #endif
 444
 445 void
 446 arch_install_interrupt_handlers()
 447 {
 448     SHOW("entering arch_install_interrupt_handlers()");
 449
 450     /* Note: The old CMU CL code here used sigtrap_handler() to handle
 451      * SIGILL as well as SIGTRAP. I couldn't see any reason to do
 452      * things that way. So, I changed to separate handlers when
 453      * debugging a problem on OpenBSD, where SBCL wasn't catching
 454      * SIGILL properly, but was instead letting the process be
 455      * terminated with an "Illegal instruction" output. If this change
 456      * turns out to break something (maybe breakpoint handling on some
 457      * OS I haven't tested on?) and we have to go back to the old CMU
 458      * CL way, I hope there will at least be a comment to explain
 459      * why.. -- WHN 2001-06-07 */
 460 #if !defined(LISP_FEATURE_MACH_EXCEPTION_HANDLER) && !defined(LISP_FEATURE_WIN32)
 461     undoably_install_low_level_interrupt_handler(SIGILL , sigill_handler);
 462     undoably_install_low_level_interrupt_handler(SIGTRAP, sigtrap_handler);
 463 #endif
 464
 465 #if defined(X86_64_SIGFPE_FIXUP) && !defined(LISP_FEATURE_WIN32)
 466     undoably_install_low_level_interrupt_handler(SIGFPE, sigfpe_handler);
 467 #endif
 468
 469     SHOW("returning from arch_install_interrupt_handlers()");
 470 }
 471 \f
 472 #ifdef LISP_FEATURE_LINKAGE_TABLE
 473 /* FIXME: It might be cleaner to generate these from the lisp side of
 474  * things.
 475  */
 476
 477 void
 478 arch_write_linkage_table_jmp(char *reloc_addr, void *target_addr)
 479 {
 480     uword_t addr = (uword_t)target_addr;
 481     int i;
 482
 483     *reloc_addr++ = 0xFF; /* Opcode for near jump to absolute reg/mem64. */
 484     *reloc_addr++ = 0x25; /* ModRM #b00 100 101, i.e. RIP-relative. */
 485     *reloc_addr++ = 0x00; /* 32-bit displacement field = 0 */
 486     *reloc_addr++ = 0x00; /* ... */
 487     *reloc_addr++ = 0x00; /* ... */
 488     *reloc_addr++ = 0x00; /* ... */
 489
 490     for (i = 0; i < 8; i++) {
 491         *reloc_addr++ = addr & 0xff;
 492         addr >>= 8;
 493     }
 494
 495     /* write a nop for good measure. */
 496     *reloc_addr = 0x90;
 497 }
 498
 499 void
 500 arch_write_linkage_table_ref(void *reloc_addr, void *target_addr)
 501 {
 502     *(uword_t *)reloc_addr = (uword_t)target_addr;
 503 }
 504
 505 #endif
 506
 507 /* These setup and check *both* the sse2 and x87 FPUs. While lisp code
 508    only uses the sse2 FPU, other code (such as libc) may use the x87 FPU.
 509  */
 510
 511 unsigned int
 512 arch_get_fp_modes()
 513 {
 514     unsigned int temp;
 515     unsigned int result;
 516     /* return the x87 exception flags ored in with the sse2
 517      * control+status flags */
 518     asm ("fnstsw %0" : "=m" (temp));
 519     result = temp;
 520     result &= 0x3F;
 521     asm ("stmxcsr %0" : "=m" (temp));
 522     result |= temp;
 523     /* flip exception mask bits */
 524     return result ^ (0x3F << 7);
 525 }
 526
 527 struct fpenv
 528 {
 529     unsigned short cw;
 530     unsigned short unused1;
 531     unsigned short sw;
 532     unsigned short unused2;
 533     unsigned int other_regs[5];
 534 };
 535
 536 void
 537 arch_set_fp_modes(unsigned int mxcsr)
 538 {
 539     struct fpenv f_env;
 540     unsigned int temp;
 541
 542     /* turn trap enable bits into exception mask */
 543     mxcsr ^= 0x3F << 7;
 544
 545     /* set x87 modes */
 546     asm ("fnstenv %0" : "=m" (f_env));
 547     /* set control word: always long double precision
 548      * get traps and rounding from mxcsr word */
 549     f_env.cw = 0x300 | ((mxcsr >> 7) & 0x3F) | (((mxcsr >> 13) & 0x3) << 10);
 550     /* set status word: only override exception flags, from mxcsr */
 551     f_env.sw &= ~0x3F;
 552     f_env.sw |= (mxcsr & 0x3F);
 553
 554     asm ("fldenv %0" : : "m" (f_env));
 555
 556     /* now, simply, load up the mxcsr register */
 557     temp = mxcsr;
 558     asm ("ldmxcsr %0" : : "m" (temp));
 559 }
 560
 561 #ifdef LISP_FEATURE_IMMOBILE_CODE
 562 /// Return the Lisp object that fdefn's raw_addr slot jumps to.
 563 /// This will either be:
 564 /// (1) a simple-fun,
 565 /// (2) a funcallable-instance with an embedded trampoline that makes
 566 ///     it resemble a simple-fun in terms of call convention, or
 567 /// (3) a code-component with no simple-fun within it, that makes
 568 ///     closures and other funcallable-instances look like simple-funs.
 569 lispobj fdefn_raw_referent(struct fdefn* fdefn) {
 570     if (((lispobj)fdefn->raw_addr & 0xFE) == 0xE8) {  // looks good
 571         unsigned int raw_fun = (int)(long)&fdefn->raw_addr + 5 // length of "JMP rel32"
 572           + *(int*)((char*)&fdefn->raw_addr + 1);
 573         switch (((unsigned char*)&fdefn->raw_addr)[5]) {
 574         case 0x00: // no closure/fin trampoline
 575           // If the target address is in read-only space, then it's a jump or call
 576           // to an asm routine, and there is no corresponding simple-fun.
 577           // While we could locate and return the code-object, it's difficult to,
 578           // and there's no reason to - scavenging it is unnecessary.
 579           return raw_fun >= IMMOBILE_SPACE_START ? raw_fun - FUN_RAW_ADDR_OFFSET : 0;
 580         case 0x48: // embedded funcallable instance trampoline
 581           return (raw_fun - (4<<WORD_SHIFT)) | FUN_POINTER_LOWTAG;
 582         case 0x90: // general closure/fin trampoline
 583           return (raw_fun - offsetof(struct code, constants)) | OTHER_POINTER_LOWTAG;
 584         }
 585     } else if (fdefn->raw_addr == 0)
 586         return 0;
 587     lose("Can't decode fdefn raw addr @ %p: %p\n", fdefn, fdefn->raw_addr);
 588 }
 589 #endif