src/runtime/x86-64-darwin-os.c

   1
   2 #ifdef LISP_FEATURE_SB_THREAD
   3 #include <architecture/i386/table.h>
   4 #include <i386/user_ldt.h>
   5 #include <mach/mach_init.h>
   6 #endif
   7
   8 #include "thread.h"
   9 #include "validate.h"
  10 #include "runtime.h"
  11 #include "interrupt.h"
  12 #include "x86-64-darwin-os.h"
  13 #include "genesis/fdefn.h"
  14
  15 #include <mach/mach.h>
  16 #include <mach/mach_error.h>
  17 #include <mach/mach_types.h>
  18 #include <mach/sync_policy.h>
  19 #include <mach/machine/thread_state.h>
  20 #include <mach/machine/thread_status.h>
  21 #include <sys/_types.h>
  22 #include <sys/ucontext.h>
  23 #include <pthread.h>
  24 #include <assert.h>
  25 #include <stdlib.h>
  26 #include <stdio.h>
  27
  28 #ifdef LISP_FEATURE_SB_THREAD
  29 pthread_mutex_t mach_exception_lock = PTHREAD_MUTEX_INITIALIZER;
  30 #endif
  31
  32 #ifdef LISP_FEATURE_MACH_EXCEPTION_HANDLER
  33
  34 kern_return_t mach_thread_init(mach_port_t thread_exception_port);
  35
  36 void sigill_handler(int signal, siginfo_t *siginfo, void *void_context);
  37 void sigtrap_handler(int signal, siginfo_t *siginfo, void *void_context);
  38 void memory_fault_handler(int signal, siginfo_t *siginfo, void *void_context);
  39
  40 /* exc_server handles mach exception messages from the kernel and
  41  * calls catch exception raise. We use the system-provided
  42  * mach_msg_server, which, I assume, calls exc_server in a loop.
  43  *
  44  */
  45 extern boolean_t exc_server();
  46
  47 /* This executes in the faulting thread as part of the signal
  48  * emulation.  It is passed a context with the uc_mcontext field
  49  * pointing to a valid block of memory. */
  50 void build_fake_signal_context(struct ucontext *context,
  51                                x86_thread_state64_t *thread_state,
  52                                x86_float_state64_t *float_state) {
  53     pthread_sigmask(0, NULL, &context->uc_sigmask);
  54     context->uc_mcontext->ss = *thread_state;
  55     context->uc_mcontext->fs = *float_state;
  56 }
  57
  58 /* This executes in the faulting thread as part of the signal
  59  * emulation.  It is effectively the inverse operation from above. */
  60 void update_thread_state_from_context(x86_thread_state64_t *thread_state,
  61                                       x86_float_state64_t *float_state,
  62                                       struct ucontext *context) {
  63     *thread_state = context->uc_mcontext->ss;
  64     *float_state = context->uc_mcontext->fs;
  65     pthread_sigmask(SIG_SETMASK, &context->uc_sigmask, NULL);
  66 }
  67
  68 /* Modify a context to push new data on its stack. */
  69 void push_context(u64 data, x86_thread_state64_t *context)
  70 {
  71     u64 *stack_pointer;
  72
  73     stack_pointer = (u64*) context->rsp;
  74     *(--stack_pointer) = data;
  75     context->rsp = (u64) stack_pointer;
  76 }
  77
  78 void align_context_stack(x86_thread_state64_t *context)
  79 {
  80     /* 16byte align the stack (provided that the stack is, as it
  81      * should be, 8byte aligned. */
  82     while (context->rsp & 15) push_context(0, context);
  83 }
  84
  85 /* Stack allocation starts with a context that has a mod-4 ESP value
  86  * and needs to leave a context with a mod-16 ESP that will restore
  87  * the old ESP value and other register state when activated.  The
  88  * first part of this is the recovery trampoline, which loads ESP from
  89  * EBP, pops EBP, and returns. */
  90 asm(".globl _stack_allocation_recover; .align 4; _stack_allocation_recover: mov %rbp, %rsp; pop %rsi; pop %rdi; pop \
  91 %rdx; pop %rcx; pop %r8; pop %r9; pop %rbp; ret;");
  92
  93 void open_stack_allocation(x86_thread_state64_t *context)
  94 {
  95     void stack_allocation_recover(void);
  96
  97     push_context(context->rip, context);
  98     push_context(context->rbp, context);
  99
 100     push_context(context->r9, context);
 101     push_context(context->r8, context);
 102     push_context(context->rcx, context);
 103     push_context(context->rdx, context);
 104     push_context(context->rsi, context);
 105     push_context(context->rdi, context);
 106
 107     context->rbp = context->rsp;
 108     context->rip = (u64) stack_allocation_recover;
 109
 110     align_context_stack(context);
 111 }
 112
 113 /* Stack allocation of data starts with a context with a mod-16 ESP
 114  * value and reserves some space on it by manipulating the ESP
 115  * register. */
 116 void *stack_allocate(x86_thread_state64_t *context, size_t size)
 117 {
 118     /* round up size to 16byte multiple */
 119     size = (size + 15) & -16;
 120
 121     context->rsp = ((u64)context->rsp) - size;
 122
 123     return (void *)context->rsp;
 124 }
 125
 126 /* Arranging to invoke a C function is tricky, as we have to assume
 127  * cdecl calling conventions (caller removes args) and x86/darwin
 128  * alignment requirements.  The simplest way to arrange this,
 129  * actually, is to open a new stack allocation.
 130  * WARNING!!! THIS DOES NOT PRESERVE REGISTERS! */
 131 void call_c_function_in_context(x86_thread_state64_t *context,
 132                                 void *function,
 133                                 int nargs,
 134                                 ...)
 135 {
 136     va_list ap;
 137     int i;
 138     u64 *stack_pointer;
 139
 140     /* Set up to restore stack on exit. */
 141     open_stack_allocation(context);
 142
 143     /* Have to keep stack 16byte aligned on x86/darwin. */
 144     for (i = (1 & -nargs); i; i--) {
 145         push_context(0, context);
 146     }
 147
 148     context->rsp = ((u64)context->rsp) - nargs * 8;
 149     stack_pointer = (u64 *)context->rsp;
 150
 151     va_start(ap, nargs);
 152     if (nargs > 0) context->rdi = va_arg(ap, u64);
 153     if (nargs > 1) context->rsi = va_arg(ap, u64);
 154     if (nargs > 2) context->rdx = va_arg(ap, u64);
 155     if (nargs > 3) context->rcx = va_arg(ap, u64);
 156     if (nargs > 4) context->r8 = va_arg(ap, u64);
 157     if (nargs > 5) context->r9 = va_arg(ap, u64);
 158     for (i = 6; i < nargs; i++) {
 159         stack_pointer[i] = va_arg(ap, u64);
 160     }
 161     va_end(ap);
 162
 163     push_context(context->rip, context);
 164     context->rip = (u64) function;
 165 }
 166
 167 void signal_emulation_wrapper(x86_thread_state64_t *thread_state,
 168                               x86_float_state64_t *float_state,
 169                               int signal,
 170                               siginfo_t *siginfo,
 171                               void (*handler)(int, siginfo_t *, void *))
 172 {
 173
 174     /* CLH: FIXME **NOTE: HACK ALERT!** Ideally, we would allocate
 175      * context and regs on the stack as local variables, but this
 176      * causes problems for the lisp debugger. When it walks the stack
 177      * for a back trace, it sees the 1) address of the local variable
 178      * on the stack and thinks that is a frame pointer to a lisp
 179      * frame, and, 2) the address of the sap that we alloc'ed in
 180      * dynamic space and thinks that is a return address, so it,
 181      * heuristicly (and wrongly), chooses that this should be
 182      * interpreted as a lisp frame instead of as a C frame.
 183      * We can work around this in this case by os_validating the
 184      * context (and regs just for symmetry).
 185      */
 186
 187     struct ucontext *context;
 188     struct mcontext *regs;
 189
 190     context = (struct ucontext*) os_validate(0, sizeof(struct ucontext));
 191     regs = (struct mcontext*) os_validate(0, sizeof(struct mcontext));
 192     context->uc_mcontext = regs;
 193
 194     /* when BSD signals are fired, they mask they signals in sa_mask
 195        which always seem to be the blockable_sigset, for us, so we
 196        need to:
 197        1) save the current sigmask
 198        2) block blockable signals
 199        3) call the signal handler
 200        4) restore the sigmask */
 201
 202     build_fake_signal_context(context, thread_state, float_state);
 203
 204     block_blockable_signals();
 205
 206     handler(signal, siginfo, context);
 207
 208     update_thread_state_from_context(thread_state, float_state, context);
 209
 210     os_invalidate((os_vm_address_t)context, sizeof(struct ucontext));
 211     os_invalidate((os_vm_address_t)regs, sizeof(struct mcontext));
 212
 213     /* Trap to restore the signal context. */
 214     asm volatile ("mov %0, %%rax; mov %1, %%rbx; .quad 0xffffffffffff0b0f"
 215                   : : "r" (thread_state), "r" (float_state));
 216 }
 217
 218 #if defined DUMP_CONTEXT
 219 void dump_context(x86_thread_state64_t *context)
 220 {
 221     int i;
 222     u64 *stack_pointer;
 223
 224     printf("rax: %08lx  rcx: %08lx  rdx: %08lx  rbx: %08lx\n",
 225            context->rax, context->rcx, context->rdx, context->rbx);
 226     printf("rsp: %08lx  rbp: %08lx  rsi: %08lx  rdi: %08lx\n",
 227            context->rsp, context->rbp, context->rsi, context->rdi);
 228     printf("rip: %08lx  eflags: %08lx\n",
 229            context->rip, context->rflags);
 230     printf("cs: %04hx  ds: %04hx  es: %04hx  "
 231            "ss: %04hx  fs: %04hx  gs: %04hx\n",
 232            context->cs, context->ds, context->rs,
 233            context->ss, context->fs, context->gs);
 234
 235     stack_pointer = (u64 *)context->rsp;
 236     for (i = 0; i < 48; i+=4) {
 237         printf("%08x:  %08x %08x %08x %08x\n",
 238                context->rsp + (i * 4),
 239                stack_pointer[i],
 240                stack_pointer[i+1],
 241                stack_pointer[i+2],
 242                stack_pointer[i+3]);
 243     }
 244 }
 245 #endif
 246
 247 void
 248 control_stack_exhausted_handler(int signal, siginfo_t *siginfo, void *void_context) {
 249     os_context_t *context = arch_os_get_context(&void_context);
 250
 251     arrange_return_to_lisp_function
 252         (context, SymbolFunction(CONTROL_STACK_EXHAUSTED_ERROR));
 253 }
 254
 255 void
 256 undefined_alien_handler(int signal, siginfo_t *siginfo, void *void_context) {
 257     os_context_t *context = arch_os_get_context(&void_context);
 258
 259     arrange_return_to_lisp_function
 260         (context, SymbolFunction(UNDEFINED_ALIEN_VARIABLE_ERROR));
 261 }
 262
 263 kern_return_t
 264 catch_exception_raise(mach_port_t exception_port,
 265                       mach_port_t thread,
 266                       mach_port_t task,
 267                       exception_type_t exception,
 268                       exception_data_t code_vector,
 269                       mach_msg_type_number_t code_count)
 270 {
 271     kern_return_t ret;
 272     int signal;
 273     siginfo_t* siginfo;
 274
 275 #ifdef LISP_FEATURE_SB_THREAD
 276     thread_mutex_lock(&mach_exception_lock);
 277 #endif
 278
 279     x86_thread_state64_t thread_state;
 280     mach_msg_type_number_t thread_state_count = x86_THREAD_STATE64_COUNT;
 281
 282     x86_float_state64_t float_state;
 283     mach_msg_type_number_t float_state_count = x86_FLOAT_STATE64_COUNT;
 284
 285     x86_exception_state64_t exception_state;
 286     mach_msg_type_number_t exception_state_count = x86_EXCEPTION_STATE64_COUNT;
 287
 288     x86_thread_state64_t backup_thread_state;
 289     x86_thread_state64_t *target_thread_state;
 290     x86_float_state64_t *target_float_state;
 291
 292     os_vm_address_t addr;
 293
 294     struct thread *th = (struct thread*) exception_port;
 295
 296     FSHOW((stderr,"/entering catch_exception_raise with exception: %d\n", exception));
 297
 298     switch (exception) {
 299
 300     case EXC_BAD_ACCESS:
 301         signal = SIGBUS;
 302         ret = thread_get_state(thread,
 303                                x86_THREAD_STATE64,
 304                                (thread_state_t)&thread_state,
 305                                &thread_state_count);
 306         ret = thread_get_state(thread,
 307                                x86_FLOAT_STATE64,
 308                                (thread_state_t)&float_state,
 309                                &float_state_count);
 310         ret = thread_get_state(thread,
 311                                x86_EXCEPTION_STATE64,
 312                                (thread_state_t)&exception_state,
 313                                &exception_state_count);
 314         addr = (void*)exception_state.faultvaddr;
 315
 316
 317         /* note the os_context hackery here.  When the signal handler returns,
 318          * it won't go back to what it was doing ... */
 319         if(addr >= CONTROL_STACK_GUARD_PAGE(th) &&
 320            addr < CONTROL_STACK_GUARD_PAGE(th) + os_vm_page_size) {
 321             /* We hit the end of the control stack: disable guard page
 322              * protection so the error handler has some headroom, protect the
 323              * previous page so that we can catch returns from the guard page
 324              * and restore it. */
 325             protect_control_stack_guard_page_thread(0, th);
 326             protect_control_stack_return_guard_page_thread(1, th);
 327
 328             backup_thread_state = thread_state;
 329             open_stack_allocation(&thread_state);
 330
 331             /* Save thread state */
 332             target_thread_state =
 333                 stack_allocate(&thread_state, sizeof(*target_thread_state));
 334             (*target_thread_state) = backup_thread_state;
 335
 336             /* Save float state */
 337             target_float_state =
 338                 stack_allocate(&thread_state, sizeof(*target_float_state));
 339             (*target_float_state) = float_state;
 340
 341             /* Set up siginfo */
 342             siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
 343             /* what do we need to put in our fake siginfo?  It looks like
 344              * the x86 code only uses si_signo and si_adrr. */
 345             siginfo->si_signo = signal;
 346             siginfo->si_addr = (void*)exception_state.faultvaddr;
 347
 348             call_c_function_in_context(&thread_state,
 349                                        signal_emulation_wrapper,
 350                                        5,
 351                                        target_thread_state,
 352                                        target_float_state,
 353                                        signal,
 354                                        siginfo,
 355                                        control_stack_exhausted_handler);
 356         }
 357         else if(addr >= CONTROL_STACK_RETURN_GUARD_PAGE(th) &&
 358                 addr < CONTROL_STACK_RETURN_GUARD_PAGE(th) + os_vm_page_size) {
 359             /* We're returning from the guard page: reprotect it, and
 360              * unprotect this one. This works even if we somehow missed
 361              * the return-guard-page, and hit it on our way to new
 362              * exhaustion instead. */
 363             protect_control_stack_guard_page_thread(1, th);
 364             protect_control_stack_return_guard_page_thread(0, th);
 365         }
 366         else if (addr >= undefined_alien_address &&
 367                  addr < undefined_alien_address + os_vm_page_size) {
 368             backup_thread_state = thread_state;
 369             open_stack_allocation(&thread_state);
 370
 371             /* Save thread state */
 372             target_thread_state =
 373                 stack_allocate(&thread_state, sizeof(*target_thread_state));
 374             (*target_thread_state) = backup_thread_state;
 375
 376             target_float_state =
 377                 stack_allocate(&thread_state, sizeof(*target_float_state));
 378             (*target_float_state) = float_state;
 379
 380             /* Set up siginfo */
 381             siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
 382             /* what do we need to put in our fake siginfo?  It looks like
 383              * the x86 code only uses si_signo and si_adrr. */
 384             siginfo->si_signo = signal;
 385             siginfo->si_addr = (void*)exception_state.faultvaddr;
 386
 387             call_c_function_in_context(&thread_state,
 388                                        signal_emulation_wrapper,
 389                                        5,
 390                                        target_thread_state,
 391                                        target_float_state,
 392                                        signal,
 393                                        siginfo,
 394                                        undefined_alien_handler);
 395         } else {
 396
 397             backup_thread_state = thread_state;
 398             open_stack_allocation(&thread_state);
 399
 400             /* Save thread state */
 401             target_thread_state =
 402                 stack_allocate(&thread_state, sizeof(*target_thread_state));
 403             (*target_thread_state) = backup_thread_state;
 404
 405             target_float_state =
 406                 stack_allocate(&thread_state, sizeof(*target_float_state));
 407             (*target_float_state) = float_state;
 408
 409             /* Set up siginfo */
 410             siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
 411             /* what do we need to put in our fake siginfo?  It looks like
 412              * the x86 code only uses si_signo and si_adrr. */
 413             siginfo->si_signo = signal;
 414             siginfo->si_addr = (void*)exception_state.faultvaddr;
 415
 416             call_c_function_in_context(&thread_state,
 417                                        signal_emulation_wrapper,
 418                                        5,
 419                                        target_thread_state,
 420                                        target_float_state,
 421                                        signal,
 422                                        siginfo,
 423                                        memory_fault_handler);
 424         }
 425         ret = thread_set_state(thread,
 426                                x86_THREAD_STATE64,
 427                                (thread_state_t)&thread_state,
 428                                thread_state_count);
 429
 430         ret = thread_set_state(thread,
 431                                x86_FLOAT_STATE64,
 432                                (thread_state_t)&float_state,
 433                                float_state_count);
 434 #ifdef LISP_FEATURE_SB_THREAD
 435         thread_mutex_unlock(&mach_exception_lock);
 436 #endif
 437         return KERN_SUCCESS;
 438
 439     case EXC_BAD_INSTRUCTION:
 440
 441         ret = thread_get_state(thread,
 442                                x86_THREAD_STATE64,
 443                                (thread_state_t)&thread_state,
 444                                &thread_state_count);
 445         ret = thread_get_state(thread,
 446                                x86_FLOAT_STATE64,
 447                                (thread_state_t)&float_state,
 448                                &float_state_count);
 449         ret = thread_get_state(thread,
 450                                x86_EXCEPTION_STATE64,
 451                                (thread_state_t)&exception_state,
 452                                &exception_state_count);
 453         if (0xffffffffffff0b0f == *((u64 *)thread_state.rip)) {
 454             /* fake sigreturn. */
 455
 456             /* When we get here, thread_state.rax is a pointer to a
 457              * thread_state to restore. */
 458             /* thread_state = *((thread_state_t *)thread_state.rax); */
 459
 460             ret = thread_set_state(thread,
 461                                    x86_THREAD_STATE64,
 462                                    (thread_state_t) thread_state.rax,
 463                                    /* &thread_state, */
 464                                    thread_state_count);
 465
 466             ret = thread_set_state(thread,
 467                                    x86_FLOAT_STATE64,
 468                                    (thread_state_t) thread_state.rbx,
 469                                    /* &thread_state, */
 470                                    float_state_count);
 471         } else {
 472
 473             backup_thread_state = thread_state;
 474             open_stack_allocation(&thread_state);
 475
 476             /* Save thread state */
 477             target_thread_state =
 478                 stack_allocate(&thread_state, sizeof(*target_thread_state));
 479             (*target_thread_state) = backup_thread_state;
 480
 481             target_float_state =
 482                 stack_allocate(&thread_state, sizeof(*target_float_state));
 483             (*target_float_state) = float_state;
 484
 485             /* Set up siginfo */
 486             siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
 487             /* what do we need to put in our fake siginfo?  It looks like
 488              * the x86 code only uses si_signo and si_adrr. */
 489             if (*((unsigned short *)target_thread_state->rip) == 0x0b0f) {
 490                 signal = SIGTRAP;
 491                 siginfo->si_signo = signal;
 492                 siginfo->si_addr = (void*)exception_state.faultvaddr;
 493                 target_thread_state->rip += 2;
 494                 call_c_function_in_context(&thread_state,
 495                                            signal_emulation_wrapper,
 496                                            5,
 497                                            target_thread_state,
 498                                            target_float_state,
 499                                            signal,
 500                                            siginfo,
 501                                            sigtrap_handler);
 502             } else {
 503                 signal = SIGILL;
 504                 siginfo->si_signo = signal;
 505                 siginfo->si_addr = (void*)exception_state.faultvaddr;
 506
 507                 call_c_function_in_context(&thread_state,
 508                                            signal_emulation_wrapper,
 509                                            5,
 510                                            target_thread_state,
 511                                            target_float_state,
 512                                            signal,
 513                                            siginfo,
 514                                            sigill_handler);
 515             }
 516             ret = thread_set_state(thread,
 517                                    x86_THREAD_STATE64,
 518                                    (thread_state_t)&thread_state,
 519                                    thread_state_count);
 520             ret = thread_set_state(thread,
 521                                    x86_FLOAT_STATE64,
 522                                    (thread_state_t)&float_state,
 523                                    float_state_count);
 524         }
 525 #ifdef LISP_FEATURE_SB_THREAD
 526         thread_mutex_unlock(&mach_exception_lock);
 527 #endif
 528         return KERN_SUCCESS;
 529
 530     default:
 531 #ifdef LISP_FEATURE_SB_THREAD
 532         thread_mutex_unlock(&mach_exception_lock);
 533 #endif
 534         return KERN_INVALID_RIGHT;
 535     }
 536 }
 537
 538 void *
 539 mach_exception_handler(void *port)
 540 {
 541   mach_msg_server(exc_server, 2048, (mach_port_t) port, 0);
 542   /* mach_msg_server should never return, but it should dispatch mach
 543    * exceptions to our catch_exception_raise function
 544    */
 545   abort();
 546 }
 547
 548 /* Sets up the thread that will listen for mach exceptions. note that
 549    the exception handlers will be run on this thread. This is
 550    different from the BSD-style signal handling situation in which the
 551    signal handlers run in the relevant thread directly. */
 552
 553 mach_port_t mach_exception_handler_port_set = MACH_PORT_NULL;
 554
 555 pthread_t
 556 setup_mach_exception_handling_thread()
 557 {
 558     kern_return_t ret;
 559     pthread_t mach_exception_handling_thread = NULL;
 560     pthread_attr_t attr;
 561
 562     /* allocate a mach_port for this process */
 563     ret = mach_port_allocate(mach_task_self(),
 564                              MACH_PORT_RIGHT_PORT_SET,
 565                              &mach_exception_handler_port_set);
 566
 567     /* create the thread that will receive the mach exceptions */
 568
 569     FSHOW((stderr, "Creating mach_exception_handler thread!\n"));
 570
 571     pthread_attr_init(&attr);
 572     pthread_create(&mach_exception_handling_thread,
 573                    &attr,
 574                    mach_exception_handler,
 575                    (void*) mach_exception_handler_port_set);
 576     pthread_attr_destroy(&attr);
 577
 578     return mach_exception_handling_thread;
 579 }
 580
 581 /* tell the kernel that we want EXC_BAD_ACCESS exceptions sent to the
 582    exception port (which is being listened to do by the mach
 583    exception handling thread). */
 584 kern_return_t
 585 mach_thread_init(mach_port_t thread_exception_port)
 586 {
 587     kern_return_t ret;
 588     /* allocate a named port for the thread */
 589
 590     FSHOW((stderr, "Allocating mach port %x\n", thread_exception_port));
 591
 592     ret = mach_port_allocate_name(mach_task_self(),
 593                                   MACH_PORT_RIGHT_RECEIVE,
 594                                   thread_exception_port);
 595     if (ret) {
 596         lose("mach_port_allocate_name failed with return_code %d\n", ret);
 597     }
 598
 599     /* establish the right for the thread_exception_port to send messages */
 600     ret = mach_port_insert_right(mach_task_self(),
 601                                  thread_exception_port,
 602                                  thread_exception_port,
 603                                  MACH_MSG_TYPE_MAKE_SEND);
 604     if (ret) {
 605         lose("mach_port_insert_right failed with return_code %d\n", ret);
 606     }
 607
 608     ret = thread_set_exception_ports(mach_thread_self(),
 609                                      EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION,
 610                                      thread_exception_port,
 611                                      EXCEPTION_DEFAULT,
 612                                      THREAD_STATE_NONE);
 613     if (ret) {
 614         lose("thread_set_exception_port failed with return_code %d\n", ret);
 615     }
 616
 617     ret = mach_port_move_member(mach_task_self(),
 618                                 thread_exception_port,
 619                                 mach_exception_handler_port_set);
 620     if (ret) {
 621         lose("mach_port_ failed with return_code %d\n", ret);
 622     }
 623
 624     return ret;
 625 }
 626
 627 void
 628 setup_mach_exceptions() {
 629     setup_mach_exception_handling_thread();
 630     mach_thread_init(THREAD_STRUCT_TO_EXCEPTION_PORT(all_threads));
 631 }
 632
 633 pid_t
 634 mach_fork() {
 635     pid_t pid = fork();
 636     if (pid == 0) {
 637         setup_mach_exceptions();
 638         return pid;
 639     } else {
 640         return pid;
 641     }
 642 }
 643
 644 #endif