wineps: Continue printing if path drawing function returns error.
[wine.git] / dlls / ntdll / signal_x86_64.c
blob5378a1ebee91afeb865664236ace2e4ec2098e4a
1 /*
2 * x86-64 signal handling routines
4 * Copyright 1999, 2005 Alexandre Julliard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
21 #ifdef __x86_64__
23 #include <stdlib.h>
24 #include <stdarg.h>
26 #define NONAMELESSUNION
27 #define NONAMELESSSTRUCT
28 #include "ntstatus.h"
29 #define WIN32_NO_STATUS
30 #include "windef.h"
31 #include "winternl.h"
32 #include "wine/exception.h"
33 #include "wine/list.h"
34 #include "ntdll_misc.h"
35 #include "wine/debug.h"
37 WINE_DEFAULT_DEBUG_CHANNEL(unwind);
38 WINE_DECLARE_DEBUG_CHANNEL(seh);
39 WINE_DECLARE_DEBUG_CHANNEL(threadname);
41 typedef struct _SCOPE_TABLE
43 ULONG Count;
44 struct
46 ULONG BeginAddress;
47 ULONG EndAddress;
48 ULONG HandlerAddress;
49 ULONG JumpTarget;
50 } ScopeRecord[1];
51 } SCOPE_TABLE, *PSCOPE_TABLE;
54 /* layering violation: the setjmp buffer is defined in msvcrt, but used by RtlUnwindEx */
55 struct MSVCRT_JUMP_BUFFER
57 ULONG64 Frame;
58 ULONG64 Rbx;
59 ULONG64 Rsp;
60 ULONG64 Rbp;
61 ULONG64 Rsi;
62 ULONG64 Rdi;
63 ULONG64 R12;
64 ULONG64 R13;
65 ULONG64 R14;
66 ULONG64 R15;
67 ULONG64 Rip;
68 ULONG MxCsr;
69 USHORT FpCsr;
70 USHORT Spare;
71 M128A Xmm6;
72 M128A Xmm7;
73 M128A Xmm8;
74 M128A Xmm9;
75 M128A Xmm10;
76 M128A Xmm11;
77 M128A Xmm12;
78 M128A Xmm13;
79 M128A Xmm14;
80 M128A Xmm15;
83 /***********************************************************************
84 * Definitions for Win32 unwind tables
87 union handler_data
89 RUNTIME_FUNCTION chain;
90 ULONG handler;
93 struct opcode
95 BYTE offset;
96 BYTE code : 4;
97 BYTE info : 4;
100 struct UNWIND_INFO
102 BYTE version : 3;
103 BYTE flags : 5;
104 BYTE prolog;
105 BYTE count;
106 BYTE frame_reg : 4;
107 BYTE frame_offset : 4;
108 struct opcode opcodes[1]; /* info->count entries */
109 /* followed by handler_data */
112 #define UWOP_PUSH_NONVOL 0
113 #define UWOP_ALLOC_LARGE 1
114 #define UWOP_ALLOC_SMALL 2
115 #define UWOP_SET_FPREG 3
116 #define UWOP_SAVE_NONVOL 4
117 #define UWOP_SAVE_NONVOL_FAR 5
118 #define UWOP_EPILOG 6
119 #define UWOP_SAVE_XMM128 8
120 #define UWOP_SAVE_XMM128_FAR 9
121 #define UWOP_PUSH_MACHFRAME 10
123 static void dump_unwind_info( ULONG64 base, RUNTIME_FUNCTION *function )
125 static const char * const reg_names[16] =
126 { "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
127 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" };
129 union handler_data *handler_data;
130 struct UNWIND_INFO *info;
131 unsigned int i, count;
133 TRACE( "**** func %lx-%lx\n", function->BeginAddress, function->EndAddress );
134 for (;;)
136 if (function->UnwindData & 1)
138 RUNTIME_FUNCTION *next = (RUNTIME_FUNCTION *)((char *)base + (function->UnwindData & ~1));
139 TRACE( "unwind info for function %p-%p chained to function %p-%p\n",
140 (char *)base + function->BeginAddress, (char *)base + function->EndAddress,
141 (char *)base + next->BeginAddress, (char *)base + next->EndAddress );
142 function = next;
143 continue;
145 info = (struct UNWIND_INFO *)((char *)base + function->UnwindData);
147 TRACE( "unwind info at %p flags %x prolog 0x%x bytes function %p-%p\n",
148 info, info->flags, info->prolog,
149 (char *)base + function->BeginAddress, (char *)base + function->EndAddress );
151 if (info->frame_reg)
152 TRACE( " frame register %s offset 0x%x(%%rsp)\n",
153 reg_names[info->frame_reg], info->frame_offset * 16 );
155 for (i = 0; i < info->count; i++)
157 TRACE( " 0x%x: ", info->opcodes[i].offset );
158 switch (info->opcodes[i].code)
160 case UWOP_PUSH_NONVOL:
161 TRACE( "pushq %%%s\n", reg_names[info->opcodes[i].info] );
162 break;
163 case UWOP_ALLOC_LARGE:
164 if (info->opcodes[i].info)
166 count = *(DWORD *)&info->opcodes[i+1];
167 i += 2;
169 else
171 count = *(USHORT *)&info->opcodes[i+1] * 8;
172 i++;
174 TRACE( "subq $0x%x,%%rsp\n", count );
175 break;
176 case UWOP_ALLOC_SMALL:
177 count = (info->opcodes[i].info + 1) * 8;
178 TRACE( "subq $0x%x,%%rsp\n", count );
179 break;
180 case UWOP_SET_FPREG:
181 TRACE( "leaq 0x%x(%%rsp),%s\n",
182 info->frame_offset * 16, reg_names[info->frame_reg] );
183 break;
184 case UWOP_SAVE_NONVOL:
185 count = *(USHORT *)&info->opcodes[i+1] * 8;
186 TRACE( "movq %%%s,0x%x(%%rsp)\n", reg_names[info->opcodes[i].info], count );
187 i++;
188 break;
189 case UWOP_SAVE_NONVOL_FAR:
190 count = *(DWORD *)&info->opcodes[i+1];
191 TRACE( "movq %%%s,0x%x(%%rsp)\n", reg_names[info->opcodes[i].info], count );
192 i += 2;
193 break;
194 case UWOP_SAVE_XMM128:
195 count = *(USHORT *)&info->opcodes[i+1] * 16;
196 TRACE( "movaps %%xmm%u,0x%x(%%rsp)\n", info->opcodes[i].info, count );
197 i++;
198 break;
199 case UWOP_SAVE_XMM128_FAR:
200 count = *(DWORD *)&info->opcodes[i+1];
201 TRACE( "movaps %%xmm%u,0x%x(%%rsp)\n", info->opcodes[i].info, count );
202 i += 2;
203 break;
204 case UWOP_PUSH_MACHFRAME:
205 TRACE( "PUSH_MACHFRAME %u\n", info->opcodes[i].info );
206 break;
207 case UWOP_EPILOG:
208 if (info->version == 2)
210 unsigned int offset;
211 if (info->opcodes[i].info)
212 offset = info->opcodes[i].offset;
213 else
214 offset = (info->opcodes[i+1].info << 8) + info->opcodes[i+1].offset;
215 TRACE("epilog %p-%p\n", (char *)base + function->EndAddress - offset,
216 (char *)base + function->EndAddress - offset + info->opcodes[i].offset );
217 i += 1;
218 break;
220 default:
221 FIXME( "unknown code %u\n", info->opcodes[i].code );
222 break;
226 handler_data = (union handler_data *)&info->opcodes[(info->count + 1) & ~1];
227 if (info->flags & UNW_FLAG_CHAININFO)
229 TRACE( " chained to function %p-%p\n",
230 (char *)base + handler_data->chain.BeginAddress,
231 (char *)base + handler_data->chain.EndAddress );
232 function = &handler_data->chain;
233 continue;
235 if (info->flags & (UNW_FLAG_EHANDLER | UNW_FLAG_UHANDLER))
236 TRACE( " handler %p data at %p\n",
237 (char *)base + handler_data->handler, &handler_data->handler + 1 );
238 break;
242 static void dump_scope_table( ULONG64 base, const SCOPE_TABLE *table )
244 unsigned int i;
246 TRACE( "scope table at %p\n", table );
247 for (i = 0; i < table->Count; i++)
248 TRACE( " %u: %p-%p handler %p target %p\n", i,
249 (char *)base + table->ScopeRecord[i].BeginAddress,
250 (char *)base + table->ScopeRecord[i].EndAddress,
251 (char *)base + table->ScopeRecord[i].HandlerAddress,
252 (char *)base + table->ScopeRecord[i].JumpTarget );
256 /***********************************************************************
257 * virtual_unwind
259 static NTSTATUS virtual_unwind( ULONG type, DISPATCHER_CONTEXT *dispatch, CONTEXT *context )
261 LDR_DATA_TABLE_ENTRY *module;
262 NTSTATUS status;
264 dispatch->ImageBase = 0;
265 dispatch->ScopeIndex = 0;
266 dispatch->ControlPc = context->Rip;
268 /* first look for PE exception information */
270 if ((dispatch->FunctionEntry = lookup_function_info( context->Rip, &dispatch->ImageBase, &module )))
272 dispatch->LanguageHandler = RtlVirtualUnwind( type, dispatch->ImageBase, context->Rip,
273 dispatch->FunctionEntry, context,
274 &dispatch->HandlerData, &dispatch->EstablisherFrame,
275 NULL );
276 return STATUS_SUCCESS;
279 /* then look for host system exception information */
281 if (!module || (module->Flags & LDR_WINE_INTERNAL))
283 struct unwind_builtin_dll_params params = { type, dispatch, context };
285 status = WINE_UNIX_CALL( unix_unwind_builtin_dll, &params );
286 if (!status && dispatch->LanguageHandler && !module)
288 FIXME( "calling personality routine in system library not supported yet\n" );
289 dispatch->LanguageHandler = NULL;
291 if (status != STATUS_UNSUCCESSFUL) return status;
293 else WARN( "exception data not found in %s\n", debugstr_w(module->BaseDllName.Buffer) );
295 /* no exception information, treat as a leaf function */
297 dispatch->EstablisherFrame = context->Rsp;
298 dispatch->LanguageHandler = NULL;
299 context->Rip = *(ULONG64 *)context->Rsp;
300 context->Rsp = context->Rsp + sizeof(ULONG64);
301 return STATUS_SUCCESS;
305 /**************************************************************************
306 * __chkstk (NTDLL.@)
308 * Supposed to touch all the stack pages, but we shouldn't need that.
310 __ASM_GLOBAL_FUNC( __chkstk, "ret" );
313 /***********************************************************************
314 * RtlCaptureContext (NTDLL.@)
316 __ASM_GLOBAL_FUNC( RtlCaptureContext,
317 "pushfq\n\t"
318 __ASM_CFI(".cfi_adjust_cfa_offset 8\n\t")
319 "movl $0x10000f,0x30(%rcx)\n\t" /* context->ContextFlags */
320 "stmxcsr 0x34(%rcx)\n\t" /* context->MxCsr */
321 "movw %cs,0x38(%rcx)\n\t" /* context->SegCs */
322 "movw %ds,0x3a(%rcx)\n\t" /* context->SegDs */
323 "movw %es,0x3c(%rcx)\n\t" /* context->SegEs */
324 "movw %fs,0x3e(%rcx)\n\t" /* context->SegFs */
325 "movw %gs,0x40(%rcx)\n\t" /* context->SegGs */
326 "movw %ss,0x42(%rcx)\n\t" /* context->SegSs */
327 "popq 0x44(%rcx)\n\t" /* context->Eflags */
328 __ASM_CFI(".cfi_adjust_cfa_offset -8\n\t")
329 "movq %rax,0x78(%rcx)\n\t" /* context->Rax */
330 "movq %rcx,0x80(%rcx)\n\t" /* context->Rcx */
331 "movq %rdx,0x88(%rcx)\n\t" /* context->Rdx */
332 "movq %rbx,0x90(%rcx)\n\t" /* context->Rbx */
333 "leaq 8(%rsp),%rax\n\t"
334 "movq %rax,0x98(%rcx)\n\t" /* context->Rsp */
335 "movq %rbp,0xa0(%rcx)\n\t" /* context->Rbp */
336 "movq %rsi,0xa8(%rcx)\n\t" /* context->Rsi */
337 "movq %rdi,0xb0(%rcx)\n\t" /* context->Rdi */
338 "movq %r8,0xb8(%rcx)\n\t" /* context->R8 */
339 "movq %r9,0xc0(%rcx)\n\t" /* context->R9 */
340 "movq %r10,0xc8(%rcx)\n\t" /* context->R10 */
341 "movq %r11,0xd0(%rcx)\n\t" /* context->R11 */
342 "movq %r12,0xd8(%rcx)\n\t" /* context->R12 */
343 "movq %r13,0xe0(%rcx)\n\t" /* context->R13 */
344 "movq %r14,0xe8(%rcx)\n\t" /* context->R14 */
345 "movq %r15,0xf0(%rcx)\n\t" /* context->R15 */
346 "movq (%rsp),%rax\n\t"
347 "movq %rax,0xf8(%rcx)\n\t" /* context->Rip */
348 "fxsave 0x100(%rcx)\n\t" /* context->FltSave */
349 "ret" );
351 static DWORD __cdecl nested_exception_handler( EXCEPTION_RECORD *rec, EXCEPTION_REGISTRATION_RECORD *frame,
352 CONTEXT *context, EXCEPTION_REGISTRATION_RECORD **dispatcher )
354 if (!(rec->ExceptionFlags & (EH_UNWINDING | EH_EXIT_UNWIND)))
355 rec->ExceptionFlags |= EH_NESTED_CALL;
357 return ExceptionContinueSearch;
360 /**********************************************************************
361 * call_handler
363 * Call a single exception handler.
364 * FIXME: Handle nested exceptions.
366 static DWORD call_handler( EXCEPTION_RECORD *rec, CONTEXT *context, DISPATCHER_CONTEXT *dispatch )
368 EXCEPTION_REGISTRATION_RECORD frame;
369 DWORD res;
371 frame.Handler = nested_exception_handler;
372 __wine_push_frame( &frame );
374 TRACE_(seh)( "calling handler %p (rec=%p, frame=%p context=%p, dispatch=%p)\n",
375 dispatch->LanguageHandler, rec, (void *)dispatch->EstablisherFrame, dispatch->ContextRecord, dispatch );
376 res = dispatch->LanguageHandler( rec, (void *)dispatch->EstablisherFrame, context, dispatch );
377 TRACE_(seh)( "handler at %p returned %lu\n", dispatch->LanguageHandler, res );
379 rec->ExceptionFlags &= EH_NONCONTINUABLE;
380 __wine_pop_frame( &frame );
381 return res;
385 /**********************************************************************
386 * call_teb_handler
388 * Call a single exception handler from the TEB chain.
389 * FIXME: Handle nested exceptions.
391 static DWORD call_teb_handler( EXCEPTION_RECORD *rec, CONTEXT *context, DISPATCHER_CONTEXT *dispatch,
392 EXCEPTION_REGISTRATION_RECORD *teb_frame )
394 DWORD res;
396 TRACE_(seh)( "calling TEB handler %p (rec=%p, frame=%p context=%p, dispatch=%p)\n",
397 teb_frame->Handler, rec, teb_frame, dispatch->ContextRecord, dispatch );
398 res = teb_frame->Handler( rec, teb_frame, context, (EXCEPTION_REGISTRATION_RECORD**)dispatch );
399 TRACE_(seh)( "handler at %p returned %lu\n", teb_frame->Handler, res );
400 return res;
404 /**********************************************************************
405 * call_stack_handlers
407 * Call the stack handlers chain.
409 static NTSTATUS call_stack_handlers( EXCEPTION_RECORD *rec, CONTEXT *orig_context )
411 EXCEPTION_REGISTRATION_RECORD *teb_frame = NtCurrentTeb()->Tib.ExceptionList;
412 UNWIND_HISTORY_TABLE table;
413 DISPATCHER_CONTEXT dispatch;
414 CONTEXT context;
415 NTSTATUS status;
417 context = *orig_context;
418 context.ContextFlags &= ~0x40; /* Clear xstate flag. */
420 dispatch.TargetIp = 0;
421 dispatch.ContextRecord = &context;
422 dispatch.HistoryTable = &table;
423 for (;;)
425 status = virtual_unwind( UNW_FLAG_EHANDLER, &dispatch, &context );
426 if (status != STATUS_SUCCESS) return status;
428 unwind_done:
429 if (!dispatch.EstablisherFrame) break;
431 if ((dispatch.EstablisherFrame & 7) ||
432 dispatch.EstablisherFrame < (ULONG64)NtCurrentTeb()->Tib.StackLimit ||
433 dispatch.EstablisherFrame > (ULONG64)NtCurrentTeb()->Tib.StackBase)
435 ERR_(seh)( "invalid frame %p (%p-%p)\n", (void *)dispatch.EstablisherFrame,
436 NtCurrentTeb()->Tib.StackLimit, NtCurrentTeb()->Tib.StackBase );
437 rec->ExceptionFlags |= EH_STACK_INVALID;
438 break;
441 if (dispatch.LanguageHandler)
443 switch (call_handler( rec, orig_context, &dispatch ))
445 case ExceptionContinueExecution:
446 if (rec->ExceptionFlags & EH_NONCONTINUABLE) return STATUS_NONCONTINUABLE_EXCEPTION;
447 return STATUS_SUCCESS;
448 case ExceptionContinueSearch:
449 break;
450 case ExceptionNestedException:
451 FIXME_(seh)( "nested exception\n" );
452 break;
453 case ExceptionCollidedUnwind: {
454 ULONG64 frame;
456 context = *dispatch.ContextRecord;
457 dispatch.ContextRecord = &context;
458 RtlVirtualUnwind( UNW_FLAG_NHANDLER, dispatch.ImageBase,
459 dispatch.ControlPc, dispatch.FunctionEntry,
460 &context, NULL, &frame, NULL );
461 goto unwind_done;
463 default:
464 return STATUS_INVALID_DISPOSITION;
467 /* hack: call wine handlers registered in the tib list */
468 else while ((ULONG64)teb_frame < context.Rsp)
470 TRACE_(seh)( "found wine frame %p rsp %p handler %p\n",
471 teb_frame, (void *)context.Rsp, teb_frame->Handler );
472 dispatch.EstablisherFrame = (ULONG64)teb_frame;
473 switch (call_teb_handler( rec, orig_context, &dispatch, teb_frame ))
475 case ExceptionContinueExecution:
476 if (rec->ExceptionFlags & EH_NONCONTINUABLE) return STATUS_NONCONTINUABLE_EXCEPTION;
477 return STATUS_SUCCESS;
478 case ExceptionContinueSearch:
479 break;
480 case ExceptionNestedException:
481 FIXME_(seh)( "nested exception\n" );
482 break;
483 case ExceptionCollidedUnwind: {
484 ULONG64 frame;
486 context = *dispatch.ContextRecord;
487 dispatch.ContextRecord = &context;
488 RtlVirtualUnwind( UNW_FLAG_NHANDLER, dispatch.ImageBase,
489 dispatch.ControlPc, dispatch.FunctionEntry,
490 &context, NULL, &frame, NULL );
491 teb_frame = teb_frame->Prev;
492 goto unwind_done;
494 default:
495 return STATUS_INVALID_DISPOSITION;
497 teb_frame = teb_frame->Prev;
500 if (context.Rsp == (ULONG64)NtCurrentTeb()->Tib.StackBase) break;
502 return STATUS_UNHANDLED_EXCEPTION;
506 NTSTATUS WINAPI dispatch_exception( EXCEPTION_RECORD *rec, CONTEXT *context )
508 NTSTATUS status;
509 DWORD c;
511 if (pWow64PrepareForException) pWow64PrepareForException( rec, context );
513 TRACE_(seh)( "code=%lx flags=%lx addr=%p ip=%Ix\n",
514 rec->ExceptionCode, rec->ExceptionFlags, rec->ExceptionAddress, context->Rip );
515 for (c = 0; c < min( EXCEPTION_MAXIMUM_PARAMETERS, rec->NumberParameters ); c++)
516 TRACE_(seh)( " info[%ld]=%016I64x\n", c, rec->ExceptionInformation[c] );
518 if (rec->ExceptionCode == EXCEPTION_WINE_STUB)
520 if (rec->ExceptionInformation[1] >> 16)
521 MESSAGE( "wine: Call from %p to unimplemented function %s.%s, aborting\n",
522 rec->ExceptionAddress,
523 (char*)rec->ExceptionInformation[0], (char*)rec->ExceptionInformation[1] );
524 else
525 MESSAGE( "wine: Call from %p to unimplemented function %s.%I64d, aborting\n",
526 rec->ExceptionAddress,
527 (char*)rec->ExceptionInformation[0], rec->ExceptionInformation[1] );
529 else if (rec->ExceptionCode == EXCEPTION_WINE_NAME_THREAD && rec->ExceptionInformation[0] == 0x1000)
531 if ((DWORD)rec->ExceptionInformation[2] == -1 || (DWORD)rec->ExceptionInformation[2] == GetCurrentThreadId())
532 WARN_(threadname)( "Thread renamed to %s\n", debugstr_a((char *)rec->ExceptionInformation[1]) );
533 else
534 WARN_(threadname)( "Thread ID %04lx renamed to %s\n", (DWORD)rec->ExceptionInformation[2],
535 debugstr_a((char *)rec->ExceptionInformation[1]) );
537 set_native_thread_name((DWORD)rec->ExceptionInformation[2], (char *)rec->ExceptionInformation[1]);
539 else if (rec->ExceptionCode == DBG_PRINTEXCEPTION_C)
541 WARN_(seh)( "%s\n", debugstr_an((char *)rec->ExceptionInformation[1], rec->ExceptionInformation[0] - 1) );
543 else if (rec->ExceptionCode == DBG_PRINTEXCEPTION_WIDE_C)
545 WARN_(seh)( "%s\n", debugstr_wn((WCHAR *)rec->ExceptionInformation[1], rec->ExceptionInformation[0] - 1) );
547 else
549 if (rec->ExceptionCode == STATUS_ASSERTION_FAILURE)
550 ERR_(seh)( "%s exception (code=%lx) raised\n", debugstr_exception_code(rec->ExceptionCode), rec->ExceptionCode );
551 else
552 WARN_(seh)( "%s exception (code=%lx) raised\n", debugstr_exception_code(rec->ExceptionCode), rec->ExceptionCode );
554 TRACE_(seh)( " rax=%016I64x rbx=%016I64x rcx=%016I64x rdx=%016I64x\n",
555 context->Rax, context->Rbx, context->Rcx, context->Rdx );
556 TRACE_(seh)( " rsi=%016I64x rdi=%016I64x rbp=%016I64x rsp=%016I64x\n",
557 context->Rsi, context->Rdi, context->Rbp, context->Rsp );
558 TRACE_(seh)( " r8=%016I64x r9=%016I64x r10=%016I64x r11=%016I64x\n",
559 context->R8, context->R9, context->R10, context->R11 );
560 TRACE_(seh)( " r12=%016I64x r13=%016I64x r14=%016I64x r15=%016I64x\n",
561 context->R12, context->R13, context->R14, context->R15 );
564 /* Legends of Runeterra depends on having SegDs == SegSs in an exception
565 * handler. */
566 context->SegDs = context->SegSs;
568 if (call_vectored_handlers( rec, context ) == EXCEPTION_CONTINUE_EXECUTION)
569 NtContinue( context, FALSE );
571 if ((status = call_stack_handlers( rec, context )) == STATUS_SUCCESS)
572 NtContinue( context, FALSE );
574 if (status != STATUS_UNHANDLED_EXCEPTION) RtlRaiseStatus( status );
575 return NtRaiseException( rec, context, FALSE );
579 NTSTATUS WINAPI dispatch_wow_exception( EXCEPTION_RECORD *rec_ptr, CONTEXT *context_ptr )
581 char buffer[sizeof(CONTEXT) + sizeof(CONTEXT_EX) + sizeof(XSTATE) + 128];
582 CONTEXT *context;
583 CONTEXT_EX *context_ex;
584 EXCEPTION_RECORD rec = *rec_ptr;
586 RtlInitializeExtendedContext( buffer, context_ptr->ContextFlags, &context_ex );
587 context = RtlLocateLegacyContext( context_ex, NULL );
588 RtlCopyContext( context, context_ptr->ContextFlags, context_ptr );
589 return dispatch_exception( &rec, context );
593 /*******************************************************************
594 * KiUserExceptionDispatcher (NTDLL.@)
596 __ASM_GLOBAL_FUNC( KiUserExceptionDispatcher,
597 "mov 0x98(%rsp),%rcx\n\t" /* context->Rsp */
598 "movw %cs,%ax\n\t"
599 "cmpw %ax,0x38(%rsp)\n\t" /* context->SegCs */
600 "je 1f\n\t"
601 "mov %rsp,%rdx\n\t" /* context */
602 "lea 0x4f0(%rsp),%rcx\n\t" /* rec */
603 "movq %r14,%rsp\n\t" /* switch to 64-bit stack */
604 "call " __ASM_NAME("dispatch_wow_exception") "\n\t"
605 "int3\n"
606 "1:\tmov 0xf8(%rsp),%rdx\n\t" /* context->Rip */
607 "mov %rdx,-0x8(%rcx)\n\t"
608 "mov %rbp,-0x10(%rcx)\n\t"
609 "mov %rdi,-0x18(%rcx)\n\t"
610 "mov %rsi,-0x20(%rcx)\n\t"
611 "lea -0x20(%rcx),%rbp\n\t"
612 "mov %rsp,%rdx\n\t" /* context */
613 "lea 0x4f0(%rsp),%rcx\n\t" /* rec */
614 __ASM_SEH(".seh_pushreg %rbp\n\t")
615 __ASM_SEH(".seh_pushreg %rdi\n\t")
616 __ASM_SEH(".seh_pushreg %rsi\n\t")
617 __ASM_SEH(".seh_setframe %rbp,0\n\t")
618 __ASM_SEH(".seh_endprologue\n\t")
620 __ASM_CFI(".cfi_signal_frame\n\t")
621 __ASM_CFI(".cfi_adjust_cfa_offset 0x20\n\t")
622 __ASM_CFI(".cfi_def_cfa %rbp,0x20\n\t")
623 __ASM_CFI(".cfi_rel_offset %rip,0x18\n\t")
624 __ASM_CFI(".cfi_rel_offset %rbp,0x10\n\t")
625 __ASM_CFI(".cfi_rel_offset %rdi,0x8\n\t")
626 __ASM_CFI(".cfi_rel_offset %rsi,0\n\t")
627 "call " __ASM_NAME("dispatch_exception") "\n\t"
628 "int3")
631 /*******************************************************************
632 * KiUserApcDispatcher (NTDLL.@)
634 void WINAPI dispatch_apc( CONTEXT *context, ULONG_PTR arg1, ULONG_PTR arg2, ULONG_PTR arg3,
635 void (CALLBACK *func)(ULONG_PTR,ULONG_PTR,ULONG_PTR,CONTEXT*) )
637 func( arg1, arg2, arg3, context );
638 NtContinue( context, TRUE );
641 __ASM_GLOBAL_FUNC( KiUserApcDispatcher,
642 "addq $0x8,%rsp\n\t"
643 "mov 0x98(%rcx),%r10\n\t" /* context->Rsp */
644 "mov 0xf8(%rcx),%r11\n\t" /* context->Rip */
645 "mov %r11,-0x8(%r10)\n\t"
646 "mov %rbp,-0x10(%r10)\n\t"
647 "lea -0x10(%r10),%rbp\n\t"
648 __ASM_SEH(".seh_pushreg %rbp\n\t")
649 __ASM_SEH(".seh_setframe %rbp,0\n\t")
650 __ASM_SEH(".seh_endprologue\n\t")
651 __ASM_CFI(".cfi_signal_frame\n\t")
652 __ASM_CFI(".cfi_adjust_cfa_offset 0x10\n\t")
653 __ASM_CFI(".cfi_def_cfa %rbp,0x10\n\t")
654 __ASM_CFI(".cfi_rel_offset %rip,0x8\n\t")
655 __ASM_CFI(".cfi_rel_offset %rbp,0\n\t")
656 "call " __ASM_NAME("dispatch_apc") "\n\t"
657 "int3")
660 /*******************************************************************
661 * KiUserCallbackDispatcher (NTDLL.@)
663 * FIXME: not binary compatible
665 void WINAPI KiUserCallbackDispatcher( ULONG id, void *args, ULONG len )
667 NTSTATUS status;
669 __TRY
671 NTSTATUS (WINAPI *func)(void *, ULONG) = ((void **)NtCurrentTeb()->Peb->KernelCallbackTable)[id];
672 status = NtCallbackReturn( NULL, 0, func( args, len ));
674 __EXCEPT_ALL
676 ERR_(seh)( "ignoring exception\n" );
677 status = NtCallbackReturn( 0, 0, 0 );
679 __ENDTRY
681 RtlRaiseStatus( status );
685 static ULONG64 get_int_reg( CONTEXT *context, int reg )
687 return *(&context->Rax + reg);
690 static void set_int_reg( CONTEXT *context, KNONVOLATILE_CONTEXT_POINTERS *ctx_ptr, int reg, ULONG64 *val )
692 *(&context->Rax + reg) = *val;
693 if (ctx_ptr) ctx_ptr->u2.IntegerContext[reg] = val;
696 static void set_float_reg( CONTEXT *context, KNONVOLATILE_CONTEXT_POINTERS *ctx_ptr, int reg, M128A *val )
698 /* Use a memcpy() to avoid issues if val is misaligned. */
699 memcpy(&context->u.s.Xmm0 + reg, val, sizeof(*val));
700 if (ctx_ptr) ctx_ptr->u.FloatingContext[reg] = val;
703 static int get_opcode_size( struct opcode op )
705 switch (op.code)
707 case UWOP_ALLOC_LARGE:
708 return 2 + (op.info != 0);
709 case UWOP_SAVE_NONVOL:
710 case UWOP_SAVE_XMM128:
711 case UWOP_EPILOG:
712 return 2;
713 case UWOP_SAVE_NONVOL_FAR:
714 case UWOP_SAVE_XMM128_FAR:
715 return 3;
716 default:
717 return 1;
721 static BOOL is_inside_epilog( BYTE *pc, ULONG64 base, const RUNTIME_FUNCTION *function )
723 /* add or lea must be the first instruction, and it must have a rex.W prefix */
724 if ((pc[0] & 0xf8) == 0x48)
726 switch (pc[1])
728 case 0x81: /* add $nnnn,%rsp */
729 if (pc[0] == 0x48 && pc[2] == 0xc4)
731 pc += 7;
732 break;
734 return FALSE;
735 case 0x83: /* add $n,%rsp */
736 if (pc[0] == 0x48 && pc[2] == 0xc4)
738 pc += 4;
739 break;
741 return FALSE;
742 case 0x8d: /* lea n(reg),%rsp */
743 if (pc[0] & 0x06) return FALSE; /* rex.RX must be cleared */
744 if (((pc[2] >> 3) & 7) != 4) return FALSE; /* dest reg mus be %rsp */
745 if ((pc[2] & 7) == 4) return FALSE; /* no SIB byte allowed */
746 if ((pc[2] >> 6) == 1) /* 8-bit offset */
748 pc += 4;
749 break;
751 if ((pc[2] >> 6) == 2) /* 32-bit offset */
753 pc += 7;
754 break;
756 return FALSE;
760 /* now check for various pop instructions */
762 for (;;)
764 if ((*pc & 0xf0) == 0x40) pc++; /* rex prefix */
766 switch (*pc)
768 case 0x58: /* pop %rax/%r8 */
769 case 0x59: /* pop %rcx/%r9 */
770 case 0x5a: /* pop %rdx/%r10 */
771 case 0x5b: /* pop %rbx/%r11 */
772 case 0x5c: /* pop %rsp/%r12 */
773 case 0x5d: /* pop %rbp/%r13 */
774 case 0x5e: /* pop %rsi/%r14 */
775 case 0x5f: /* pop %rdi/%r15 */
776 pc++;
777 continue;
778 case 0xc2: /* ret $nn */
779 case 0xc3: /* ret */
780 return TRUE;
781 case 0xe9: /* jmp nnnn */
782 pc += 5 + *(LONG *)(pc + 1);
783 if (pc - (BYTE *)base >= function->BeginAddress && pc - (BYTE *)base < function->EndAddress)
784 continue;
785 break;
786 case 0xeb: /* jmp n */
787 pc += 2 + (signed char)pc[1];
788 if (pc - (BYTE *)base >= function->BeginAddress && pc - (BYTE *)base < function->EndAddress)
789 continue;
790 break;
791 case 0xf3: /* rep; ret (for amd64 prediction bug) */
792 return pc[1] == 0xc3;
794 return FALSE;
798 /* execute a function epilog, which must have been validated with is_inside_epilog() */
799 static void interpret_epilog( BYTE *pc, CONTEXT *context, KNONVOLATILE_CONTEXT_POINTERS *ctx_ptr )
801 for (;;)
803 BYTE rex = 0;
805 if ((*pc & 0xf0) == 0x40) rex = *pc++ & 0x0f; /* rex prefix */
807 switch (*pc)
809 case 0x58: /* pop %rax/r8 */
810 case 0x59: /* pop %rcx/r9 */
811 case 0x5a: /* pop %rdx/r10 */
812 case 0x5b: /* pop %rbx/r11 */
813 case 0x5c: /* pop %rsp/r12 */
814 case 0x5d: /* pop %rbp/r13 */
815 case 0x5e: /* pop %rsi/r14 */
816 case 0x5f: /* pop %rdi/r15 */
817 set_int_reg( context, ctx_ptr, *pc - 0x58 + (rex & 1) * 8, (ULONG64 *)context->Rsp );
818 context->Rsp += sizeof(ULONG64);
819 pc++;
820 continue;
821 case 0x81: /* add $nnnn,%rsp */
822 context->Rsp += *(LONG *)(pc + 2);
823 pc += 2 + sizeof(LONG);
824 continue;
825 case 0x83: /* add $n,%rsp */
826 context->Rsp += (signed char)pc[2];
827 pc += 3;
828 continue;
829 case 0x8d:
830 if ((pc[1] >> 6) == 1) /* lea n(reg),%rsp */
832 context->Rsp = get_int_reg( context, (pc[1] & 7) + (rex & 1) * 8 ) + (signed char)pc[2];
833 pc += 3;
835 else /* lea nnnn(reg),%rsp */
837 context->Rsp = get_int_reg( context, (pc[1] & 7) + (rex & 1) * 8 ) + *(LONG *)(pc + 2);
838 pc += 2 + sizeof(LONG);
840 continue;
841 case 0xc2: /* ret $nn */
842 context->Rip = *(ULONG64 *)context->Rsp;
843 context->Rsp += sizeof(ULONG64) + *(WORD *)(pc + 1);
844 return;
845 case 0xc3: /* ret */
846 case 0xf3: /* rep; ret */
847 context->Rip = *(ULONG64 *)context->Rsp;
848 context->Rsp += sizeof(ULONG64);
849 return;
850 case 0xe9: /* jmp nnnn */
851 pc += 5 + *(LONG *)(pc + 1);
852 continue;
853 case 0xeb: /* jmp n */
854 pc += 2 + (signed char)pc[1];
855 continue;
857 return;
861 /**********************************************************************
862 * RtlVirtualUnwind (NTDLL.@)
864 PVOID WINAPI RtlVirtualUnwind( ULONG type, ULONG64 base, ULONG64 pc,
865 RUNTIME_FUNCTION *function, CONTEXT *context,
866 PVOID *data, ULONG64 *frame_ret,
867 KNONVOLATILE_CONTEXT_POINTERS *ctx_ptr )
869 union handler_data *handler_data;
870 ULONG64 frame, off;
871 struct UNWIND_INFO *info;
872 unsigned int i, prolog_offset;
873 BOOL mach_frame = FALSE;
875 TRACE( "type %lx rip %I64x rsp %I64x\n", type, pc, context->Rsp );
876 if (TRACE_ON(seh)) dump_unwind_info( base, function );
878 frame = *frame_ret = context->Rsp;
879 for (;;)
881 info = (struct UNWIND_INFO *)((char *)base + function->UnwindData);
882 handler_data = (union handler_data *)&info->opcodes[(info->count + 1) & ~1];
884 if (info->version != 1 && info->version != 2)
886 FIXME( "unknown unwind info version %u at %p\n", info->version, info );
887 return NULL;
890 if (info->frame_reg)
891 frame = get_int_reg( context, info->frame_reg ) - info->frame_offset * 16;
893 /* check if in prolog */
894 if (pc >= base + function->BeginAddress && pc < base + function->BeginAddress + info->prolog)
896 TRACE("inside prolog.\n");
897 prolog_offset = pc - base - function->BeginAddress;
899 else
901 prolog_offset = ~0;
902 /* Since Win10 1809 epilogue does not have a special treatment in case of zero opcode count. */
903 if (info->count && is_inside_epilog( (BYTE *)pc, base, function ))
905 TRACE("inside epilog.\n");
906 interpret_epilog( (BYTE *)pc, context, ctx_ptr );
907 *frame_ret = frame;
908 return NULL;
912 for (i = 0; i < info->count; i += get_opcode_size(info->opcodes[i]))
914 if (prolog_offset < info->opcodes[i].offset) continue; /* skip it */
916 switch (info->opcodes[i].code)
918 case UWOP_PUSH_NONVOL: /* pushq %reg */
919 set_int_reg( context, ctx_ptr, info->opcodes[i].info, (ULONG64 *)context->Rsp );
920 context->Rsp += sizeof(ULONG64);
921 break;
922 case UWOP_ALLOC_LARGE: /* subq $nn,%rsp */
923 if (info->opcodes[i].info) context->Rsp += *(DWORD *)&info->opcodes[i+1];
924 else context->Rsp += *(USHORT *)&info->opcodes[i+1] * 8;
925 break;
926 case UWOP_ALLOC_SMALL: /* subq $n,%rsp */
927 context->Rsp += (info->opcodes[i].info + 1) * 8;
928 break;
929 case UWOP_SET_FPREG: /* leaq nn(%rsp),%framereg */
930 context->Rsp = *frame_ret = frame;
931 break;
932 case UWOP_SAVE_NONVOL: /* movq %reg,n(%rsp) */
933 off = frame + *(USHORT *)&info->opcodes[i+1] * 8;
934 set_int_reg( context, ctx_ptr, info->opcodes[i].info, (ULONG64 *)off );
935 break;
936 case UWOP_SAVE_NONVOL_FAR: /* movq %reg,nn(%rsp) */
937 off = frame + *(DWORD *)&info->opcodes[i+1];
938 set_int_reg( context, ctx_ptr, info->opcodes[i].info, (ULONG64 *)off );
939 break;
940 case UWOP_SAVE_XMM128: /* movaps %xmmreg,n(%rsp) */
941 off = frame + *(USHORT *)&info->opcodes[i+1] * 16;
942 set_float_reg( context, ctx_ptr, info->opcodes[i].info, (M128A *)off );
943 break;
944 case UWOP_SAVE_XMM128_FAR: /* movaps %xmmreg,nn(%rsp) */
945 off = frame + *(DWORD *)&info->opcodes[i+1];
946 set_float_reg( context, ctx_ptr, info->opcodes[i].info, (M128A *)off );
947 break;
948 case UWOP_PUSH_MACHFRAME:
949 if (info->flags & UNW_FLAG_CHAININFO)
951 FIXME("PUSH_MACHFRAME with chained unwind info.\n");
952 break;
954 if (i + get_opcode_size(info->opcodes[i]) < info->count )
956 FIXME("PUSH_MACHFRAME is not the last opcode.\n");
957 break;
960 if (info->opcodes[i].info)
961 context->Rsp += 0x8;
963 context->Rip = *(ULONG64 *)context->Rsp;
964 context->Rsp = *(ULONG64 *)(context->Rsp + 24);
965 mach_frame = TRUE;
966 break;
967 case UWOP_EPILOG:
968 if (info->version == 2)
969 break; /* nothing to do */
970 default:
971 FIXME( "unknown code %u\n", info->opcodes[i].code );
972 break;
976 if (!(info->flags & UNW_FLAG_CHAININFO)) break;
977 function = &handler_data->chain; /* restart with the chained info */
980 if (!mach_frame)
982 /* now pop return address */
983 context->Rip = *(ULONG64 *)context->Rsp;
984 context->Rsp += sizeof(ULONG64);
987 if (!(info->flags & type)) return NULL; /* no matching handler */
988 if (prolog_offset != ~0) return NULL; /* inside prolog */
990 *data = &handler_data->handler + 1;
991 return (char *)base + handler_data->handler;
994 struct unwind_exception_frame
996 EXCEPTION_REGISTRATION_RECORD frame;
997 DISPATCHER_CONTEXT *dispatch;
1000 /**********************************************************************
1001 * unwind_exception_handler
1003 * Handler for exceptions happening while calling an unwind handler.
1005 static DWORD __cdecl unwind_exception_handler( EXCEPTION_RECORD *rec, EXCEPTION_REGISTRATION_RECORD *frame,
1006 CONTEXT *context, EXCEPTION_REGISTRATION_RECORD **dispatcher )
1008 struct unwind_exception_frame *unwind_frame = (struct unwind_exception_frame *)frame;
1009 DISPATCHER_CONTEXT *dispatch = (DISPATCHER_CONTEXT *)dispatcher;
1011 /* copy the original dispatcher into the current one, except for the TargetIp */
1012 dispatch->ControlPc = unwind_frame->dispatch->ControlPc;
1013 dispatch->ImageBase = unwind_frame->dispatch->ImageBase;
1014 dispatch->FunctionEntry = unwind_frame->dispatch->FunctionEntry;
1015 dispatch->EstablisherFrame = unwind_frame->dispatch->EstablisherFrame;
1016 dispatch->ContextRecord = unwind_frame->dispatch->ContextRecord;
1017 dispatch->LanguageHandler = unwind_frame->dispatch->LanguageHandler;
1018 dispatch->HandlerData = unwind_frame->dispatch->HandlerData;
1019 dispatch->HistoryTable = unwind_frame->dispatch->HistoryTable;
1020 dispatch->ScopeIndex = unwind_frame->dispatch->ScopeIndex;
1021 TRACE( "detected collided unwind\n" );
1022 return ExceptionCollidedUnwind;
1025 /**********************************************************************
1026 * call_unwind_handler
1028 * Call a single unwind handler.
1030 static DWORD call_unwind_handler( EXCEPTION_RECORD *rec, DISPATCHER_CONTEXT *dispatch )
1032 struct unwind_exception_frame frame;
1033 DWORD res;
1035 frame.frame.Handler = unwind_exception_handler;
1036 frame.dispatch = dispatch;
1037 __wine_push_frame( &frame.frame );
1039 TRACE( "calling handler %p (rec=%p, frame=%p context=%p, dispatch=%p)\n",
1040 dispatch->LanguageHandler, rec, (void *)dispatch->EstablisherFrame, dispatch->ContextRecord, dispatch );
1041 res = dispatch->LanguageHandler( rec, (void *)dispatch->EstablisherFrame, dispatch->ContextRecord, dispatch );
1042 TRACE( "handler %p returned %lx\n", dispatch->LanguageHandler, res );
1044 __wine_pop_frame( &frame.frame );
1046 switch (res)
1048 case ExceptionContinueSearch:
1049 case ExceptionCollidedUnwind:
1050 break;
1051 default:
1052 raise_status( STATUS_INVALID_DISPOSITION, rec );
1053 break;
1056 return res;
1060 /**********************************************************************
1061 * call_teb_unwind_handler
1063 * Call a single unwind handler from the TEB chain.
1065 static DWORD call_teb_unwind_handler( EXCEPTION_RECORD *rec, DISPATCHER_CONTEXT *dispatch,
1066 EXCEPTION_REGISTRATION_RECORD *teb_frame )
1068 DWORD res;
1070 TRACE( "calling TEB handler %p (rec=%p, frame=%p context=%p, dispatch=%p)\n",
1071 teb_frame->Handler, rec, teb_frame, dispatch->ContextRecord, dispatch );
1072 res = teb_frame->Handler( rec, teb_frame, dispatch->ContextRecord, (EXCEPTION_REGISTRATION_RECORD**)dispatch );
1073 TRACE( "handler at %p returned %lu\n", teb_frame->Handler, res );
1075 switch (res)
1077 case ExceptionContinueSearch:
1078 case ExceptionCollidedUnwind:
1079 break;
1080 default:
1081 raise_status( STATUS_INVALID_DISPOSITION, rec );
1082 break;
1085 return res;
1089 /**********************************************************************
1090 * call_consolidate_callback
1092 * Wrapper function to call a consolidate callback from a fake frame.
1093 * If the callback executes RtlUnwindEx (like for example done in C++ handlers),
1094 * we have to skip all frames which were already processed. To do that we
1095 * trick the unwinding functions into thinking the call came from the specified
1096 * context. All CFI instructions are either DW_CFA_def_cfa_expression or
1097 * DW_CFA_expression, and the expressions have the following format:
1099 * DW_OP_breg6; sleb128 0x10 | Load %rbp + 0x10
1100 * DW_OP_deref | Get *(%rbp + 0x10) == context
1101 * DW_OP_plus_uconst; uleb128 <OFFSET> | Add offset to get struct member
1102 * [DW_OP_deref] | Dereference, only for CFA
1104 extern void * WINAPI call_consolidate_callback( CONTEXT *context,
1105 void *(CALLBACK *callback)(EXCEPTION_RECORD *),
1106 EXCEPTION_RECORD *rec );
1107 __ASM_GLOBAL_FUNC( call_consolidate_callback,
1108 "pushq %rbp\n\t"
1109 __ASM_CFI(".cfi_adjust_cfa_offset 8\n\t")
1110 __ASM_CFI(".cfi_rel_offset %rbp,0\n\t")
1111 "movq %rsp,%rbp\n\t"
1112 __ASM_CFI(".cfi_def_cfa_register %rbp\n\t")
1114 /* Setup SEH machine frame. */
1115 "subq $0x28,%rsp\n\t"
1116 __ASM_CFI(".cfi_adjust_cfa_offset 0x28\n\t")
1117 "movq 0xf8(%rcx),%rax\n\t" /* Context->Rip */
1118 "movq %rax,(%rsp)\n\t"
1119 "movq 0x98(%rcx),%rax\n\t" /* context->Rsp */
1120 "movq %rax,0x18(%rsp)\n\t"
1121 __ASM_SEH(".seh_pushframe\n\t")
1122 __ASM_SEH(".seh_endprologue\n\t")
1124 "subq $0x108,%rsp\n\t" /* 10*16 (float regs) + 8*8 (int regs) + 32 (shadow store) + 8 (align). */
1125 __ASM_SEH(".seh_stackalloc 0x108\n\t")
1126 __ASM_CFI(".cfi_adjust_cfa_offset 0x108\n\t")
1128 /* Setup CFI unwind to context. */
1129 "movq %rcx,0x10(%rbp)\n\t"
1130 __ASM_CFI(".cfi_remember_state\n\t")
1131 __ASM_CFI(".cfi_escape 0x0f,0x07,0x76,0x10,0x06,0x23,0x98,0x01,0x06\n\t") /* CFA */
1132 __ASM_CFI(".cfi_escape 0x10,0x03,0x06,0x76,0x10,0x06,0x23,0x90,0x01\n\t") /* %rbx */
1133 __ASM_CFI(".cfi_escape 0x10,0x04,0x06,0x76,0x10,0x06,0x23,0xa8,0x01\n\t") /* %rsi */
1134 __ASM_CFI(".cfi_escape 0x10,0x05,0x06,0x76,0x10,0x06,0x23,0xb0,0x01\n\t") /* %rdi */
1135 __ASM_CFI(".cfi_escape 0x10,0x06,0x06,0x76,0x10,0x06,0x23,0xa0,0x01\n\t") /* %rbp */
1136 __ASM_CFI(".cfi_escape 0x10,0x0c,0x06,0x76,0x10,0x06,0x23,0xd8,0x01\n\t") /* %r12 */
1137 __ASM_CFI(".cfi_escape 0x10,0x0d,0x06,0x76,0x10,0x06,0x23,0xe0,0x01\n\t") /* %r13 */
1138 __ASM_CFI(".cfi_escape 0x10,0x0e,0x06,0x76,0x10,0x06,0x23,0xe8,0x01\n\t") /* %r14 */
1139 __ASM_CFI(".cfi_escape 0x10,0x0f,0x06,0x76,0x10,0x06,0x23,0xf0,0x01\n\t") /* %r15 */
1140 __ASM_CFI(".cfi_escape 0x10,0x10,0x06,0x76,0x10,0x06,0x23,0xf8,0x01\n\t") /* %rip */
1141 __ASM_CFI(".cfi_escape 0x10,0x17,0x06,0x76,0x10,0x06,0x23,0x80,0x04\n\t") /* %xmm6 */
1142 __ASM_CFI(".cfi_escape 0x10,0x18,0x06,0x76,0x10,0x06,0x23,0x90,0x04\n\t") /* %xmm7 */
1143 __ASM_CFI(".cfi_escape 0x10,0x19,0x06,0x76,0x10,0x06,0x23,0xa0,0x04\n\t") /* %xmm8 */
1144 __ASM_CFI(".cfi_escape 0x10,0x1a,0x06,0x76,0x10,0x06,0x23,0xb0,0x04\n\t") /* %xmm9 */
1145 __ASM_CFI(".cfi_escape 0x10,0x1b,0x06,0x76,0x10,0x06,0x23,0xc0,0x04\n\t") /* %xmm10 */
1146 __ASM_CFI(".cfi_escape 0x10,0x1c,0x06,0x76,0x10,0x06,0x23,0xd0,0x04\n\t") /* %xmm11 */
1147 __ASM_CFI(".cfi_escape 0x10,0x1d,0x06,0x76,0x10,0x06,0x23,0xe0,0x04\n\t") /* %xmm12 */
1148 __ASM_CFI(".cfi_escape 0x10,0x1e,0x06,0x76,0x10,0x06,0x23,0xf0,0x04\n\t") /* %xmm13 */
1149 __ASM_CFI(".cfi_escape 0x10,0x1f,0x06,0x76,0x10,0x06,0x23,0x80,0x05\n\t") /* %xmm14 */
1150 __ASM_CFI(".cfi_escape 0x10,0x20,0x06,0x76,0x10,0x06,0x23,0x90,0x05\n\t") /* %xmm15 */
1152 /* Setup SEH unwind registers restore. */
1153 "movq 0xa0(%rcx),%rax\n\t" /* context->Rbp */
1154 "movq %rax,0x100(%rsp)\n\t"
1155 __ASM_SEH(".seh_savereg %rbp, 0x100\n\t")
1156 "movq 0x90(%rcx),%rax\n\t" /* context->Rbx */
1157 "movq %rax,0x20(%rsp)\n\t"
1158 __ASM_SEH(".seh_savereg %rbx, 0x20\n\t")
1159 "movq 0xa8(%rcx),%rax\n\t" /* context->Rsi */
1160 "movq %rax,0x28(%rsp)\n\t"
1161 __ASM_SEH(".seh_savereg %rsi, 0x28\n\t")
1162 "movq 0xb0(%rcx),%rax\n\t" /* context->Rdi */
1163 "movq %rax,0x30(%rsp)\n\t"
1164 __ASM_SEH(".seh_savereg %rdi, 0x30\n\t")
1166 "movq 0xd8(%rcx),%rax\n\t" /* context->R12 */
1167 "movq %rax,0x38(%rsp)\n\t"
1168 __ASM_SEH(".seh_savereg %r12, 0x38\n\t")
1169 "movq 0xe0(%rcx),%rax\n\t" /* context->R13 */
1170 "movq %rax,0x40(%rsp)\n\t"
1171 __ASM_SEH(".seh_savereg %r13, 0x40\n\t")
1172 "movq 0xe8(%rcx),%rax\n\t" /* context->R14 */
1173 "movq %rax,0x48(%rsp)\n\t"
1174 __ASM_SEH(".seh_savereg %r14, 0x48\n\t")
1175 "movq 0xf0(%rcx),%rax\n\t" /* context->R15 */
1176 "movq %rax,0x50(%rsp)\n\t"
1177 __ASM_SEH(".seh_savereg %r15, 0x50\n\t")
1178 "pushq %rsi\n\t"
1179 "pushq %rdi\n\t"
1180 "leaq 0x200(%rcx),%rsi\n\t"
1181 "leaq 0x70(%rsp),%rdi\n\t"
1182 "movq $0x14,%rcx\n\t"
1183 "cld\n\t"
1184 "rep; movsq\n\t"
1185 "popq %rdi\n\t"
1186 "popq %rsi\n\t"
1187 __ASM_SEH(".seh_savexmm %xmm6, 0x60\n\t")
1188 __ASM_SEH(".seh_savexmm %xmm7, 0x70\n\t")
1189 __ASM_SEH(".seh_savexmm %xmm8, 0x80\n\t")
1190 __ASM_SEH(".seh_savexmm %xmm9, 0x90\n\t")
1191 __ASM_SEH(".seh_savexmm %xmm10, 0xa0\n\t")
1192 __ASM_SEH(".seh_savexmm %xmm11, 0xb0\n\t")
1193 __ASM_SEH(".seh_savexmm %xmm12, 0xc0\n\t")
1194 __ASM_SEH(".seh_savexmm %xmm13, 0xd0\n\t")
1195 __ASM_SEH(".seh_savexmm %xmm14, 0xe0\n\t")
1196 __ASM_SEH(".seh_savexmm %xmm15, 0xf0\n\t")
1198 /* call the callback. */
1199 "movq %r8,%rcx\n\t"
1200 "callq *%rdx\n\t"
1201 __ASM_CFI(".cfi_restore_state\n\t")
1202 "nop\n\t" /* Otherwise RtlVirtualUnwind() will think we are inside epilogue and
1203 * interpret / execute the rest of opcodes here instead of unwind through
1204 * machine frame. */
1205 "leaq 0(%rbp),%rsp\n\t"
1206 __ASM_CFI(".cfi_def_cfa_register %rsp\n\t")
1207 "popq %rbp\n\t"
1208 __ASM_CFI(".cfi_adjust_cfa_offset -8\n\t")
1209 __ASM_CFI(".cfi_same_value %rbp\n\t")
1210 "ret")
1212 /*******************************************************************
1213 * RtlRestoreContext (NTDLL.@)
1215 void CDECL RtlRestoreContext( CONTEXT *context, EXCEPTION_RECORD *rec )
1217 EXCEPTION_REGISTRATION_RECORD *teb_frame = NtCurrentTeb()->Tib.ExceptionList;
1219 if (rec && rec->ExceptionCode == STATUS_LONGJUMP && rec->NumberParameters >= 1)
1221 struct MSVCRT_JUMP_BUFFER *jmp = (struct MSVCRT_JUMP_BUFFER *)rec->ExceptionInformation[0];
1222 context->Rbx = jmp->Rbx;
1223 context->Rsp = jmp->Rsp;
1224 context->Rbp = jmp->Rbp;
1225 context->Rsi = jmp->Rsi;
1226 context->Rdi = jmp->Rdi;
1227 context->R12 = jmp->R12;
1228 context->R13 = jmp->R13;
1229 context->R14 = jmp->R14;
1230 context->R15 = jmp->R15;
1231 context->Rip = jmp->Rip;
1232 context->u.s.Xmm6 = jmp->Xmm6;
1233 context->u.s.Xmm7 = jmp->Xmm7;
1234 context->u.s.Xmm8 = jmp->Xmm8;
1235 context->u.s.Xmm9 = jmp->Xmm9;
1236 context->u.s.Xmm10 = jmp->Xmm10;
1237 context->u.s.Xmm11 = jmp->Xmm11;
1238 context->u.s.Xmm12 = jmp->Xmm12;
1239 context->u.s.Xmm13 = jmp->Xmm13;
1240 context->u.s.Xmm14 = jmp->Xmm14;
1241 context->u.s.Xmm15 = jmp->Xmm15;
1242 context->MxCsr = jmp->MxCsr;
1243 context->u.FltSave.MxCsr = jmp->MxCsr;
1244 context->u.FltSave.ControlWord = jmp->FpCsr;
1246 else if (rec && rec->ExceptionCode == STATUS_UNWIND_CONSOLIDATE && rec->NumberParameters >= 1)
1248 PVOID (CALLBACK *consolidate)(EXCEPTION_RECORD *) = (void *)rec->ExceptionInformation[0];
1249 TRACE_(seh)( "calling consolidate callback %p (rec=%p)\n", consolidate, rec );
1250 context->Rip = (ULONG64)call_consolidate_callback( context, consolidate, rec );
1253 /* hack: remove no longer accessible TEB frames */
1254 while ((ULONG64)teb_frame < context->Rsp)
1256 TRACE_(seh)( "removing TEB frame: %p\n", teb_frame );
1257 teb_frame = __wine_pop_frame( teb_frame );
1260 TRACE_(seh)( "returning to %p stack %p\n", (void *)context->Rip, (void *)context->Rsp );
1261 NtContinue( context, FALSE );
1265 /*******************************************************************
1266 * RtlUnwindEx (NTDLL.@)
1268 void WINAPI RtlUnwindEx( PVOID end_frame, PVOID target_ip, EXCEPTION_RECORD *rec,
1269 PVOID retval, CONTEXT *context, UNWIND_HISTORY_TABLE *table )
1271 EXCEPTION_REGISTRATION_RECORD *teb_frame = NtCurrentTeb()->Tib.ExceptionList;
1272 EXCEPTION_RECORD record;
1273 DISPATCHER_CONTEXT dispatch;
1274 CONTEXT new_context;
1275 NTSTATUS status;
1276 DWORD i;
1278 RtlCaptureContext( context );
1279 new_context = *context;
1281 /* build an exception record, if we do not have one */
1282 if (!rec)
1284 record.ExceptionCode = STATUS_UNWIND;
1285 record.ExceptionFlags = 0;
1286 record.ExceptionRecord = NULL;
1287 record.ExceptionAddress = (void *)context->Rip;
1288 record.NumberParameters = 0;
1289 rec = &record;
1292 rec->ExceptionFlags |= EH_UNWINDING | (end_frame ? 0 : EH_EXIT_UNWIND);
1294 TRACE( "code=%lx flags=%lx end_frame=%p target_ip=%p rip=%016I64x\n",
1295 rec->ExceptionCode, rec->ExceptionFlags, end_frame, target_ip, context->Rip );
1296 for (i = 0; i < min( EXCEPTION_MAXIMUM_PARAMETERS, rec->NumberParameters ); i++)
1297 TRACE( " info[%ld]=%016I64x\n", i, rec->ExceptionInformation[i] );
1298 TRACE(" rax=%016I64x rbx=%016I64x rcx=%016I64x rdx=%016I64x\n",
1299 context->Rax, context->Rbx, context->Rcx, context->Rdx );
1300 TRACE(" rsi=%016I64x rdi=%016I64x rbp=%016I64x rsp=%016I64x\n",
1301 context->Rsi, context->Rdi, context->Rbp, context->Rsp );
1302 TRACE(" r8=%016I64x r9=%016I64x r10=%016I64x r11=%016I64x\n",
1303 context->R8, context->R9, context->R10, context->R11 );
1304 TRACE(" r12=%016I64x r13=%016I64x r14=%016I64x r15=%016I64x\n",
1305 context->R12, context->R13, context->R14, context->R15 );
1307 dispatch.EstablisherFrame = context->Rsp;
1308 dispatch.TargetIp = (ULONG64)target_ip;
1309 dispatch.ContextRecord = context;
1310 dispatch.HistoryTable = table;
1312 for (;;)
1314 status = virtual_unwind( UNW_FLAG_UHANDLER, &dispatch, &new_context );
1315 if (status != STATUS_SUCCESS) raise_status( status, rec );
1317 unwind_done:
1318 if (!dispatch.EstablisherFrame) break;
1320 if ((dispatch.EstablisherFrame & 7) ||
1321 dispatch.EstablisherFrame < (ULONG64)NtCurrentTeb()->Tib.StackLimit ||
1322 dispatch.EstablisherFrame > (ULONG64)NtCurrentTeb()->Tib.StackBase)
1324 ERR( "invalid frame %p (%p-%p)\n", (void *)dispatch.EstablisherFrame,
1325 NtCurrentTeb()->Tib.StackLimit, NtCurrentTeb()->Tib.StackBase );
1326 rec->ExceptionFlags |= EH_STACK_INVALID;
1327 break;
1330 if (dispatch.LanguageHandler)
1332 if (end_frame && (dispatch.EstablisherFrame > (ULONG64)end_frame))
1334 ERR( "invalid end frame %p/%p\n", (void *)dispatch.EstablisherFrame, end_frame );
1335 raise_status( STATUS_INVALID_UNWIND_TARGET, rec );
1337 if (dispatch.EstablisherFrame == (ULONG64)end_frame) rec->ExceptionFlags |= EH_TARGET_UNWIND;
1338 if (call_unwind_handler( rec, &dispatch ) == ExceptionCollidedUnwind)
1340 ULONG64 frame;
1342 new_context = *dispatch.ContextRecord;
1343 new_context.ContextFlags &= ~0x40;
1344 *context = new_context;
1345 dispatch.ContextRecord = context;
1346 RtlVirtualUnwind( UNW_FLAG_NHANDLER, dispatch.ImageBase,
1347 dispatch.ControlPc, dispatch.FunctionEntry,
1348 &new_context, NULL, &frame, NULL );
1349 rec->ExceptionFlags |= EH_COLLIDED_UNWIND;
1350 goto unwind_done;
1352 rec->ExceptionFlags &= ~EH_COLLIDED_UNWIND;
1354 else /* hack: call builtin handlers registered in the tib list */
1356 DWORD64 backup_frame = dispatch.EstablisherFrame;
1357 while ((ULONG64)teb_frame < new_context.Rsp && (ULONG64)teb_frame < (ULONG64)end_frame)
1359 TRACE( "found builtin frame %p handler %p\n", teb_frame, teb_frame->Handler );
1360 dispatch.EstablisherFrame = (ULONG64)teb_frame;
1361 if (call_teb_unwind_handler( rec, &dispatch, teb_frame ) == ExceptionCollidedUnwind)
1363 ULONG64 frame;
1365 teb_frame = __wine_pop_frame( teb_frame );
1367 new_context = *dispatch.ContextRecord;
1368 new_context.ContextFlags &= ~0x40;
1369 *context = new_context;
1370 dispatch.ContextRecord = context;
1371 RtlVirtualUnwind( UNW_FLAG_NHANDLER, dispatch.ImageBase,
1372 dispatch.ControlPc, dispatch.FunctionEntry,
1373 &new_context, NULL, &frame, NULL );
1374 rec->ExceptionFlags |= EH_COLLIDED_UNWIND;
1375 goto unwind_done;
1377 teb_frame = __wine_pop_frame( teb_frame );
1379 if ((ULONG64)teb_frame == (ULONG64)end_frame && (ULONG64)end_frame < new_context.Rsp) break;
1380 dispatch.EstablisherFrame = backup_frame;
1383 if (dispatch.EstablisherFrame == (ULONG64)end_frame) break;
1384 *context = new_context;
1387 context->Rax = (ULONG64)retval;
1388 context->Rip = (ULONG64)target_ip;
1389 RtlRestoreContext(context, rec);
1393 /*******************************************************************
1394 * RtlUnwind (NTDLL.@)
1396 void WINAPI RtlUnwind( void *frame, void *target_ip, EXCEPTION_RECORD *rec, void *retval )
1398 CONTEXT context;
1399 RtlUnwindEx( frame, target_ip, rec, retval, &context, NULL );
1403 /*******************************************************************
1404 * _local_unwind (NTDLL.@)
1406 void WINAPI _local_unwind( void *frame, void *target_ip )
1408 CONTEXT context;
1409 RtlUnwindEx( frame, target_ip, NULL, NULL, &context, NULL );
1412 /*******************************************************************
1413 * __C_specific_handler (NTDLL.@)
1415 EXCEPTION_DISPOSITION WINAPI __C_specific_handler( EXCEPTION_RECORD *rec,
1416 void *frame,
1417 CONTEXT *context,
1418 struct _DISPATCHER_CONTEXT *dispatch )
1420 SCOPE_TABLE *table = dispatch->HandlerData;
1421 ULONG i;
1423 TRACE_(seh)( "%p %p %p %p\n", rec, frame, context, dispatch );
1424 if (TRACE_ON(seh)) dump_scope_table( dispatch->ImageBase, table );
1426 if (rec->ExceptionFlags & (EH_UNWINDING | EH_EXIT_UNWIND))
1428 for (i = dispatch->ScopeIndex; i < table->Count; i++)
1430 if (dispatch->ControlPc >= dispatch->ImageBase + table->ScopeRecord[i].BeginAddress &&
1431 dispatch->ControlPc < dispatch->ImageBase + table->ScopeRecord[i].EndAddress)
1433 PTERMINATION_HANDLER handler;
1435 if (table->ScopeRecord[i].JumpTarget) continue;
1437 if (rec->ExceptionFlags & EH_TARGET_UNWIND &&
1438 dispatch->TargetIp >= dispatch->ImageBase + table->ScopeRecord[i].BeginAddress &&
1439 dispatch->TargetIp < dispatch->ImageBase + table->ScopeRecord[i].EndAddress)
1441 break;
1444 handler = (PTERMINATION_HANDLER)(dispatch->ImageBase + table->ScopeRecord[i].HandlerAddress);
1445 dispatch->ScopeIndex = i+1;
1447 TRACE_(seh)( "calling __finally %p frame %p\n", handler, frame );
1448 handler( TRUE, frame );
1451 return ExceptionContinueSearch;
1454 for (i = dispatch->ScopeIndex; i < table->Count; i++)
1456 if (dispatch->ControlPc >= dispatch->ImageBase + table->ScopeRecord[i].BeginAddress &&
1457 dispatch->ControlPc < dispatch->ImageBase + table->ScopeRecord[i].EndAddress)
1459 if (!table->ScopeRecord[i].JumpTarget) continue;
1460 if (table->ScopeRecord[i].HandlerAddress != EXCEPTION_EXECUTE_HANDLER)
1462 EXCEPTION_POINTERS ptrs;
1463 PEXCEPTION_FILTER filter;
1465 filter = (PEXCEPTION_FILTER)(dispatch->ImageBase + table->ScopeRecord[i].HandlerAddress);
1466 ptrs.ExceptionRecord = rec;
1467 ptrs.ContextRecord = context;
1468 TRACE_(seh)( "calling filter %p ptrs %p frame %p\n", filter, &ptrs, frame );
1469 switch (filter( &ptrs, frame ))
1471 case EXCEPTION_EXECUTE_HANDLER:
1472 break;
1473 case EXCEPTION_CONTINUE_SEARCH:
1474 continue;
1475 case EXCEPTION_CONTINUE_EXECUTION:
1476 return ExceptionContinueExecution;
1479 TRACE( "unwinding to target %p\n", (char *)dispatch->ImageBase + table->ScopeRecord[i].JumpTarget );
1480 RtlUnwindEx( frame, (char *)dispatch->ImageBase + table->ScopeRecord[i].JumpTarget,
1481 rec, 0, dispatch->ContextRecord, dispatch->HistoryTable );
1484 return ExceptionContinueSearch;
1488 /***********************************************************************
1489 * RtlRaiseException (NTDLL.@)
1491 __ASM_GLOBAL_FUNC( RtlRaiseException,
1492 "sub $0x4f8,%rsp\n\t"
1493 __ASM_SEH(".seh_stackalloc 0x4f8\n\t")
1494 __ASM_SEH(".seh_endprologue\n\t")
1495 __ASM_CFI(".cfi_adjust_cfa_offset 0x4f8\n\t")
1496 "movq %rcx,0x500(%rsp)\n\t"
1497 "leaq 0x20(%rsp),%rcx\n\t"
1498 "call " __ASM_NAME("RtlCaptureContext") "\n\t"
1499 "leaq 0x20(%rsp),%rdx\n\t" /* context pointer */
1500 "leaq 0x500(%rsp),%rax\n\t" /* orig stack pointer */
1501 "movq %rax,0x98(%rdx)\n\t" /* context->Rsp */
1502 "movq (%rax),%rcx\n\t" /* original first parameter */
1503 "movq %rcx,0x80(%rdx)\n\t" /* context->Rcx */
1504 "movq 0x4f8(%rsp),%rax\n\t" /* return address */
1505 "movq %rax,0xf8(%rdx)\n\t" /* context->Rip */
1506 "movq %rax,0x10(%rcx)\n\t" /* rec->ExceptionAddress */
1507 "movl $1,%r8d\n\t"
1508 "movq %gs:(0x30),%rax\n\t" /* Teb */
1509 "movq 0x60(%rax),%rax\n\t" /* Peb */
1510 "cmpb $0,0x02(%rax)\n\t" /* BeingDebugged */
1511 "jne 1f\n\t"
1512 "call " __ASM_NAME("dispatch_exception") "\n"
1513 "1:\tcall " __ASM_NAME("NtRaiseException") "\n\t"
1514 "movq %rax,%rcx\n\t"
1515 "call " __ASM_NAME("RtlRaiseStatus") /* does not return */ );
1518 static inline ULONG hash_pointers( void **ptrs, ULONG count )
1520 /* Based on MurmurHash2, which is in the public domain */
1521 static const ULONG m = 0x5bd1e995;
1522 static const ULONG r = 24;
1523 ULONG hash = count * sizeof(void*);
1524 for (; count > 0; ptrs++, count--)
1526 ULONG_PTR data = (ULONG_PTR)*ptrs;
1527 ULONG k1 = (ULONG)(data & 0xffffffff), k2 = (ULONG)(data >> 32);
1528 k1 *= m;
1529 k1 = (k1 ^ (k1 >> r)) * m;
1530 k2 *= m;
1531 k2 = (k2 ^ (k2 >> r)) * m;
1532 hash = (((hash * m) ^ k1) * m) ^ k2;
1534 hash = (hash ^ (hash >> 13)) * m;
1535 return hash ^ (hash >> 15);
1539 /*************************************************************************
1540 * RtlCaptureStackBackTrace (NTDLL.@)
1542 USHORT WINAPI RtlCaptureStackBackTrace( ULONG skip, ULONG count, PVOID *buffer, ULONG *hash )
1544 UNWIND_HISTORY_TABLE table;
1545 DISPATCHER_CONTEXT dispatch;
1546 CONTEXT context;
1547 NTSTATUS status;
1548 ULONG i;
1549 USHORT num_entries = 0;
1551 TRACE( "(%lu, %lu, %p, %p)\n", skip, count, buffer, hash );
1553 RtlCaptureContext( &context );
1554 dispatch.TargetIp = 0;
1555 dispatch.ContextRecord = &context;
1556 dispatch.HistoryTable = &table;
1557 if (hash) *hash = 0;
1558 for (i = 0; i < skip + count; i++)
1560 status = virtual_unwind( UNW_FLAG_NHANDLER, &dispatch, &context );
1561 if (status != STATUS_SUCCESS) return i;
1563 if (!dispatch.EstablisherFrame) break;
1565 if ((dispatch.EstablisherFrame & 7) ||
1566 dispatch.EstablisherFrame < (ULONG64)NtCurrentTeb()->Tib.StackLimit ||
1567 dispatch.EstablisherFrame > (ULONG64)NtCurrentTeb()->Tib.StackBase)
1569 ERR( "invalid frame %p (%p-%p)\n", (void *)dispatch.EstablisherFrame,
1570 NtCurrentTeb()->Tib.StackLimit, NtCurrentTeb()->Tib.StackBase );
1571 break;
1574 if (context.Rsp == (ULONG64)NtCurrentTeb()->Tib.StackBase) break;
1576 if (i >= skip) buffer[num_entries++] = (void *)context.Rip;
1578 if (hash && num_entries > 0) *hash = hash_pointers( buffer, num_entries );
1579 TRACE( "captured %hu frames\n", num_entries );
1580 return num_entries;
1584 /***********************************************************************
1585 * signal_start_thread
1587 __ASM_GLOBAL_FUNC( signal_start_thread,
1588 "movq %rcx,%rbx\n\t" /* context */
1589 /* clear the thread stack */
1590 "andq $~0xfff,%rcx\n\t" /* round down to page size */
1591 "leaq -0xf0000(%rcx),%rdi\n\t"
1592 "movq %rdi,%rsp\n\t"
1593 "subq %rdi,%rcx\n\t"
1594 "xorl %eax,%eax\n\t"
1595 "shrq $3,%rcx\n\t"
1596 "rep; stosq\n\t"
1597 /* switch to the initial context */
1598 "leaq -32(%rbx),%rsp\n\t"
1599 "movq %rbx,%rcx\n\t"
1600 "movl $1,%edx\n\t"
1601 "call " __ASM_NAME("NtContinue") )
1604 /**********************************************************************
1605 * DbgBreakPoint (NTDLL.@)
1607 __ASM_STDCALL_FUNC( DbgBreakPoint, 0, "int $3; ret"
1608 "\n\tnop; nop; nop; nop; nop; nop; nop; nop"
1609 "\n\tnop; nop; nop; nop; nop; nop" );
1611 /**********************************************************************
1612 * DbgUserBreakPoint (NTDLL.@)
1614 __ASM_STDCALL_FUNC( DbgUserBreakPoint, 0, "int $3; ret"
1615 "\n\tnop; nop; nop; nop; nop; nop; nop; nop"
1616 "\n\tnop; nop; nop; nop; nop; nop" );
1618 #endif /* __x86_64__ */