2 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
4 * Copyright (c) 2018 Joyent, Inc.
8 * Copyright (c) 1989, 1990 William F. Jolitz.
9 * Copyright (c) 1990 The Regents of the University of California.
10 * All rights reserved.
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.113 2003/10/15 02:04:52 peter Exp $
43 #include <sys/asm_linkage.h>
44 #include <sys/asm_misc.h>
47 #include <sys/regset.h>
48 #include <sys/privregs.h>
49 #include <sys/dtrace.h>
50 #include <sys/x86_archext.h>
51 #include <sys/traptrace.h>
52 #include <sys/machparam.h>
55 * only one routine in this file is interesting to lint
62 * push $0 on stack for traps that do not
63 * generate an error code. This is so the rest
64 * of the kernel can expect a consistent stack
65 * from from any exception.
67 * Note that for all exceptions for amd64
68 * %r11 and %rcx are on the stack. Just pop
69 * them back into their appropriate registers and let
70 * it get saved as is running native.
73 #if defined(__xpv) && defined(__amd64)
75 #define NPTRAP_NOERR(trapno) \
79 #define TRAP_NOERR(trapno) \
84 * error code already pushed by hw
87 #define TRAP_ERR(trapno) \
91 #else /* __xpv && __amd64 */
93 #define TRAP_NOERR(trapno) \
97 #define NPTRAP_NOERR(trapno) TRAP_NOERR(trapno)
100 * error code already pushed by hw
103 #define TRAP_ERR(trapno) \
106 #endif /* __xpv && __amd64 */
109 * These are the stacks used on cpu0 for taking double faults,
110 * NMIs and MCEs (the latter two only on amd64 where we have IST).
112 * We define them here instead of in a C file so that we can page-align
113 * them (gcc won't do that in a .c file).
116 DGDEF3
(dblfault_stack0
, DEFAULTSTKSZ
, MMU_PAGESIZE
)
117 .fill DEFAULTSTKSZ, 1, 0
118 DGDEF3
(nmi_stack0
, DEFAULTSTKSZ
, MMU_PAGESIZE
)
119 .fill DEFAULTSTKSZ, 1, 0
120 DGDEF3
(mce_stack0
, DEFAULTSTKSZ
, MMU_PAGESIZE
)
121 .fill DEFAULTSTKSZ, 1, 0
127 TRAP_NOERR
(T_ZERODIV
) /* $0 */
134 * Fetch %dr6 and clear it, handing off the value to the
135 * cmntrap code in %r15/%esi
138 TRAP_NOERR
(T_SGLSTP
) /* $1 */
141 #if !defined(__xpv) /* no sysenter support yet */
143 * If we get here as a result of single-stepping a sysenter
144 * instruction, we suddenly find ourselves taking a #db
145 * in kernel mode -before- we've swapgs'ed. So before we can
146 * take the trap, we do the swapgs here, and fix the return
147 * %rip in trap() so that we return immediately after the
148 * swapgs in the sysenter handler to avoid doing the swapgs again.
150 * Nobody said that the design of sysenter was particularly
157 * At this point the stack looks like this:
159 * (high address) r_ss
163 * r_rip <-- %rsp + 24
164 * r_err <-- %rsp + 16
165 * r_trapno <-- %rsp + 8
166 * (low address) %r11 <-- %rsp
168 leaq sys_sysenter
(%rip
), %r11
169 cmpq
%r11, 24(%rsp
) /* Compare to saved r_rip on the stack */
171 leaq brand_sys_sysenter
(%rip
), %r11
172 cmpq
%r11, 24(%rsp
) /* Compare to saved r_rip on the stack */
174 leaq tr_sys_sysenter
(%rip
), %r11
177 leaq tr_brand_sys_sysenter
(%rip
), %r11
188 movq
%rax
, %r15 /* %db6 -> %r15 */
191 call kdi_dreg_set
/* 0 -> %db6 */
198 #elif defined(__i386)
205 movl
%eax
, %esi
/* %dr6 -> %esi */
208 call kdi_dreg_set
/* 0 -> %dr6 */
224 * Macro to set the gsbase or kgsbase to the address of the struct cpu
225 * for this processor. If we came from userland, set kgsbase else
226 * set gsbase. We find the proper cpu struct by looping through
227 * the cpu structs for all processors till we find a match for the gdt
228 * of the trapping processor. The stack is expected to be pointing at
229 * the standard regs pushed by hardware on a trap (plus error code and trapno).
231 * It's ok for us to clobber gsbase here (and possibly end up with both gsbase
232 * and kgsbase set to the same value) because we're not going back the normal
233 * way out of here (via IRET). Where we're going, we don't need no user %gs.
235 #define SET_CPU_GSBASE \
236 subq $REGOFF_TRAPNO
, %rsp;
/* save regs */ \
237 movq
%rax
, REGOFF_RAX
(%rsp
); \
238 movq
%rbx
, REGOFF_RBX
(%rsp
); \
239 movq
%rcx
, REGOFF_RCX
(%rsp
); \
240 movq
%rdx
, REGOFF_RDX
(%rsp
); \
241 movq
%rbp
, REGOFF_RBP
(%rsp
); \
243 subq $
16, %rsp;
/* space for gdt */ \
245 movq
8(%rsp
), %rcx;
/* %rcx has gdt to match */ \
246 xorl
%ebx
, %ebx;
/* loop index */ \
247 leaq cpu
(%rip
), %rdx;
/* cpu pointer array */ \
249 movq
(%rdx
, %rbx
, CLONGSIZE
), %rax;
/* get cpu[i] */ \
250 cmpq $
0x0, %rax;
/* cpu[i] == NULL ? */ \
251 je
2f;
/* yes, continue */ \
252 cmpq
%rcx
, CPU_GDT
(%rax
);
/* gdt == cpu[i]->cpu_gdt ? */ \
253 je
3f;
/* yes, go set gsbase */ \
255 incl
%ebx;
/* i++ */ \
256 cmpl $NCPU
, %ebx;
/* i < NCPU ? */ \
257 jb
1b;
/* yes, loop */ \
258 /* XXX BIG trouble if we fall thru here. We didn't find a gdt match */ \
260 movl $MSR_AMD_KGSBASE
, %ecx; \
261 cmpw $KCS_SEL
, REGOFF_CS
(%rbp
);
/* trap from kernel? */ \
262 jne
4f;
/* no, go set KGSBASE */ \
263 movl $MSR_AMD_GSBASE
, %ecx;
/* yes, set GSBASE */ \
264 mfence;
/* OPTERON_ERRATUM_88 */ \
266 movq
%rax
, %rdx;
/* write base register */ \
269 movq REGOFF_RDX
(%rbp
), %rdx;
/* restore regs */ \
270 movq REGOFF_RCX
(%rbp
), %rcx; \
271 movq REGOFF_RBX
(%rbp
), %rbx; \
272 movq REGOFF_RAX
(%rbp
), %rax; \
274 movq REGOFF_RBP
(%rsp
), %rbp; \
275 addq $REGOFF_TRAPNO
, %rsp
/* pop stack */
279 #define SET_CPU_GSBASE /* noop on the hypervisor */
293 TRAP_NOERR
(T_NMIFLT
) /* $2 */
298 * Save all registers and setup segment registers
299 * with kernel selectors.
302 INTGATE_INIT_KERNEL_FLAGS
304 TRACE_PTR
(%r12, %rax
, %eax
, %rdx
, $TT_TRAP
)
305 TRACE_REGS
(%r12, %rsp
, %rax
, %rbx
)
311 call av_dispatch_nmivect
318 #elif defined(__i386)
324 TRAP_NOERR
(T_NMIFLT
) /* $2 */
327 * Save all registers and setup segment registers
328 * with kernel selectors.
331 INTGATE_INIT_KERNEL_FLAGS
333 TRACE_PTR
(%edi
, %ebx
, %ebx
, %ecx
, $TT_TRAP
)
334 TRACE_REGS
(%edi
, %esp
, %ebx
, %ecx
)
340 call av_dispatch_nmivect
356 cmpw $KCS_SEL
, 8(%rsp
)
360 * This is a breakpoint in the kernel -- it is very likely that this
361 * is DTrace-induced. To unify DTrace handling, we spoof this as an
362 * invalid opcode (#UD) fault. Note that #BP is a trap, not a fault --
363 * we must decrement the trapping %rip to make it appear as a fault.
364 * We then push a non-zero error code to indicate that this is coming
368 push $
1 /* error code -- non-zero for #BP */
374 NPTRAP_NOERR
(T_BPTFLT
) /* $3 */
383 TRAP_NOERR
(T_OVFLW
) /* $4 */
391 TRAP_NOERR
(T_BOUNDFLT
) /* $5 */
401 cmpw $KCS_SEL
, 8(%rsp
)
405 movb $
0, 12(%rsp
) /* clear saved upcall_mask from %cs */
407 push $
0 /* error code -- zero for #UD */
409 push $
0xdddd /* a dummy trap number */
411 movq REGOFF_RIP
(%rsp
), %rdi
412 movq REGOFF_RSP
(%rsp
), %rsi
413 movq REGOFF_RAX
(%rsp
), %rdx
418 ALTENTRY
(dtrace_invop_callsite
)
420 cmpl $DTRACE_INVOP_PUSHL_EBP
, %eax
422 cmpl $DTRACE_INVOP_LEAVE
, %eax
424 cmpl $DTRACE_INVOP_NOP
, %eax
426 cmpl $DTRACE_INVOP_RET
, %eax
432 * We must emulate a "pushq %rbp". To do this, we pull the stack
433 * down 8 bytes, and then store the base pointer.
436 subq $
16, %rsp
/* make room for %rbp */
437 pushq
%rax
/* push temp */
438 movq
24(%rsp
), %rax
/* load calling RIP */
439 addq $
1, %rax
/* increment over trapping instr */
440 movq
%rax
, 8(%rsp
) /* store calling RIP */
441 movq
32(%rsp
), %rax
/* load calling CS */
442 movq
%rax
, 16(%rsp
) /* store calling CS */
443 movq
40(%rsp
), %rax
/* load calling RFLAGS */
444 movq
%rax
, 24(%rsp
) /* store calling RFLAGS */
445 movq
48(%rsp
), %rax
/* load calling RSP */
446 subq $
8, %rax
/* make room for %rbp */
447 movq
%rax
, 32(%rsp
) /* store calling RSP */
448 movq
56(%rsp
), %rax
/* load calling SS */
449 movq
%rax
, 40(%rsp
) /* store calling SS */
450 movq
32(%rsp
), %rax
/* reload calling RSP */
451 movq
%rbp
, (%rax
) /* store %rbp there */
452 popq
%rax
/* pop off temp */
453 jmp tr_iret_kernel
/* return from interrupt */
458 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp"
459 * followed by a "popq %rbp". This is quite a bit simpler on amd64
460 * than it is on i386 -- we can exploit the fact that the %rsp is
461 * explicitly saved to effect the pop without having to reshuffle
462 * the other data pushed for the trap.
465 pushq
%rax
/* push temp */
466 movq
8(%rsp
), %rax
/* load calling RIP */
467 addq $
1, %rax
/* increment over trapping instr */
468 movq
%rax
, 8(%rsp
) /* store calling RIP */
469 movq
(%rbp
), %rax
/* get new %rbp */
470 addq $
8, %rbp
/* adjust new %rsp */
471 movq
%rbp
, 32(%rsp
) /* store new %rsp */
472 movq
%rax
, %rbp
/* set new %rbp */
473 popq
%rax
/* pop off temp */
474 jmp tr_iret_kernel
/* return from interrupt */
479 * We must emulate a "nop". This is obviously not hard: we need only
480 * advance the %rip by one.
489 pushq
%rax
/* push temp */
490 movq
32(%rsp
), %rax
/* load %rsp */
491 movq
(%rax
), %rax
/* load calling RIP */
492 movq
%rax
, 8(%rsp
) /* store calling RIP */
493 addq $
8, 32(%rsp
) /* adjust new %rsp */
494 popq
%rax
/* pop off temp */
495 jmp tr_iret_kernel
/* return from interrupt */
500 * We're going to let the kernel handle this as a normal #UD. If,
501 * however, we came through #BP and are spoofing #UD (in this case,
502 * the stored error value will be non-zero), we need to de-spoof
503 * the trap by incrementing %rip and pushing T_BPTFLT.
505 cmpq $
0, REGOFF_ERR
(%rsp
)
507 incq REGOFF_RIP
(%rsp
)
508 addq $REGOFF_RIP
, %rsp
509 NPTRAP_NOERR
(T_BPTFLT
) /* $3 */
513 addq $REGOFF_RIP
, %rsp
515 NPTRAP_NOERR
(T_ILLINST
)
519 #elif defined(__i386)
526 * If we are taking an invalid opcode trap while in the kernel, this
527 * is likely an FBT probe point.
530 cmpw $KGS_SEL
, (%esp
)
535 movb $
0, 6(%esp
) /* clear saved upcall_mask from %cs */
538 pushl
%eax
/* push %eax -- may be return value */
539 pushl
%esp
/* push stack pointer */
540 addl $
48, (%esp
) /* adjust to incoming args */
541 pushl
40(%esp
) /* push calling EIP */
543 ALTENTRY
(dtrace_invop_callsite
)
545 cmpl $DTRACE_INVOP_PUSHL_EBP
, %eax
547 cmpl $DTRACE_INVOP_POPL_EBP
, %eax
549 cmpl $DTRACE_INVOP_LEAVE
, %eax
551 cmpl $DTRACE_INVOP_NOP
, %eax
556 * We must emulate a "pushl %ebp". To do this, we pull the stack
557 * down 4 bytes, and then store the base pointer.
560 subl $
4, %esp
/* make room for %ebp */
561 pushl
%eax
/* push temp */
562 movl
8(%esp
), %eax
/* load calling EIP */
563 incl
%eax
/* increment over LOCK prefix */
564 movl
%eax
, 4(%esp
) /* store calling EIP */
565 movl
12(%esp
), %eax
/* load calling CS */
566 movl
%eax
, 8(%esp
) /* store calling CS */
567 movl
16(%esp
), %eax
/* load calling EFLAGS */
568 movl
%eax
, 12(%esp
) /* store calling EFLAGS */
569 movl
%ebp
, 16(%esp
) /* push %ebp */
570 popl
%eax
/* pop off temp */
574 * We must emulate a "popl %ebp". To do this, we do the opposite of
575 * the above: we remove the %ebp from the stack, and squeeze up the
576 * saved state from the trap.
579 pushl
%eax
/* push temp */
580 movl
16(%esp
), %ebp
/* pop %ebp */
581 movl
12(%esp
), %eax
/* load calling EFLAGS */
582 movl
%eax
, 16(%esp
) /* store calling EFLAGS */
583 movl
8(%esp
), %eax
/* load calling CS */
584 movl
%eax
, 12(%esp
) /* store calling CS */
585 movl
4(%esp
), %eax
/* load calling EIP */
586 incl
%eax
/* increment over LOCK prefix */
587 movl
%eax
, 8(%esp
) /* store calling EIP */
588 popl
%eax
/* pop off temp */
589 addl $
4, %esp
/* adjust stack pointer */
593 * We must emulate a "leave", which is the same as a "movl %ebp, %esp"
594 * followed by a "popl %ebp". This looks similar to the above, but
595 * requires two temporaries: one for the new base pointer, and one
596 * for the staging register.
599 pushl
%eax
/* push temp */
600 pushl
%ebx
/* push temp */
601 movl
%ebp
, %ebx
/* set temp to old %ebp */
602 movl
(%ebx
), %ebp
/* pop %ebp */
603 movl
16(%esp
), %eax
/* load calling EFLAGS */
604 movl
%eax
, (%ebx
) /* store calling EFLAGS */
605 movl
12(%esp
), %eax
/* load calling CS */
606 movl
%eax
, -4(%ebx
) /* store calling CS */
607 movl
8(%esp
), %eax
/* load calling EIP */
608 incl
%eax
/* increment over LOCK prefix */
609 movl
%eax
, -8(%ebx
) /* store calling EIP */
610 movl
%ebx
, -4(%esp
) /* temporarily store new %esp */
611 popl
%ebx
/* pop off temp */
612 popl
%eax
/* pop off temp */
613 movl
-12(%esp
), %esp
/* set stack pointer */
614 subl $
8, %esp
/* adjust for three pushes, one pop */
618 * We must emulate a "nop". This is obviously not hard: we need only
619 * advance the %eip by one.
624 IRET
/* return from interrupt */
628 pushl $T_ILLINST
/* $6 */
633 pushl $T_ILLINST
/* $6 */
648 * (On the hypervisor we must make a hypercall so we might as well
649 * save everything and handle as in a normal trap.)
651 TRAP_NOERR
(T_NOEXTFLT
) /* $7 */
655 * We want to do this quickly as every lwp using fp will take this
656 * after a context switch -- we do the frequent path in ndptrap_frstor
657 * below; for all other cases, we let the trap code handle it
659 LOADCPU
(%rax
) /* swapgs handled in hypervisor */
660 cmpl $
0, fpu_exists
(%rip
)
661 je
.handle_in_trap /* let trap handle no fp case */
662 movq CPU_THREAD
(%rax
), %rbx
/* %rbx = curthread */
664 movq T_LWP
(%rbx
), %rbx
/* %rbx = lwp */
666 jz
.handle_in_trap /* should not happen? */
668 addq $LWP_PCB_FPU
, %rbx
/* &lwp->lwp_pcb.pcb_fpu */
670 testl
%eax
, PCB_FPU_FLAGS
(%rbx
)
671 jz
.handle_in_trap /* must be the first fault */
673 andl $_BITNOT
(FPU_VALID
), PCB_FPU_FLAGS
(%rbx
)
674 #if FPU_CTX_FPU_REGS != 0
675 addq $FPU_CTX_FPU_REGS
, %rbx
678 movl FPU_CTX_FPU_XSAVE_MASK
(%rbx
), %eax
/* for xrstor */
679 movl FPU_CTX_FPU_XSAVE_MASK+
4(%rbx
), %edx
/* for xrstor */
682 * the label below is used in trap.c to detect FP faults in
683 * kernel due to user fault.
685 ALTENTRY
(ndptrap_frstor
)
686 movq
(%rbx
), %rbx
/* fpu_regs.kfpu_u.kfpu_XX pointer */
687 .globl _patch_xrstorq_rbx
690 cmpw $KCS_SEL
, REGOFF_CS
(%rsp
)
693 ASSERT_UPCALL_MASK_IS_SET
695 IRET
/* return to user mode */
705 pushq $
0 /* can not use TRAP_NOERR */
708 SET_SIZE
(ndptrap_frstor
)
715 * We want to do this quickly as every lwp using fp will take this
716 * after a context switch -- we do the frequent path in ndptrap_frstor
717 * below; for all other cases, we let the trap code handle it
721 cmpw $KCS_SEL
, 24(%rsp
) /* did we come from kernel mode? */
723 LOADCPU
(%rax
) /* if yes, don't swapgs */
726 SWAPGS
/* if from user, need swapgs */
731 * Xrstor needs to use edx as part of its flag.
732 * NOTE: have to push rdx after "cmpw ...24(%rsp)", otherwise rsp+$24
733 * will not point to CS.
736 cmpl $
0, fpu_exists
(%rip
)
737 je
.handle_in_trap /* let trap handle no fp case */
738 movq CPU_THREAD
(%rax
), %rbx
/* %rbx = curthread */
740 movq T_LWP
(%rbx
), %rbx
/* %rbx = lwp */
742 jz
.handle_in_trap /* should not happen? */
744 addq $LWP_PCB_FPU
, %rbx
/* &lwp->lwp_pcb.pcb_fpu */
746 testl
%eax
, PCB_FPU_FLAGS
(%rbx
)
747 jz
.handle_in_trap /* must be the first fault */
749 andl $_BITNOT
(FPU_VALID
), PCB_FPU_FLAGS
(%rbx
)
750 #if FPU_CTX_FPU_REGS != 0
751 addq $FPU_CTX_FPU_REGS
, %rbx
754 movl FPU_CTX_FPU_XSAVE_MASK
(%rbx
), %eax
/* for xrstor */
755 movl FPU_CTX_FPU_XSAVE_MASK+
4(%rbx
), %edx
/* for xrstor */
758 * the label below is used in trap.c to detect FP faults in
759 * kernel due to user fault.
761 ALTENTRY
(ndptrap_frstor
)
762 movq
(%rbx
), %rbx
/* fpu_regs.kfpu_u.kfpu_XX pointer */
763 .globl _patch_xrstorq_rbx
776 TRAP_NOERR
(T_NOEXTFLT
) /* $7 */
778 SET_SIZE
(ndptrap_frstor
)
783 #elif defined(__i386)
787 * We want to do this quickly as every lwp using fp will take this
788 * after a context switch -- we do the frequent path in fpnoextflt
789 * below; for all other cases, we let the trap code handle it
793 pushl
%edx
/* for xrstor */
802 je
.handle_in_trap /* let trap handle no fp case */
803 movl CPU_THREAD
(%eax
), %ebx
/* %ebx = curthread */
805 movl T_LWP
(%ebx
), %ebx
/* %ebx = lwp */
807 jz
.handle_in_trap /* should not happen? */
809 addl $LWP_PCB_FPU
, %ebx
/* &lwp->lwp_pcb.pcb_fpu */
811 testl
%eax
, PCB_FPU_FLAGS
(%ebx
)
812 jz
.handle_in_trap /* must be the first fault */
814 andl $_BITNOT
(FPU_VALID
), PCB_FPU_FLAGS
(%ebx
)
815 #if FPU_CTX_FPU_REGS != 0
816 addl $FPU_CTX_FPU_REGS
, %ebx
819 movl FPU_CTX_FPU_XSAVE_MASK
(%ebx
), %eax
/* for xrstor */
820 movl FPU_CTX_FPU_XSAVE_MASK+
4(%ebx
), %edx
/* for xrstor */
823 * the label below is used in trap.c to detect FP faults in kernel
826 ALTENTRY
(ndptrap_frstor
)
827 movl
(%ebx
), %ebx
/* fpu_regs.kfpu_u.kfpu_XX pointer */
828 .globl _patch_fxrstor_ebx
830 .globl _patch_xrstor_ebx
832 frstor
(%ebx
) /* may be patched to fxrstor or xrstor */
846 TRAP_NOERR
(T_NOEXTFLT
) /* $7 */
848 SET_SIZE
(ndptrap_frstor
)
863 * We share this handler with kmdb (if kmdb is loaded). As such, we
864 * may have reached this point after encountering a #df in kmdb. If
865 * that happens, we'll still be on kmdb's IDT. We need to switch back
866 * to this CPU's IDT before proceeding. Furthermore, if we did arrive
867 * here from kmdb, kmdb is probably in a very sickly state, and
868 * shouldn't be entered from the panic flow. We'll suppress that
869 * entry by setting nopanicdebug.
872 subq $DESCTBR_SIZE
, %rsp
874 movq
%gs
:CPU_IDT
, %rax
875 cmpq
%rax
, DTR_BASE
(%rsp
)
878 movq
%rax
, DTR_BASE
(%rsp
)
879 movw $_MUL
(NIDT
, GATE_DESC_SIZE
), DTR_LIMIT
(%rsp
)
882 movl $
1, nopanicdebug
884 1: addq $DESCTBR_SIZE
, %rsp
893 leaq trap_trace_freeze
(%rip
), %r11
899 movq
%rsp
, %rdi
/* ®s */
900 xorl
%esi
, %esi
/* clear address */
901 xorl
%edx
, %edx
/* cpuid = 0 */
906 #elif defined(__i386)
912 cli /* disable interrupts */
915 * We share this handler with kmdb (if kmdb is loaded). As such, we
916 * may have reached this point after encountering a #df in kmdb. If
917 * that happens, we'll still be on kmdb's IDT. We need to switch back
918 * to this CPU's IDT before proceeding. Furthermore, if we did arrive
919 * here from kmdb, kmdb is probably in a very sickly state, and
920 * shouldn't be entered from the panic flow. We'll suppress that
921 * entry by setting nopanicdebug.
924 subl $DESCTBR_SIZE
, %esp
925 movl
%gs
:CPU_IDT
, %eax
927 cmpl DTR_BASE
(%esp
), %eax
930 movl
%eax
, DTR_BASE
(%esp
)
931 movw $_MUL
(NIDT
, GATE_DESC_SIZE
), DTR_LIMIT
(%esp
)
934 movl $
1, nopanicdebug
936 1: addl $DESCTBR_SIZE
, %esp
939 * Check the CPL in the TSS to see what mode
940 * (user or kernel) we took the fault in. At this
941 * point we are running in the context of the double
942 * fault task (dftss) but the CPU's task points to
943 * the previous task (ktss) where the process context
944 * has been saved as the result of the task switch.
946 movl
%gs
:CPU_TSS
, %eax
/* get the TSS */
947 movl TSS_SS
(%eax
), %ebx
/* save the fault SS */
948 movl TSS_ESP
(%eax
), %edx
/* save the fault ESP */
949 testw $CPL_MASK
, TSS_CS
(%eax
) /* user mode ? */
951 movw TSS_SS0
(%eax
), %ss
/* get on the kernel stack */
952 movl TSS_ESP0
(%eax
), %esp
955 * Clear the NT flag to avoid a task switch when the process
956 * finally pops the EFL off the stack via an iret. Clear
957 * the TF flag since that is what the processor does for
958 * a normal exception. Clear the IE flag so that interrupts
961 movl TSS_EFL
(%eax
), %ecx
962 andl $_BITNOT
(PS_NT|PS_T|PS_IE
), %ecx
964 popfl
/* restore the EFL */
965 movw TSS_LDT
(%eax
), %cx
/* restore the LDT */
969 * Restore process segment selectors.
971 movw TSS_DS
(%eax
), %ds
972 movw TSS_ES
(%eax
), %es
973 movw TSS_FS
(%eax
), %fs
974 movw TSS_GS
(%eax
), %gs
977 * Restore task segment selectors.
979 movl $KDS_SEL
, TSS_DS
(%eax
)
980 movl $KDS_SEL
, TSS_ES
(%eax
)
981 movl $KDS_SEL
, TSS_SS
(%eax
)
982 movl $KFS_SEL
, TSS_FS
(%eax
)
983 movl $KGS_SEL
, TSS_GS
(%eax
)
986 * Clear the TS bit, the busy bits in both task
987 * descriptors, and switch tasks.
991 movl DFTSS_SEL+
4(%ecx
), %esi
992 andl $_BITNOT
(0x200), %esi
993 movl
%esi
, DFTSS_SEL+
4(%ecx
)
994 movl KTSS_SEL+
4(%ecx
), %esi
995 andl $_BITNOT
(0x200), %esi
996 movl
%esi
, KTSS_SEL+
4(%ecx
)
1001 * Restore part of the process registers.
1003 movl TSS_EBP
(%eax
), %ebp
1004 movl TSS_ECX
(%eax
), %ecx
1005 movl TSS_ESI
(%eax
), %esi
1006 movl TSS_EDI
(%eax
), %edi
1010 * Make a trap frame. Leave the error code (0) on
1011 * the stack since the first word on a trap stack is
1014 pushl
%ebx
/ fault SS
1015 pushl
%edx
/ fault ESP
1016 pushl TSS_EFL
(%eax
) / fault EFL
1017 pushl TSS_CS
(%eax
) / fault CS
1018 pushl TSS_EIP
(%eax
) / fault EIP
1019 pushl $
0 / error code
1020 pushl $T_DBLFLT
/ trap number
8
1021 movl TSS_EBX
(%eax
), %ebx
/ restore EBX
1022 movl TSS_EDX
(%eax
), %edx
/ restore EDX
1023 movl TSS_EAX
(%eax
), %eax
/ restore EAX
1024 sti
/ enable interrupts
1026 SET_SIZE
(syserrtrap
)
1032 TRAP_NOERR
(T_EXTOVRFLT
) /* $9 i386 only - not generated */
1039 ENTRY_NP
(invtsstrap
)
1040 TRAP_ERR
(T_TSSFLT
) /* $10 already have error code on stack */
1042 SET_SIZE
(invtsstrap
)
1048 TRAP_ERR
(T_SEGFLT
) /* $11 already have error code on stack */
1049 #if defined(__amd64)
1059 TRAP_ERR
(T_STKFLT
) /* $12 already have error code on stack */
1060 #if defined(__amd64)
1070 TRAP_ERR
(T_GPFLT
) /* $13 already have error code on stack */
1071 #if defined(__amd64)
1081 TRAP_ERR
(T_PGFLT
) /* $14 already have error code on stack */
1084 #if defined(__amd64)
1086 #elif defined(__i386)
1093 #if !defined(__amd64)
1095 .globl idt0_default_r
1098 * #PF pentium bug workaround
1100 ENTRY_NP
(pentium_pftrap
)
1103 andl $MMU_STD_PAGEMASK
, %eax
1105 cmpl %eax
, %cs
:idt0_default_r+
2 /* fixme */
1107 je check_for_user_address
1110 pushl $T_PGFLT
/* $14 */
1112 check_for_user_address
:
1114 * Before we assume that we have an unmapped trap on our hands,
1115 * check to see if this is a fault from user mode. If it is,
1116 * we'll kick back into the page fault handler.
1118 movl
4(%esp
), %eax
/* error code */
1119 andl $PF_ERR_USER
, %eax
1123 * We now know that this is the invalid opcode trap.
1126 addl $
4, %esp
/* pop error code */
1128 SET_SIZE
(pentium_pftrap
)
1130 #endif /* !__amd64 */
1133 TRAP_NOERR
(T_RESVTRAP
) /* (reserved) */
1141 TRAP_NOERR
(T_EXTERRFLT
) /* $16 */
1149 TRAP_ERR
(T_ALIGNMENT
) /* $17 */
1156 .globl cmi_mca_trap /* see uts/i86pc/os/cmi.c */
1158 #if defined(__amd64)
1161 TRAP_NOERR
(T_MCE
) /* $18 */
1166 INTGATE_INIT_KERNEL_FLAGS
1168 TRACE_PTR
(%rdi
, %rbx
, %ebx
, %rcx
, $TT_TRAP
)
1169 TRACE_REGS
(%rdi
, %rsp
, %rbx
, %rcx
)
1174 movq
%rsp
, %rdi
/* arg0 = struct regs *rp */
1175 call cmi_mca_trap
/* cmi_mca_trap(rp); */
1183 TRAP_NOERR
(T_MCE
) /* $18 */
1186 INTGATE_INIT_KERNEL_FLAGS
1188 TRACE_PTR
(%edi
, %ebx
, %ebx
, %ecx
, $TT_TRAP
)
1189 TRACE_REGS
(%edi
, %esp
, %ebx
, %ecx
)
1195 pushl
%ecx
/* arg0 = struct regs *rp */
1196 call cmi_mca_trap
/* cmi_mca_trap(rp) */
1197 addl $
4, %esp
/* pop arg0 */
1208 TRAP_NOERR
(T_SIMDFPE
) /* $19 */
1213 TRAP_NOERR
(T_INVALTRAP
) /* very invalid */
1219 #if defined(__amd64)
1222 cmpl $T_LASTFAST
, %eax
1224 orl
%eax
, %eax
/* (zero extend top 32-bits) */
1225 leaq fasttable
(%rip
), %r11
1226 leaq
(%r11, %rax
, CLONGSIZE
), %r11
1230 * Fast syscall number was illegal. Make it look
1231 * as if the INT failed. Modify %rip to point before the
1232 * INT, push the expected error code and fake a GP fault.
1234 * XXX Why make the error code be offset into idt + 1?
1235 * Instead we should push a real (soft?) error code
1236 * on the stack and #gp handler could know about fasttraps?
1240 subq $
2, (%rsp
) /* XXX int insn 2-bytes */
1241 pushq $_CONST
(_MUL
(T_FASTTRAP
, GATE_DESC_SIZE
) + 2)
1250 #elif defined(__i386)
1253 cmpl $T_LASTFAST
, %eax
1255 jmp
*%cs
:fasttable
(, %eax
, CLONGSIZE
)
1258 * Fast syscall number was illegal. Make it look
1259 * as if the INT failed. Modify %eip to point before the
1260 * INT, push the expected error code and fake a GP fault.
1262 * XXX Why make the error code be offset into idt + 1?
1263 * Instead we should push a real (soft?) error code
1264 * on the stack and #gp handler could know about fasttraps?
1266 subl $
2, (%esp
) /* XXX int insn 2-bytes */
1267 pushl $_CONST
(_MUL
(T_FASTTRAP
, GATE_DESC_SIZE
) + 2)
1273 ENTRY_NP
(dtrace_ret
)
1274 TRAP_NOERR
(T_DTRACE_RET
)
1276 SET_SIZE
(dtrace_ret
)
1278 #if defined(__amd64)
1281 * RFLAGS 24 bytes up the stack from %rsp.
1282 * XXX a constant would be nicer.
1286 orq $PS_C
, 24(%rsp
) /* set carry bit in user flags */
1291 #elif defined(__i386)
1294 orw $PS_C
, 8(%esp
) /* set carry bit in user flags */
1301 * Interrupts start at 32