4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 * Copyright 2018 Joyent, Inc.
30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
36 * Copyright (c) 2009, Intel Corporation.
37 * All rights reserved.
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
50 #include <sys/asm_linkage.h>
51 #include <sys/asm_misc.h>
52 #include <sys/panic.h>
53 #include <sys/ontrap.h>
54 #include <sys/regset.h>
55 #include <sys/privregs.h>
56 #include <sys/reboot.h>
58 #include <sys/x86_archext.h>
61 #include <sys/dditypes.h>
66 * Catch lofault faults. Like setjmp except it returns one
67 * if code following causes uncorrectable fault. Turned off
68 * by calling no_fault(). Note that while under on_fault(),
69 * SMAP is disabled. For more information see
70 * uts/intel/ia32/ml/copy.s.
77 movq
%gs
:CPU_THREAD
, %rsi
78 leaq catch_fault
(%rip
), %rdx
79 movq
%rdi
, T_ONFAULT
(%rsi
) /* jumpbuf in t_onfault */
80 movq
%rdx
, T_LOFAULT
(%rsi
) /* catch_fault in t_lofault */
81 call smap_disable
/* allow user accesses */
82 jmp setjmp
/* let setjmp do the rest */
85 movq
%gs
:CPU_THREAD
, %rsi
86 movq T_ONFAULT
(%rsi
), %rdi
/* address of save area */
88 movq
%rax
, T_ONFAULT
(%rsi
) /* turn off onfault */
89 movq
%rax
, T_LOFAULT
(%rsi
) /* turn off lofault */
90 call smap_enable
/* disallow user accesses */
91 jmp longjmp
/* let longjmp do the rest */
95 movq
%gs
:CPU_THREAD
, %rsi
97 movq
%rax
, T_ONFAULT
(%rsi
) /* turn off onfault */
98 movq
%rax
, T_LOFAULT
(%rsi
) /* turn off lofault */
99 call smap_enable
/* disallow user accesses */
103 #elif defined(__i386)
106 movl
%gs
:CPU_THREAD
, %edx
107 movl
4(%esp
), %eax
/* jumpbuf address */
108 leal catch_fault
, %ecx
109 movl
%eax
, T_ONFAULT
(%edx
) /* jumpbuf in t_onfault */
110 movl
%ecx
, T_LOFAULT
(%edx
) /* catch_fault in t_lofault */
111 jmp setjmp
/* let setjmp do the rest */
114 movl
%gs
:CPU_THREAD
, %edx
116 movl T_ONFAULT
(%edx
), %ecx
/* address of save area */
117 movl
%eax
, T_ONFAULT
(%edx
) /* turn off onfault */
118 movl
%eax
, T_LOFAULT
(%edx
) /* turn off lofault */
120 call longjmp
/* let longjmp do the rest */
124 movl
%gs
:CPU_THREAD
, %edx
126 movl
%eax
, T_ONFAULT
(%edx
) /* turn off onfault */
127 movl
%eax
, T_LOFAULT
(%edx
) /* turn off lofault */
134 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
135 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
141 ENTRY
(on_trap_trampoline
)
142 movq
%gs
:CPU_THREAD
, %rsi
143 movq T_ONTRAP
(%rsi
), %rdi
144 addq $OT_JMPBUF
, %rdi
146 SET_SIZE
(on_trap_trampoline
)
148 #elif defined(__i386)
150 ENTRY
(on_trap_trampoline
)
151 movl
%gs
:CPU_THREAD
, %eax
152 movl T_ONTRAP
(%eax
), %eax
153 addl $OT_JMPBUF
, %eax
156 SET_SIZE
(on_trap_trampoline
)
161 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
162 * more information about the on_trap() mechanism. If the on_trap_data is the
163 * same as the topmost stack element, we just modify that element.
169 movw
%si
, OT_PROT
(%rdi
) /* ot_prot = prot */
170 movw $
0, OT_TRAP
(%rdi
) /* ot_trap = 0 */
171 leaq on_trap_trampoline
(%rip
), %rdx
/* rdx = &on_trap_trampoline */
172 movq
%rdx
, OT_TRAMPOLINE
(%rdi
) /* ot_trampoline = rdx */
174 movq
%rcx
, OT_HANDLE
(%rdi
) /* ot_handle = NULL */
175 movq
%rcx
, OT_PAD1
(%rdi
) /* ot_pad1 = NULL */
176 movq
%gs
:CPU_THREAD
, %rdx
/* rdx = curthread */
177 movq T_ONTRAP
(%rdx
), %rcx
/* rcx = curthread->t_ontrap */
178 cmpq
%rdi
, %rcx
/* if (otp == %rcx) */
179 je
0f
/* don't modify t_ontrap */
181 movq
%rcx
, OT_PREV
(%rdi
) /* ot_prev = t_ontrap */
182 movq
%rdi
, T_ONTRAP
(%rdx
) /* curthread->t_ontrap = otp */
184 0: addq $OT_JMPBUF
, %rdi
/* &ot_jmpbuf */
188 #elif defined(__i386)
191 movl
4(%esp
), %eax
/* %eax = otp */
192 movl
8(%esp
), %edx
/* %edx = prot */
194 movw
%dx
, OT_PROT
(%eax
) /* ot_prot = prot */
195 movw $
0, OT_TRAP
(%eax
) /* ot_trap = 0 */
196 leal on_trap_trampoline
, %edx
/* %edx = &on_trap_trampoline */
197 movl
%edx
, OT_TRAMPOLINE
(%eax
) /* ot_trampoline = %edx */
198 movl $
0, OT_HANDLE
(%eax
) /* ot_handle = NULL */
199 movl $
0, OT_PAD1
(%eax
) /* ot_pad1 = NULL */
200 movl
%gs
:CPU_THREAD
, %edx
/* %edx = curthread */
201 movl T_ONTRAP
(%edx
), %ecx
/* %ecx = curthread->t_ontrap */
202 cmpl %eax
, %ecx
/* if (otp == %ecx) */
203 je
0f
/* don't modify t_ontrap */
205 movl
%ecx
, OT_PREV
(%eax
) /* ot_prev = t_ontrap */
206 movl
%eax
, T_ONTRAP
(%edx
) /* curthread->t_ontrap = otp */
208 0: addl $OT_JMPBUF
, %eax
/* %eax = &ot_jmpbuf */
209 movl
%eax
, 4(%esp
) /* put %eax back on the stack */
210 jmp setjmp
/* let setjmp do the rest */
216 * Setjmp and longjmp implement non-local gotos using state vectors
222 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
223 #endif /* LABEL_PC != 0 */
228 movq
%rsp
, LABEL_SP
(%rdi
)
229 movq
%rbp
, LABEL_RBP
(%rdi
)
230 movq
%rbx
, LABEL_RBX
(%rdi
)
231 movq
%r12, LABEL_R12
(%rdi
)
232 movq
%r13, LABEL_R13
(%rdi
)
233 movq
%r14, LABEL_R14
(%rdi
)
234 movq
%r15, LABEL_R15
(%rdi
)
235 movq
(%rsp
), %rdx
/* return address */
236 movq
%rdx
, (%rdi
) /* LABEL_PC is 0 */
237 xorl
%eax
, %eax
/* return 0 */
242 movq LABEL_SP
(%rdi
), %rsp
243 movq LABEL_RBP
(%rdi
), %rbp
244 movq LABEL_RBX
(%rdi
), %rbx
245 movq LABEL_R12
(%rdi
), %r12
246 movq LABEL_R13
(%rdi
), %r13
247 movq LABEL_R14
(%rdi
), %r14
248 movq LABEL_R15
(%rdi
), %r15
249 movq
(%rdi
), %rdx
/* return address; LABEL_PC is 0 */
252 incl
%eax
/* return 1 */
256 #elif defined(__i386)
259 movl
4(%esp
), %edx
/* address of save area */
260 movl
%ebp
, LABEL_EBP
(%edx
)
261 movl
%ebx
, LABEL_EBX
(%edx
)
262 movl
%esi
, LABEL_ESI
(%edx
)
263 movl
%edi
, LABEL_EDI
(%edx
)
265 movl
(%esp
), %ecx
/* %eip (return address) */
266 movl
%ecx
, (%edx
) /* LABEL_PC is 0 */
267 subl
%eax
, %eax
/* return 0 */
272 movl
4(%esp
), %edx
/* address of save area */
273 movl LABEL_EBP
(%edx
), %ebp
274 movl LABEL_EBX
(%edx
), %ebx
275 movl LABEL_ESI
(%edx
), %esi
276 movl LABEL_EDI
(%edx
), %edi
278 movl
(%edx
), %ecx
/* %eip (return addr); LABEL_PC is 0 */
280 addl $
4, %esp
/* pop ret adr */
281 jmp
*%ecx
/* indirect */
287 * if a() calls b() calls caller(),
288 * caller() returns return address in a().
289 * (Note: We assume a() and b() are C routines which do the normal entry/exit
297 movq
8(%rbp
), %rax
/* b()'s return pc, in a() */
301 #elif defined(__i386)
304 movl
4(%ebp
), %eax
/* b()'s return pc, in a() */
311 * if a() calls callee(), callee() returns the
312 * return address in a();
319 movq
(%rsp
), %rax
/* callee()'s return pc, in a() */
323 #elif defined(__i386)
326 movl
(%esp
), %eax
/* callee()'s return pc, in a() */
333 * return the current frame pointer
344 #elif defined(__i386)
354 * Invalidate a single page table entry in the TLB
364 * Get/Set the value of various control registers
382 movq
%gs
:CPU_VCPU_INFO
, %rax
383 movq VCPU_INFO_ARCH_CR2
(%rax
), %rax
430 #elif defined(__i386)
444 * "lock mov %cr0" is used on processors which indicate it is
445 * supported via CPUID. Normally the 32 bit TPR is accessed via
463 movl
%gs
:CPU_VCPU_INFO
, %eax
464 movl VCPU_INFO_ARCH_CR2
(%eax
), %eax
512 movl
(%rdi
), %eax
/* %eax = regs->cp_eax */
513 movl
0x4(%rdi
), %ebx
/* %ebx = regs->cp_ebx */
514 movl
0x8(%rdi
), %ecx
/* %ecx = regs->cp_ecx */
515 movl
0xc(%rdi
), %edx
/* %edx = regs->cp_edx */
517 movl
%eax
, (%rdi
) /* regs->cp_eax = %eax */
518 movl
%ebx
, 0x4(%rdi
) /* regs->cp_ebx = %ebx */
519 movl
%ecx
, 0x8(%rdi
) /* regs->cp_ecx = %ecx */
520 movl
%edx
, 0xc(%rdi
) /* regs->cp_edx = %edx */
525 SET_SIZE
(__cpuid_insn
)
527 #elif defined(__i386)
531 movl
0x8(%esp
), %ebp
/* %ebp = regs */
535 movl
(%ebp
), %eax
/* %eax = regs->cp_eax */
536 movl
0x4(%ebp
), %ebx
/* %ebx = regs->cp_ebx */
537 movl
0x8(%ebp
), %ecx
/* %ecx = regs->cp_ecx */
538 movl
0xc(%ebp
), %edx
/* %edx = regs->cp_edx */
540 movl
%eax
, (%ebp
) /* regs->cp_eax = %eax */
541 movl
%ebx
, 0x4(%ebp
) /* regs->cp_ebx = %ebx */
542 movl
%ecx
, 0x8(%ebp
) /* regs->cp_ecx = %ecx */
543 movl
%edx
, 0xc(%ebp
) /* regs->cp_edx = %edx */
549 SET_SIZE
(__cpuid_insn
)
556 ENTRY_NP
(i86_monitor
)
559 movq
%rdi
, %rax
/* addr */
560 movq
%rsi
, %rcx
/* extensions */
561 /* rdx contains input arg3: hints */
563 .byte 0x0f, 0x01, 0xc8 /* monitor */
566 SET_SIZE
(i86_monitor
)
568 #elif defined(__i386)
570 ENTRY_NP
(i86_monitor
)
573 movl
0x8(%ebp
),%eax
/* addr */
574 movl
0xc(%ebp
),%ecx
/* extensions */
575 movl
0x10(%ebp
),%edx
/* hints */
577 .byte 0x0f, 0x01, 0xc8 /* monitor */
580 SET_SIZE
(i86_monitor
)
590 movq
%rdi
, %rax
/* data */
591 movq
%rsi
, %rcx
/* extensions */
592 .byte 0x0f, 0x01, 0xc9 /* mwait */
597 #elif defined(__i386)
602 movl
0x8(%ebp
),%eax
/* data */
603 movl
0xc(%ebp
),%ecx
/* extensions */
604 .byte 0x0f, 0x01, 0xc9 /* mwait */
624 .globl _tsc_mfence_start
631 .globl _tsc_mfence_end
635 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
641 .globl _no_rdtsc_start
648 .globl _tsc_lfence_start
655 .globl _tsc_lfence_end
668 .globl _tsc_mfence_start
673 .globl _tsc_mfence_end
677 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
681 .globl _no_rdtsc_start
688 .globl _tsc_lfence_start
693 .globl _tsc_lfence_end
701 * Do not use this function for obtaining clock tick. This
702 * is called by callers who do not need to have a guarenteed
703 * correct tick value. The proper routine to use is tsc_read().
721 * Insert entryp after predp in a doubly linked list.
727 movq
(%rsi
), %rax
/* predp->forw */
728 movq
%rsi
, CPTRSIZE
(%rdi
) /* entryp->back = predp */
729 movq
%rax
, (%rdi
) /* entryp->forw = predp->forw */
730 movq
%rdi
, (%rsi
) /* predp->forw = entryp */
731 movq
%rdi
, CPTRSIZE
(%rax
) /* predp->forw->back = entryp */
735 #elif defined(__i386)
740 movl
(%edx
), %eax
/* predp->forw */
741 movl
%edx
, CPTRSIZE
(%ecx
) /* entryp->back = predp */
742 movl
%eax
, (%ecx
) /* entryp->forw = predp->forw */
743 movl
%ecx
, (%edx
) /* predp->forw = entryp */
744 movl
%ecx
, CPTRSIZE
(%eax
) /* predp->forw->back = entryp */
751 * Remove entryp from a doubly linked list
758 movq
(%rdi
), %rax
/* entry->forw */
759 movq CPTRSIZE
(%rdi
), %rdx
/* entry->back */
760 movq
%rax
, (%rdx
) /* entry->back->forw = entry->forw */
761 movq
%rdx
, CPTRSIZE
(%rax
) /* entry->forw->back = entry->back */
765 #elif defined(__i386)
769 movl
(%ecx
), %eax
/* entry->forw */
770 movl CPTRSIZE
(%ecx
), %edx
/* entry->back */
771 movl
%eax
, (%edx
) /* entry->back->forw = entry->forw */
772 movl
%edx
, CPTRSIZE
(%eax
) /* entry->forw->back = entry->back */
779 * Returns the number of
780 * non-NULL bytes in string argument.
787 * This is close to a simple transliteration of a C version of this
788 * routine. We should either just -make- this be a C version, or
789 * justify having it in assembler by making it significantly faster.
792 * strlen(const char *s)
796 * if ((uintptr_t)s < KERNELBASE)
797 * panic(.str_panic_msg);
799 * for (s0 = s; *s; s++)
807 movq postbootkernelbase
(%rip
), %rax
812 leaq
.str_panic_msg(%rip), %rdi
831 #elif defined(__i386)
835 movl postbootkernelbase
, %eax
840 pushl $
.str_panic_msg
845 movl
4(%esp
), %eax
/* %eax = string address */
846 testl $
3, %eax
/* if %eax not word aligned */
847 jnz
.not_word_aligned /* goto .not_word_aligned */
850 movl
(%eax
), %edx
/* move 1 word from (%eax) to %edx */
851 movl $
0x7f7f7f7f, %ecx
852 andl
%edx
, %ecx
/* %ecx = %edx & 0x7f7f7f7f */
853 addl $
4, %eax
/* next word */
854 addl $
0x7f7f7f7f, %ecx
/* %ecx += 0x7f7f7f7f */
855 orl
%edx
, %ecx
/* %ecx |= %edx */
856 andl $
0x80808080, %ecx
/* %ecx &= 0x80808080 */
857 cmpl $
0x80808080, %ecx
/* if no null byte in this word */
858 je
.word_aligned /* goto .word_aligned */
859 subl $
4, %eax
/* post-incremented */
861 cmpb $
0, (%eax
) /* if a byte in (%eax) is null */
862 je
.null_found /* goto .null_found */
863 incl
%eax
/* next byte */
864 testl $
3, %eax
/* if %eax not word aligned */
865 jnz
.not_word_aligned /* goto .not_word_aligned */
866 jmp
.word_aligned /* goto .word_aligned */
869 subl
4(%esp
), %eax
/* %eax -= string address */
878 .string "strlen: argument below kernelbase"
883 * Berkeley 4.3 introduced symbolically named interrupt levels
884 * as a way deal with priority in a machine independent fashion.
885 * Numbered priorities are machine specific, and should be
886 * discouraged where possible.
888 * Note, for the machine specific priorities there are
889 * examples listed for devices that use a particular priority.
890 * It should not be construed that all devices of that
891 * type should be at that priority. It is currently were
892 * the current devices fit into the priority scheme based
893 * upon time criticalness.
895 * The underlying assumption of these assignments is that
896 * IPL 10 is the highest level from which a device
897 * routine can call wakeup. Devices that interrupt from higher
898 * levels are restricted in what they can do. If they need
899 * kernels services they should schedule a routine at a lower
900 * level (via software interrupt) to do the required
903 * Examples of this higher usage:
905 * 14 Profiling clock (and PROM uart polling clock)
908 * The serial ports request lower level processing on level 6.
910 * Also, almost all splN routines (where N is a number or a
911 * mnemonic) will do a RAISE(), on the assumption that they are
912 * never used to lower our priority.
913 * The exceptions are:
914 * spl8() Because you can't be above 15 to begin with!
915 * splzs() Because this is used at boot time to lower our
916 * priority, to allow the PROM to poll the uart.
917 * spl0() Used to lower priority to 0.
923 #define SETPRI(level) \
924 movl $
/**/level
, %edi;
/* new priority */ \
925 jmp do_splx
/* redirect to do_splx */
927 #define RAISE(level) \
928 movl $
/**/level
, %edi;
/* new priority */ \
929 jmp splr
/* redirect to splr */
931 #elif defined(__i386)
933 #define SETPRI(level) \
934 pushl $
/**/level;
/* new priority */ \
935 call do_splx;
/* invoke common splx code */ \
936 addl $
4, %esp;
/* unstack arg */ \
939 #define RAISE(level) \
940 pushl $
/**/level;
/* new priority */ \
941 call splr;
/* invoke common splr code */ \
942 addl $
4, %esp;
/* unstack args */ \
947 /* locks out all interrupts, including memory errors */
952 /* just below the level that profiling runs */
957 /* sun specific - highest priority onboard serial i/o asy ports */
959 SETPRI
(12) /* Can't be a RAISE, as it's used to lower us */
965 ALTENTRY
(i_ddi_splhigh
)
969 SET_SIZE
(i_ddi_splhigh
)
974 /* allow all interrupts */
980 /* splx implementation */
982 jmp do_splx
/* redirect to common splx code */
989 * Read and write the %gs register
1008 #if defined(__amd64)
1010 #elif defined(__i386)
1018 #if defined(__amd64)
1020 #elif defined(__i386)
1024 SET_SIZE
(wait_500ms
)
1026 #define RESET_METHOD_KBC 1
1027 #define RESET_METHOD_PORT92 2
1028 #define RESET_METHOD_PCI 4
1030 DGDEF3
(pc_reset_methods
, 4, 8)
1031 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1036 testl $RESET_METHOD_KBC
, pc_reset_methods
1037 #elif defined(__amd64)
1038 testl $RESET_METHOD_KBC
, pc_reset_methods
(%rip
)
1043 / Try the classic keyboard controller-triggered reset.
1049 / Wait up to
500 milliseconds here for the keyboard controller
1050 / to pull the reset line. On some systems where the keyboard
1051 / controller is slow to pull the reset line
, the next reset method
1052 / may
be executed
(which may
be bad if those systems hang when the
1053 / next reset method is used
, e.g. Ferrari
3400 (doesn
't like port 92),
1054 / and Ferrari 4000 (doesn't like the cf9 reset method
))
1060 testl $RESET_METHOD_PORT92
, pc_reset_methods
1061 #elif defined(__amd64)
1062 testl $RESET_METHOD_PORT92
, pc_reset_methods
(%rip
)
1067 / Try port
0x92 fast reset
1071 cmpb $
0xff, %al
/ If port
's not there, we should get back 0xFF
1073 testb $1, %al / If bit 0
1074 jz 2f / is clear, jump to perform the reset
1075 andb $0xfe, %al / otherwise,
1076 outb (%dx) / clear bit 0 first, then
1078 orb $1, %al / Set bit 0
1079 outb (%dx) / and reset the system
1086 testl $RESET_METHOD_PCI, pc_reset_methods
1087 #elif defined(__amd64)
1088 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1092 / Try the PCI (soft) reset vector (should work on all modern systems,
1093 / but has been shown to cause problems on 450NX systems, and some newer
1094 / systems (e.g. ATI IXP400-equipped systems))
1095 / When resetting via this method, 2 writes are required. The first
1096 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1098 / The reset occurs on the second write, during bit 2's transition from
1101 movb $
0x2, %al
/ Reset mode
= hard
, no power cycle
1110 / port
0xcf9 failed also. Last-ditch effort is to
1111 / triple-fault the CPU.
1112 / Also
, use triple fault for EFI firmware
1115 #if defined(__amd64)
1117 pushq $
0x0 / IDT base of
0, limit of
0 + 2 unused bytes
1119 #elif defined(__i386)
1121 pushl $
0x0 / IDT base of
0, limit of
0 + 2 unused bytes
1124 int $
0x0 / Trigger interrupt
, generate triple-fault
1134 * C callable in and out routines
1138 #if defined(__amd64)
1147 #elif defined(__i386)
1153 movw PORT
(%esp
), %dx
1154 movl VAL
(%esp
), %eax
1162 #if defined(__amd64)
1167 D16 outl
(%dx
) /* XX64 why not outw? */
1171 #elif defined(__i386)
1174 movw PORT
(%esp
), %dx
1183 #if defined(__amd64)
1192 #elif defined(__i386)
1195 movw PORT
(%esp
), %dx
1204 #if defined(__amd64)
1213 #elif defined(__i386)
1216 movw PORT
(%esp
), %dx
1224 #if defined(__amd64)
1233 #elif defined(__i386)
1237 movw PORT
(%esp
), %dx
1246 #if defined(__amd64)
1255 #elif defined(__i386)
1259 movw PORT
(%esp
), %dx
1268 #if defined(__amd64)
1278 #elif defined(__i386)
1281 * The arguments and saved registers are on the stack in the
1288 * If additional values are pushed onto the stack, make sure
1289 * to adjust the following constants accordingly.
1297 movl PORT
(%esp
), %edx
1298 movl ADDR
(%esp
), %esi
1299 movl COUNT
(%esp
), %ecx
1310 #if defined(__amd64)
1320 #elif defined(__i386)
1324 movl PORT
(%esp
), %edx
1325 movl ADDR
(%esp
), %edi
1326 movl COUNT
(%esp
), %ecx
1337 #if defined(__amd64)
1348 #elif defined(__i386)
1351 * The arguments and saved registers are on the stack in the
1358 * If additional values are pushed onto the stack, make sure
1359 * to adjust the following constants accordingly.
1367 movl IO_ADDR
(%esp
), %edi
1368 movl IO_COUNT
(%esp
), %ecx
1369 movl IO_PORT
(%esp
), %edx
1380 * Input a stream of 32-bit words.
1381 * NOTE: count is a DWORD count.
1384 #if defined(__amd64)
1395 #elif defined(__i386)
1399 movl IO_ADDR
(%esp
), %edi
1400 movl IO_COUNT
(%esp
), %ecx
1401 movl IO_PORT
(%esp
), %edx
1411 * Output a stream of bytes
1412 * NOTE: count is a byte count
1415 #if defined(__amd64)
1425 #elif defined(__i386)
1429 movl IO_ADDR
(%esp
), %esi
1430 movl IO_COUNT
(%esp
), %ecx
1431 movl IO_PORT
(%esp
), %edx
1441 * Output a stream of 32-bit words
1442 * NOTE: count is a DWORD count
1445 #if defined(__amd64)
1455 #elif defined(__i386)
1459 movl IO_ADDR
(%esp
), %esi
1460 movl IO_COUNT
(%esp
), %ecx
1461 movl IO_PORT
(%esp
), %edx
1474 * void int_cmci(void)
1489 movl boothowto
, %eax
1490 andl $RB_DEBUG
, %eax
1495 rep; ret
/* use 2 byte return instruction when branch target */
1496 /* AMD Software Optimization Guide - Section 6.2 */
1506 #if defined(__amd64)
1513 addq
%rsi
, %rdi
/* end = &cp[size] */
1515 cmpq
%rdi
, %rsi
/* while (cp < end */
1517 movzbq
(%rsi
), %r8 /* %r8 = *cp */
1518 incq
%rsi
/* cp++ */
1519 testb
%cl
, (%r8, %rdx
)
1520 jz
.scanloop /* && (table[*cp] & mask) == 0) */
1521 decq
%rsi
/* (fix post-increment) */
1524 subl
%esi
, %eax
/* return (end - cp) */
1528 #elif defined(__i386)
1533 movb
24(%esp
), %cl
/* mask = %cl */
1534 movl
16(%esp
), %esi
/* cp = %esi */
1535 movl
20(%esp
), %edx
/* table = %edx */
1537 addl
12(%esp
), %edi
/* end = &cp[size]; */
1539 cmpl %edi
, %esi
/* while (cp < end */
1541 movzbl
(%esi
), %eax
/* %al = *cp */
1542 incl
%esi
/* cp++ */
1543 movb
(%edx
, %eax
), %al
/* %al = table[*cp] */
1545 jz
.scanloop /* && (table[*cp] & mask) == 0) */
1546 dec %esi
/* post-incremented */
1549 subl
%esi
, %eax
/* return (end - cp) */
1558 * Replacement functions for ones that are normally inlined.
1559 * In addition to the copy in i86.il, they are defined here just in case.
1563 #if defined(__amd64)
1566 ENTRY
(clear_int_flag
)
1570 leaq xpv_panicking
, %rdi
1574 CLIRET
(%rdi
, %dl
) /* returns event mask in %dl */
1576 * Synthesize the PS_IE bit from the event mask bit
1578 andq $_BITNOT
(PS_IE
), %rax
1588 SET_SIZE
(clear_int_flag
)
1589 SET_SIZE
(intr_clear
)
1591 #elif defined(__i386)
1594 ENTRY
(clear_int_flag
)
1598 leal xpv_panicking
, %edx
1602 CLIRET
(%edx
, %cl
) /* returns event mask in %cl */
1604 * Synthesize the PS_IE bit from the event mask bit
1606 andl $_BITNOT
(PS_IE
), %eax
1616 SET_SIZE
(clear_int_flag
)
1617 SET_SIZE
(intr_clear
)
1622 #if defined(__amd64)
1625 movq
%gs
:CPU_SELF
, %rax
1629 #elif defined(__i386)
1632 movl
%gs
:CPU_SELF
, %eax
1638 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
1639 * These functions reverse the byte order of the input parameter and returns
1640 * the result. This is to convert the byte order from host byte order
1641 * (little endian) to network byte order (big endian), or vice versa.
1645 #if defined(__amd64)
1655 /* XX64 there must be shorter sequences for this */
1664 /* XX64 there must be better sequences for this */
1674 #elif defined(__i386)
1707 #if defined(__amd64)
1710 ENTRY
(restore_int_flag
)
1714 leaq xpv_panicking
, %rsi
1719 * Since we're -really- running unprivileged, our attempt
1720 * to change the state of the IF bit will be ignored.
1721 * The virtual IF bit is tweaked by CLI and STI.
1723 IE_TO_EVENT_MASK
(%rsi
, %rdi
)
1729 SET_SIZE
(restore_int_flag
)
1730 SET_SIZE
(intr_restore
)
1732 #elif defined(__i386)
1735 ENTRY
(restore_int_flag
)
1736 testl $PS_IE
, 4(%esp
)
1739 leal xpv_panicking
, %edx
1744 * Since we're -really- running unprivileged, our attempt
1745 * to change the state of the IF bit will be ignored.
1746 * The virtual IF bit is tweaked by CLI and STI.
1748 IE_TO_EVENT_MASK
(%edx
, 4(%esp
))
1754 SET_SIZE
(restore_int_flag
)
1755 SET_SIZE
(intr_restore
)
1766 #if defined(__amd64)
1768 #elif defined(__i386)
1776 #if defined(__amd64)
1778 ENTRY
(dtrace_interrupt_disable
)
1782 leaq xpv_panicking
, %rdi
1785 jne
.dtrace_interrupt_disable_done
1786 CLIRET
(%rdi
, %dl
) /* returns event mask in %dl */
1788 * Synthesize the PS_IE bit from the event mask bit
1790 andq $_BITNOT
(PS_IE
), %rax
1792 jnz
.dtrace_interrupt_disable_done
1797 .dtrace_interrupt_disable_done:
1799 SET_SIZE
(dtrace_interrupt_disable
)
1801 #elif defined(__i386)
1803 ENTRY
(dtrace_interrupt_disable
)
1807 leal xpv_panicking
, %edx
1810 jne
.dtrace_interrupt_disable_done
1811 CLIRET
(%edx
, %cl
) /* returns event mask in %cl */
1813 * Synthesize the PS_IE bit from the event mask bit
1815 andl $_BITNOT
(PS_IE
), %eax
1817 jnz
.dtrace_interrupt_disable_done
1822 .dtrace_interrupt_disable_done:
1824 SET_SIZE
(dtrace_interrupt_disable
)
1829 #if defined(__amd64)
1831 ENTRY
(dtrace_interrupt_enable
)
1835 leaq xpv_panicking
, %rdx
1838 jne
.dtrace_interrupt_enable_done
1840 * Since we're -really- running unprivileged, our attempt
1841 * to change the state of the IF bit will be ignored. The
1842 * virtual IF bit is tweaked by CLI and STI.
1844 IE_TO_EVENT_MASK
(%rdx
, %rdi
)
1846 .dtrace_interrupt_enable_done:
1848 SET_SIZE
(dtrace_interrupt_enable
)
1850 #elif defined(__i386)
1852 ENTRY
(dtrace_interrupt_enable
)
1857 leal xpv_panicking
, %edx
1860 jne
.dtrace_interrupt_enable_done
1862 * Since we're -really- running unprivileged, our attempt
1863 * to change the state of the IF bit will be ignored. The
1864 * virtual IF bit is tweaked by CLI and STI.
1866 IE_TO_EVENT_MASK
(%edx
, %eax
)
1868 .dtrace_interrupt_enable_done:
1870 SET_SIZE
(dtrace_interrupt_enable
)
1876 ENTRY
(dtrace_membar_producer
)
1877 rep; ret
/* use 2 byte return instruction when branch target */
1878 /* AMD Software Optimization Guide - Section 6.2 */
1879 SET_SIZE
(dtrace_membar_producer
)
1881 ENTRY
(dtrace_membar_consumer
)
1882 rep; ret
/* use 2 byte return instruction when branch target */
1883 /* AMD Software Optimization Guide - Section 6.2 */
1884 SET_SIZE
(dtrace_membar_consumer
)
1888 #if defined(__amd64)
1891 movq
%gs
:CPU_THREAD
, %rax
1895 #elif defined(__i386)
1898 movl
%gs
:CPU_THREAD
, %eax
1905 * Checksum routine for Internet Protocol Headers
1909 #if defined(__amd64)
1915 movq postbootkernelbase
(%rip
), %rax
1920 leaq
.ip_ocsum_panic_msg(%rip), %rdi
1923 .ip_ocsum_panic_msg:
1924 .string "ip_ocsum: address 0x%p below kernelbase\n"
1927 movl
%esi
, %ecx
/* halfword_count */
1928 movq
%rdi
, %rsi
/* address */
1929 /* partial sum in %edx */
1934 jnz
.ip_csum_notaligned
1935 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
1937 /* XX64 opportunities for prefetch? */
1938 /* XX64 compute csum with 64 bit quantities? */
1972 adcl
60(%rsi
), %eax
/* could be adding -1 and -1 with a carry */
1974 adcl $
0, %eax
/* could be adding -1 in eax with a carry */
1984 movl
%edx
, %eax
/* form a 16 bit checksum by */
1985 shrl $
16, %eax
/* adding two halves of 32 bit checksum */
1992 .ip_csum_notaligned:
1999 jmp
.ip_csum_aligned
2006 movzwl
(%rsi
, %rcx
, 2), %edi
2015 leaq
.ip_ocsum_jmptbl(%rip), %rdi
2016 leaq
(%rdi
, %rcx
, 8), %rdi
2023 .quad .only0, .only4, .only8, .only12, .only16, .only20
2024 .quad .only24, .only28, .only32, .only36, .only40, .only44
2025 .quad .only48, .only52, .only56, .only60
2028 #elif defined(__i386)
2036 movl
12(%ebp
), %ecx
/* count of half words */
2037 movl
16(%ebp
), %edx
/* partial checksum */
2044 jnz
.ip_csum_notaligned
2080 adcl
60(%esi
), %eax
/* We could be adding -1 and -1 with a carry */
2082 adcl $
0, %eax
/* we could be adding -1 in eax with a carry */
2092 movl
%edx
, %eax
/* form a 16 bit checksum by */
2093 shrl $
16, %eax
/* adding two halves of 32 bit checksum */
2097 popl
%edi
/* restore registers */
2103 .ip_csum_notaligned:
2110 jmp
.ip_csum_aligned
2117 movzwl
(%esi
, %ecx
, 2), %edi
2126 movl $
.ip_ocsum_jmptbl, %edi
2127 lea
(%edi
, %ecx
, 4), %edi
2137 .long .only0, .only4, .only8, .only12, .only16, .only20
2138 .long .only24, .only28, .only32, .only36, .only40, .only44
2139 .long .only48, .only52, .only56, .only60
2145 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2146 * Provided to manipulate hrtime_t values.
2149 #if defined(__amd64)
2152 xorl
%edx
, %edx
/* XX64 joe, paranoia? */
2160 #elif defined(__i386)
2171 #if defined(notused)
2180 #endif /* notused */
2183 #if defined(__amd64)
2186 shrq $
3, %rsi
/* convert %rsi from byte to quadword count */
2188 movq
%rsi
, %rcx
/* move count into rep control register */
2189 movq
%rdi
, %rsi
/* move addr into lodsq control reg. */
2190 rep lodsq
/* scan the memory range */
2192 rep; ret
/* use 2 byte return instruction when branch target */
2193 /* AMD Software Optimization Guide - Section 6.2 */
2194 SET_SIZE
(scan_memory
)
2196 #elif defined(__i386)
2201 movl
16(%esp
), %ecx
/* move 2nd arg into rep control register */
2202 shrl $
2, %ecx
/* convert from byte count to word count */
2204 movl
12(%esp
), %esi
/* move 1st arg into lodsw control register */
2205 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2211 SET_SIZE
(scan_memory
)
2217 #if defined(__amd64)
2227 #elif defined(__i386)
2242 #if defined(__amd64)
2254 #elif defined(__i386)
2276 #define XMSR_ACCESS_VAL $0x9c5a203a
2278 #if defined(__amd64)
2301 movl XMSR_ACCESS_VAL
, %edi
/* this value is needed to access MSR */
2313 movl XMSR_ACCESS_VAL
, %edi
/* this value is needed to access MSR */
2325 .byte 0x0f,0x01,0xd0
2337 .byte 0x0f,0x01,0xd1
2341 #elif defined(__i386)
2362 movl XMSR_ACCESS_VAL
, %edi
/* this value is needed to access MSR */
2376 movl XMSR_ACCESS_VAL
, %edi
/* this value is needed to access MSR */
2386 .byte 0x0f,0x01,0xd0
2395 .byte 0x0f,0x01,0xd1
2401 ENTRY
(invalidate_cache
)
2404 SET_SIZE
(invalidate_cache
)
2408 #if defined(__amd64)
2413 * Only a few of the hardware control registers or descriptor tables
2414 * are directly accessible to us, so just zero the structure.
2416 * XXPV Perhaps it would be helpful for the hypervisor to return
2417 * virtualized versions of these for post-mortem use.
2418 * (Need to reevaluate - perhaps it already does!)
2420 pushq
%rdi
/* save *crp */
2426 * Dump what limited information we can
2429 movq
%rax
, CREG_CR0
(%rdi
) /* cr0 */
2431 movq
%rax
, CREG_CR2
(%rdi
) /* cr2 */
2433 movq
%rax
, CREG_CR3
(%rdi
) /* cr3 */
2435 movq
%rax
, CREG_CR4
(%rdi
) /* cr4 */
2439 #define GETMSR(r, off, d) \
2442 movl
%eax
, off
(d
); \
2446 movq
%rax
, CREG_GDT+
8(%rdi
)
2447 sgdt CREG_GDT
(%rdi
) /* 10 bytes */
2448 movq
%rax
, CREG_IDT+
8(%rdi
)
2449 sidt CREG_IDT
(%rdi
) /* 10 bytes */
2450 movq
%rax
, CREG_LDT
(%rdi
)
2451 sldt CREG_LDT
(%rdi
) /* 2 bytes */
2452 movq
%rax
, CREG_TASKR
(%rdi
)
2453 str CREG_TASKR
(%rdi
) /* 2 bytes */
2455 movq
%rax
, CREG_CR0
(%rdi
) /* cr0 */
2457 movq
%rax
, CREG_CR2
(%rdi
) /* cr2 */
2459 movq
%rax
, CREG_CR3
(%rdi
) /* cr3 */
2461 movq
%rax
, CREG_CR4
(%rdi
) /* cr4 */
2463 movq
%rax
, CREG_CR8
(%rdi
) /* cr8 */
2464 GETMSR
(MSR_AMD_KGSBASE
, CREG_KGSBASE
, %rdi
)
2465 GETMSR
(MSR_AMD_EFER
, CREG_EFER
, %rdi
)
2472 #elif defined(__i386)
2477 * Only a few of the hardware control registers or descriptor tables
2478 * are directly accessible to us, so just zero the structure.
2480 * XXPV Perhaps it would be helpful for the hypervisor to return
2481 * virtualized versions of these for post-mortem use.
2482 * (Need to reevaluate - perhaps it already does!)
2492 * Dump what limited information we can
2495 movl
%eax
, CREG_CR0
(%edx
) /* cr0 */
2497 movl
%eax
, CREG_CR2
(%edx
) /* cr2 */
2499 movl
%eax
, CREG_CR3
(%edx
) /* cr3 */
2501 movl
%eax
, CREG_CR4
(%edx
) /* cr4 */
2506 movw $
0, CREG_GDT+
6(%edx
)
2507 movw $
0, CREG_IDT+
6(%edx
)
2508 sgdt CREG_GDT
(%edx
) /* gdt */
2509 sidt CREG_IDT
(%edx
) /* idt */
2510 sldt CREG_LDT
(%edx
) /* ldt */
2511 str CREG_TASKR
(%edx
) /* task */
2513 movl
%eax
, CREG_CR0
(%edx
) /* cr0 */
2515 movl
%eax
, CREG_CR2
(%edx
) /* cr2 */
2517 movl
%eax
, CREG_CR3
(%edx
) /* cr3 */
2518 bt $X86FSET_LARGEPAGE
, x86_featureset
2521 movl
%eax
, CREG_CR4
(%edx
) /* cr4 */
2524 movl $
0, CREG_CR4
(%edx
)
2534 * A panic trigger is a word which is updated atomically and can only be set
2535 * once. We atomically store 0xDEFACEDD and load the old value. If the
2536 * previous value was 0, we succeed and return 1; otherwise return 0.
2537 * This allows a partially corrupt trigger to still trigger correctly. DTrace
2538 * has its own version of this function to allow it to panic correctly from
2542 #if defined(__amd64)
2544 ENTRY_NP
(panic_trigger
)
2546 movl $
0xdefacedd, %edx
2555 SET_SIZE
(panic_trigger
)
2557 ENTRY_NP
(dtrace_panic_trigger
)
2559 movl $
0xdefacedd, %edx
2568 SET_SIZE
(dtrace_panic_trigger
)
2570 #elif defined(__i386)
2572 ENTRY_NP
(panic_trigger
)
2573 movl
4(%esp
), %edx
/ %edx
= address of trigger
2574 movl $
0xdefacedd, %eax
/ %eax
= 0xdefacedd
2576 xchgl
%eax
, (%edx
) / exchange
%eax
and the trigger
2577 cmpl $
0, %eax
/ if
(%eax
== 0x0)
2579 movl $
0, %eax
/ else
2583 SET_SIZE
(panic_trigger
)
2585 ENTRY_NP
(dtrace_panic_trigger
)
2586 movl
4(%esp
), %edx
/ %edx
= address of trigger
2587 movl $
0xdefacedd, %eax
/ %eax
= 0xdefacedd
2589 xchgl
%eax
, (%edx
) / exchange
%eax
and the trigger
2590 cmpl $
0, %eax
/ if
(%eax
== 0x0)
2592 movl $
0, %eax
/ else
2596 SET_SIZE
(dtrace_panic_trigger
)
2601 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
2602 * into the panic code implemented in panicsys(). vpanic() is responsible
2603 * for passing through the format string and arguments, and constructing a
2604 * regs structure on the stack into which it saves the current register
2605 * values. If we are not dying due to a fatal trap, these registers will
2606 * then be preserved in panicbuf as the current processor state. Before
2607 * invoking panicsys(), vpanic() activates the first panic trigger (see
2608 * kernel/os/panic.c) and switches to the panic_stack if successful. Note that
2609 * DTrace takes a slightly different panic path if it must panic from probe
2610 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
2611 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
2612 * branches back into vpanic().
2615 #if defined(__amd64)
2617 ENTRY_NP
(vpanic
) /* Initial stack layout: */
2619 pushq
%rbp
/* | %rip | 0x60 */
2620 movq
%rsp
, %rbp
/* | %rbp | 0x58 */
2621 pushfq
/* | rfl | 0x50 */
2622 pushq
%r11 /* | %r11 | 0x48 */
2623 pushq
%r10 /* | %r10 | 0x40 */
2624 pushq
%rbx
/* | %rbx | 0x38 */
2625 pushq
%rax
/* | %rax | 0x30 */
2626 pushq
%r9 /* | %r9 | 0x28 */
2627 pushq
%r8 /* | %r8 | 0x20 */
2628 pushq
%rcx
/* | %rcx | 0x18 */
2629 pushq
%rdx
/* | %rdx | 0x10 */
2630 pushq
%rsi
/* | %rsi | 0x8 alist */
2631 pushq
%rdi
/* | %rdi | 0x0 format */
2633 movq
%rsp
, %rbx
/* %rbx = current %rsp */
2635 leaq panic_quiesce
(%rip
), %rdi
/* %rdi = &panic_quiesce */
2636 call panic_trigger
/* %eax = panic_trigger() */
2640 * The panic_trigger result is in %eax from the call above, and
2641 * dtrace_panic places it in %eax before branching here.
2642 * The rdmsr instructions that follow below will clobber %eax so
2643 * we stash the panic_trigger result in %r11d.
2650 * If panic_trigger() was successful, we are the first to initiate a
2651 * panic: we now switch to the reserved panic_stack before continuing.
2653 leaq panic_stack
(%rip
), %rsp
2654 addq $PANICSTKSIZE
, %rsp
2655 0: subq $REGSIZE
, %rsp
2657 * Now that we've got everything set up, store the register values as
2658 * they were when we entered vpanic() to the designated location in
2659 * the regs structure we allocated on the stack.
2661 movq
0x0(%rbx
), %rcx
2662 movq
%rcx
, REGOFF_RDI
(%rsp
)
2663 movq
0x8(%rbx
), %rcx
2664 movq
%rcx
, REGOFF_RSI
(%rsp
)
2665 movq
0x10(%rbx
), %rcx
2666 movq
%rcx
, REGOFF_RDX
(%rsp
)
2667 movq
0x18(%rbx
), %rcx
2668 movq
%rcx
, REGOFF_RCX
(%rsp
)
2669 movq
0x20(%rbx
), %rcx
2671 movq
%rcx
, REGOFF_R8
(%rsp
)
2672 movq
0x28(%rbx
), %rcx
2673 movq
%rcx
, REGOFF_R9
(%rsp
)
2674 movq
0x30(%rbx
), %rcx
2675 movq
%rcx
, REGOFF_RAX
(%rsp
)
2676 movq
0x38(%rbx
), %rcx
2677 movq
%rcx
, REGOFF_RBX
(%rsp
)
2678 movq
0x58(%rbx
), %rcx
2680 movq
%rcx
, REGOFF_RBP
(%rsp
)
2681 movq
0x40(%rbx
), %rcx
2682 movq
%rcx
, REGOFF_R10
(%rsp
)
2683 movq
0x48(%rbx
), %rcx
2684 movq
%rcx
, REGOFF_R11
(%rsp
)
2685 movq
%r12, REGOFF_R12
(%rsp
)
2687 movq
%r13, REGOFF_R13
(%rsp
)
2688 movq
%r14, REGOFF_R14
(%rsp
)
2689 movq
%r15, REGOFF_R15
(%rsp
)
2693 movq
%rcx
, REGOFF_DS
(%rsp
)
2695 movq
%rcx
, REGOFF_ES
(%rsp
)
2697 movq
%rcx
, REGOFF_FS
(%rsp
)
2699 movq
%rcx
, REGOFF_GS
(%rsp
)
2701 movq $
0, REGOFF_TRAPNO
(%rsp
)
2703 movq $
0, REGOFF_ERR
(%rsp
)
2704 leaq vpanic
(%rip
), %rcx
2705 movq
%rcx
, REGOFF_RIP
(%rsp
)
2708 movq
%rcx
, REGOFF_CS
(%rsp
)
2709 movq
0x50(%rbx
), %rcx
2710 movq
%rcx
, REGOFF_RFL
(%rsp
)
2713 movq
%rcx
, REGOFF_RSP
(%rsp
)
2716 movq
%rcx
, REGOFF_SS
(%rsp
)
2719 * panicsys(format, alist, rp, on_panic_stack)
2721 movq REGOFF_RDI
(%rsp
), %rdi
/* format */
2722 movq REGOFF_RSI
(%rsp
), %rsi
/* alist */
2723 movq
%rsp
, %rdx
/* struct regs */
2724 movl
%r11d
, %ecx
/* on_panic_stack */
2742 ENTRY_NP
(dtrace_vpanic
) /* Initial stack layout: */
2744 pushq
%rbp
/* | %rip | 0x60 */
2745 movq
%rsp
, %rbp
/* | %rbp | 0x58 */
2746 pushfq
/* | rfl | 0x50 */
2747 pushq
%r11 /* | %r11 | 0x48 */
2748 pushq
%r10 /* | %r10 | 0x40 */
2749 pushq
%rbx
/* | %rbx | 0x38 */
2750 pushq
%rax
/* | %rax | 0x30 */
2751 pushq
%r9 /* | %r9 | 0x28 */
2752 pushq
%r8 /* | %r8 | 0x20 */
2753 pushq
%rcx
/* | %rcx | 0x18 */
2754 pushq
%rdx
/* | %rdx | 0x10 */
2755 pushq
%rsi
/* | %rsi | 0x8 alist */
2756 pushq
%rdi
/* | %rdi | 0x0 format */
2758 movq
%rsp
, %rbx
/* %rbx = current %rsp */
2760 leaq panic_quiesce
(%rip
), %rdi
/* %rdi = &panic_quiesce */
2761 call dtrace_panic_trigger
/* %eax = dtrace_panic_trigger() */
2764 SET_SIZE
(dtrace_vpanic
)
2766 #elif defined(__i386)
2768 ENTRY_NP
(vpanic
) / Initial stack layout
:
2770 pushl
%ebp
/ |
%eip |
20
2771 movl
%esp
, %ebp
/ |
%ebp |
16
2772 pushl
%eax
/ |
%eax |
12
2773 pushl
%ebx
/ |
%ebx |
8
2774 pushl
%ecx
/ |
%ecx |
4
2775 pushl
%edx
/ |
%edx |
0
2777 movl
%esp
, %ebx
/ %ebx
= current stack pointer
2779 lea panic_quiesce
, %eax
/ %eax
= &panic_quiesce
2780 pushl
%eax
/ push
&panic_quiesce
2781 call panic_trigger
/ %eax
= panic_trigger
()
2782 addl $
4, %esp
/ reset stack pointer
2785 cmpl $
0, %eax
/ if
(%eax
== 0)
2789 * If panic_trigger() was successful, we are the first to initiate a
2790 * panic: we now switch to the reserved panic_stack before continuing.
2792 lea panic_stack
, %esp
/ %esp
= panic_stack
2793 addl $PANICSTKSIZE
, %esp
/ %esp
+= PANICSTKSIZE
2795 0: subl $REGSIZE
, %esp
/ allocate struct regs
2798 * Now that we've got everything set up, store the register values as
2799 * they were when we entered vpanic() to the designated location in
2800 * the regs structure we allocated on the stack.
2802 #if !defined(__GNUC_AS__)
2804 movl
%edx
, REGOFF_GS
(%esp
)
2806 movl
%edx
, REGOFF_FS
(%esp
)
2808 movl
%edx
, REGOFF_ES
(%esp
)
2810 movl
%edx
, REGOFF_DS
(%esp
)
2811 #else /* __GNUC_AS__ */
2813 mov
%edx
, REGOFF_GS
(%esp
)
2815 mov
%edx
, REGOFF_FS
(%esp
)
2817 mov
%edx
, REGOFF_ES
(%esp
)
2819 mov
%edx
, REGOFF_DS
(%esp
)
2820 #endif /* __GNUC_AS__ */
2821 movl
%edi
, REGOFF_EDI
(%esp
)
2822 movl
%esi
, REGOFF_ESI
(%esp
)
2824 movl
%ecx
, REGOFF_EBP
(%esp
)
2827 movl
%ecx
, REGOFF_ESP
(%esp
)
2829 movl
%ecx
, REGOFF_EBX
(%esp
)
2831 movl
%ecx
, REGOFF_EDX
(%esp
)
2833 movl
%ecx
, REGOFF_ECX
(%esp
)
2835 movl
%ecx
, REGOFF_EAX
(%esp
)
2836 movl $
0, REGOFF_TRAPNO
(%esp
)
2837 movl $
0, REGOFF_ERR
(%esp
)
2839 movl
%ecx
, REGOFF_EIP
(%esp
)
2840 #if !defined(__GNUC_AS__)
2842 #else /* __GNUC_AS__ */
2844 #endif /* __GNUC_AS__ */
2845 movl
%edx
, REGOFF_CS
(%esp
)
2850 * Synthesize the PS_IE bit from the event mask bit
2853 KPREEMPT_DISABLE
(%edx
)
2854 EVENT_MASK_TO_IE
(%edx
, %ecx
)
2856 KPREEMPT_ENABLE_NOKP
(%edx
)
2858 movl
%ecx
, REGOFF_EFL
(%esp
)
2859 movl $
0, REGOFF_UESP
(%esp
)
2860 #if !defined(__GNUC_AS__)
2862 #else /* __GNUC_AS__ */
2864 #endif /* __GNUC_AS__ */
2865 movl
%edx
, REGOFF_SS
(%esp
)
2867 movl
%esp
, %ecx
/ %ecx
= ®s
2868 pushl
%eax
/ push on_panic_stack
2869 pushl
%ecx
/ push
®s
2870 movl
12(%ebp
), %ecx
/ %ecx
= alist
2871 pushl
%ecx
/ push alist
2872 movl
8(%ebp
), %ecx
/ %ecx
= format
2873 pushl
%ecx
/ push format
2874 call panicsys
/ panicsys
();
2875 addl $
16, %esp
/ pop arguments
2886 ENTRY_NP
(dtrace_vpanic
) / Initial stack layout
:
2888 pushl
%ebp
/ |
%eip |
20
2889 movl
%esp
, %ebp
/ |
%ebp |
16
2890 pushl
%eax
/ |
%eax |
12
2891 pushl
%ebx
/ |
%ebx |
8
2892 pushl
%ecx
/ |
%ecx |
4
2893 pushl
%edx
/ |
%edx |
0
2895 movl
%esp
, %ebx
/ %ebx
= current stack pointer
2897 lea panic_quiesce
, %eax
/ %eax
= &panic_quiesce
2898 pushl
%eax
/ push
&panic_quiesce
2899 call dtrace_panic_trigger
/ %eax
= dtrace_panic_trigger
()
2900 addl $
4, %esp
/ reset stack pointer
2901 jmp vpanic_common
/ jump back to common code
2903 SET_SIZE
(dtrace_vpanic
)
2908 DGDEF3
(timedelta
, 8, 8)
2912 * initialized to a non zero value to make pc_gethrtime()
2913 * work correctly even before clock is initialized
2915 DGDEF3
(hrtime_base
, 8, 8)
2916 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
2918 DGDEF3
(adj_shift
, 4, 4)
2921 #if defined(__amd64)
2928 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
2929 * hres_last_tick can only be modified while holding CLOCK_LOCK).
2930 * At worst, performing this now instead of under CLOCK_LOCK may
2931 * introduce some jitter in pc_gethrestime().
2933 call
*gethrtimef
(%rip
)
2936 leaq hres_lock
(%rip
), %rax
2941 jz
.CL3 /* got it */
2943 cmpb $
0, (%rax
) /* possible to get lock? */
2946 jmp
.CL1 /* yes, try again */
2949 * compute the interval since last time hres_tick was called
2950 * and adjust hrtime_base and hrestime accordingly
2951 * hrtime_base is an 8 byte value (in nsec), hrestime is
2952 * a timestruc_t (sec, nsec)
2954 leaq hres_last_tick
(%rip
), %rax
2957 addq
%r8, hrtime_base
(%rip
) /* add interval to hrtime_base */
2958 addq
%r8, hrestime+
8(%rip
) /* add interval to hrestime.tv_nsec */
2960 * Now that we have CLOCK_LOCK, we can update hres_last_tick
2967 * release the hres_lock
2969 incl hres_lock
(%rip
)
2974 #elif defined(__i386)
2983 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
2984 * hres_last_tick can only be modified while holding CLOCK_LOCK).
2985 * At worst, performing this now instead of under CLOCK_LOCK may
2986 * introduce some jitter in pc_gethrestime().
2992 movl $hres_lock
, %eax
2999 cmpb $
0, (%eax
) / possible to get lock?
3002 jmp
.CL1 / yes, try again
3005 * compute the interval since last time hres_tick was called
3006 * and adjust hrtime_base and hrestime accordingly
3007 * hrtime_base is an 8 byte value (in nsec), hrestime is
3008 * timestruc_t (sec, nsec)
3011 lea hres_last_tick
, %eax
3019 addl
%edx
, hrtime_base
/ add interval to hrtime_base
3020 adcl
%ecx
, hrtime_base+
4
3022 addl
%edx
, hrestime+
4 / add interval to hrestime.tv_nsec
3025 / Now that we have CLOCK_LOCK
, we can update hres_last_tick.
3030 / get hrestime at this moment. used as base for pc_gethrestime
3032 / Apply adjustment
, if any
3034 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3042 / if
(hrestime_adj
== 0)
3044 / else if
(hrestime_adj
> 0) {
3045 / if
(hrestime_adj
< HRES_ADJ
)
3046 / adj
= hrestime_adj;
3051 / if
(hrestime_adj
< -(HRES_ADJ
))
3052 / adj
= -(HRES_ADJ
);
3054 / adj
= hrestime_adj;
3058 / hrestime_adj
= timedelta;
3059 / hrestime.tv_nsec
+= adj;
3061 / while
(hrestime.tv_nsec
>= NANOSEC
) {
3063 / hrestime.tv_sec+
+;
3064 / hrestime.tv_nsec
-= NANOSEC;
3068 movl hrestime_adj
, %esi
/ if
(hrestime_adj
== 0)
3069 movl hrestime_adj+
4, %edx
3074 subl
%ecx
, %ecx
/ yes
, adj
= 0;
3082 andl
%eax
, %eax
/ if
(hrestime_adj
> 0)
3085 / In the following comments
, HRES_ADJ is used
, while in the code
3086 / max_hres_adj is used.
3088 / The test for
"hrestime_adj < HRES_ADJ" is complicated because
3089 / hrestime_adj is
64-bits
, while HRES_ADJ is
32-bits. We rely
3090 / on the logical equivalence of
:
3092 / !(hrestime_adj
< HRES_ADJ
)
3094 / and the two step sequence
:
3096 / (HRES_ADJ
- lsw
(hrestime_adj
)) generates
a Borrow
/Carry
3098 / which computes whether
or not the least significant
32-bits
3099 / of hrestime_adj is greater than HRES_ADJ
, followed by
:
3101 / Previous Borrow
/Carry
+ -1 + msw
(hrestime_adj
) generates
a Carry
3103 / which generates
a carry whenever step
1 is true
or the most
3104 / significant long of the longlong hrestime_adj is non-zero.
3106 movl max_hres_adj
, %ecx
/ hrestime_adj is positive
3111 movl max_hres_adj
, %ecx
/ adj
= HRES_ADJ;
3115 / The following computation is similar to the one above.
3117 / The test for
"hrestime_adj < -(HRES_ADJ)" is complicated because
3118 / hrestime_adj is
64-bits
, while HRES_ADJ is
32-bits. We rely
3119 / on the logical equivalence of
:
3121 / (hrestime_adj
> -HRES_ADJ
)
3123 / and the two step sequence
:
3125 / (HRES_ADJ
+ lsw
(hrestime_adj
)) generates
a Carry
3127 / which means the least significant
32-bits of hrestime_adj is
3128 / greater than
-HRES_ADJ
, followed by
:
3130 / Previous Carry
+ 0 + msw
(hrestime_adj
) generates
a Carry
3132 / which generates
a carry only when step
1 is true
and the most
3133 / significant long of the longlong hrestime_adj is
-1.
3135 .CL6: / hrestime_adj is negative
3137 addl max_hres_adj
, %ecx
3142 subl max_hres_adj
, %ecx
/ adj
= -(HRES_ADJ
);
3146 movl
%esi
, %ecx
/ adj
= hrestime_adj;
3148 movl timedelta
, %esi
3150 movl timedelta+
4, %eax
3152 movl
%esi
, timedelta
3153 movl
%eax
, timedelta+
4 / timedelta
-= adj;
3154 movl
%esi
, hrestime_adj
3155 movl
%eax
, hrestime_adj+
4 / hrestime_adj
= timedelta;
3156 addl hrestime+
4, %ecx
3158 movl
%ecx
, %eax
/ eax
= tv_nsec
3160 cmpl $NANOSEC
, %eax
/ if
((unsigned long
)tv_nsec
>= NANOSEC
)
3162 incl one_sec
/ yes
, one_sec+
+;
3163 incl hrestime
/ hrestime.tv_sec+
+;
3164 addl $
-NANOSEC
, %eax
/ tv_nsec
-= NANOSEC
3165 jmp
1b / check for more seconds
3168 movl
%eax
, hrestime+
4 / store final into hrestime.tv_nsec
3169 incl hres_lock
/ release the hres_lock
3180 * void prefetch_smap_w(void *)
3182 * Prefetch ahead within a linear list of smap structures.
3183 * Not implemented for ia32. Stub for compatibility.
3187 ENTRY
(prefetch_smap_w
)
3188 rep; ret
/* use 2 byte return instruction when branch target */
3189 /* AMD Software Optimization Guide - Section 6.2 */
3190 SET_SIZE
(prefetch_smap_w
)
3194 * prefetch_page_r(page_t *)
3195 * issue prefetch instructions for a page_t
3198 ENTRY
(prefetch_page_r
)
3199 rep; ret
/* use 2 byte return instruction when branch target */
3200 /* AMD Software Optimization Guide - Section 6.2 */
3201 SET_SIZE
(prefetch_page_r
)
3205 #if defined(__amd64)
3213 movq postbootkernelbase
(%rip
), %r11
3218 0: leaq
.bcmp_panic_msg(%rip), %rdi
3231 #elif defined(__i386)
3235 #define ARG_LENGTH 16
3239 movl
%esp
, %ebp
/ create new stack frame
3241 cmpl $
0, ARG_LENGTH
(%ebp
)
3243 movl postbootkernelbase
, %eax
3244 cmpl %eax
, ARG_S1
(%ebp
)
3246 cmpl %eax
, ARG_S2
(%ebp
)
3248 0: pushl $
.bcmp_panic_msg
3253 pushl
%edi
/ save register variable
3254 movl ARG_S1
(%ebp
), %eax
/ %eax
= address of string
1
3255 movl ARG_S2
(%ebp
), %ecx
/ %ecx
= address of string
2
3256 cmpl %eax
, %ecx
/ if the same string
3257 je
.equal / goto .equal
3258 movl ARG_LENGTH
(%ebp
), %edi
/ %edi
= length in bytes
3259 cmpl $
4, %edi
/ if
%edi
< 4
3260 jb
.byte_check / goto .byte_check
3263 movl
(%ecx
), %edx
/ move
1 word from
(%ecx
) to
%edx
3264 leal
-4(%edi
), %edi
/ %edi
-= 4
3265 cmpl (%eax
), %edx
/ compare
1 word from
(%eax
) with
%edx
3266 jne
.word_not_equal / if not equal, goto .word_not_equal
3267 leal
4(%ecx
), %ecx
/ %ecx
+= 4 (next word
)
3268 leal
4(%eax
), %eax
/ %eax
+= 4 (next word
)
3269 cmpl $
4, %edi
/ if
%edi
>= 4
3270 jae
.word_loop / goto .word_loop
3272 cmpl $
0, %edi
/ if
%edi
== 0
3273 je
.equal / goto .equal
3274 jmp
.byte_loop / goto .byte_loop (checks in bytes)
3276 leal
4(%edi
), %edi
/ %edi
+= 4 (post-decremented
)
3279 movb
(%ecx
), %dl
/ move
1 byte from
(%ecx
) to
%dl
3280 cmpb
%dl
, (%eax
) / compare
%dl with
1 byte from
(%eax
)
3281 jne
.not_equal / if not equal, goto .not_equal
3282 incl
%ecx
/ %ecx+
+ (next byte
)
3283 incl
%eax
/ %eax+
+ (next byte
)
3285 jnz
.byte_loop / if not zero, goto .byte_loop
3287 xorl
%eax
, %eax
/ %eax
= 0
3288 popl
%edi
/ restore register variable
3289 leave
/ restore old stack frame
3293 movl $
1, %eax
/ return
1
3294 popl
%edi
/ restore register variable
3295 leave
/ restore old stack frame
3304 .string "bcmp: arguments below kernelbase"
3309 #if defined(__amd64)
3317 #elif defined(__i386)
3331 ENTRY_NP
(atomic_btr32
)
3339 SET_SIZE
(atomic_btr32
)
3344 #if defined(__amd64)
3346 ENTRY_NP
(switch_sp_and_call
)
3348 movq
%rsp
, %rbp
/* set up stack frame */
3349 movq
%rdi
, %rsp
/* switch stack pointer */
3350 movq
%rdx
, %rdi
/* pass func arg 1 */
3351 movq
%rsi
, %r11 /* save function to call */
3352 movq
%rcx
, %rsi
/* pass func arg 2 */
3353 call
*%r11 /* call function */
3354 leave
/* restore stack */
3356 SET_SIZE
(switch_sp_and_call
)
3358 #elif defined(__i386)
3360 ENTRY_NP
(switch_sp_and_call
)
3362 mov
%esp
, %ebp
/* set up stack frame */
3363 movl
8(%ebp
), %esp
/* switch stack pointer */
3364 pushl
20(%ebp
) /* push func arg 2 */
3365 pushl
16(%ebp
) /* push func arg 1 */
3366 call
*12(%ebp
) /* call function */
3367 addl $
8, %esp
/* pop arguments */
3368 leave
/* restore stack */
3370 SET_SIZE
(switch_sp_and_call
)
3375 #if defined(__amd64)
3377 ENTRY_NP
(kmdb_enter
)
3382 * Save flags, do a 'cli' then return the saved flags
3389 * Restore the saved flags
3396 SET_SIZE
(kmdb_enter
)
3398 #elif defined(__i386)
3400 ENTRY_NP
(kmdb_enter
)
3405 * Save flags, do a 'cli' then return the saved flags
3412 * Restore the saved flags
3420 SET_SIZE
(kmdb_enter
)
3425 ENTRY_NP
(return_instr
)
3426 rep; ret
/* use 2 byte instruction when branch target */
3427 /* AMD Software Optimization Guide - Section 6.2 */
3428 SET_SIZE
(return_instr
)
3432 #if defined(__amd64)
3439 KPREEMPT_DISABLE
(%rdi
)
3441 * Synthesize the PS_IE bit from the event mask bit
3444 andq $_BITNOT
(PS_IE
), %rax
3445 XEN_TEST_UPCALL_MASK
(%r11)
3449 KPREEMPT_ENABLE_NOKP
(%rdi
)
3454 #elif defined(__i386)
3461 KPREEMPT_DISABLE
(%ecx
)
3463 * Synthesize the PS_IE bit from the event mask bit
3466 andl $_BITNOT
(PS_IE
), %eax
3467 XEN_TEST_UPCALL_MASK
(%edx
)
3471 KPREEMPT_ENABLE_NOKP
(%ecx
)
3480 #if defined(__amd64)
3482 ENTRY
(ftrace_interrupt_disable
)
3487 SET_SIZE
(ftrace_interrupt_disable
)
3489 #elif defined(__i386)
3491 ENTRY
(ftrace_interrupt_disable
)
3496 SET_SIZE
(ftrace_interrupt_disable
)
3501 #if defined(__amd64)
3503 ENTRY
(ftrace_interrupt_enable
)
3507 SET_SIZE
(ftrace_interrupt_enable
)
3509 #elif defined(__i386)
3511 ENTRY
(ftrace_interrupt_enable
)
3516 SET_SIZE
(ftrace_interrupt_enable
)
3521 #if defined (__amd64)
3525 SET_SIZE
(clflush_insn
)
3526 #elif defined (__i386)
3531 SET_SIZE
(clflush_insn
)
3536 #if defined (__amd64)
3540 SET_SIZE
(mfence_insn
)
3541 #elif defined (__i386)
3545 SET_SIZE
(mfence_insn
)
3550 * VMware implements an I/O port that programs can query to detect if software
3551 * is running in a VMware hypervisor. This hypervisor port behaves differently
3552 * depending on magic values in certain registers and modifies some registers
3555 * References: http://kb.vmware.com/kb/1009458
3559 #if defined(__amd64)
3563 movl $VMWARE_HVMAGIC
, %eax
3564 movl $
0xffffffff, %ebx
3566 movl $VMWARE_HVPORT
, %edx
3574 SET_SIZE
(vmware_port
)
3576 #elif defined(__i386)
3581 movl $VMWARE_HVMAGIC
, %eax
3582 movl $
0xffffffff, %ebx
3584 movl $VMWARE_HVPORT
, %edx
3594 SET_SIZE
(vmware_port
)