?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
4681 bcmp(NULL, NULL, 0) panics in DEBUG kernel
[unleashed.git] / usr / src / uts / intel / ia32 / ml / i86_subr.s
blob5b78115943cad0569b994a1f9e81bf57bf1993f3
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 */
26
27 /*
28 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
29 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
30 * All Rights Reserved
31 */
32
33 /*
34 * Copyright (c) 2009, Intel Corporation.
35 * All rights reserved.
36 */
37
38 /*
39 * General assembly language routines.
40 * It is the intent of this file to contain routines that are
41 * independent of the specific kernel architecture, and those that are
42 * common across kernel architectures.
43 * As architectures diverge, and implementations of specific
44 * architecture-dependent routines change, the routines should be moved
45 * from this file into the respective ../`arch -k`/subr.s file.
46 */
47
48 #include <sys/asm_linkage.h>
49 #include <sys/asm_misc.h>
50 #include <sys/panic.h>
51 #include <sys/ontrap.h>
52 #include <sys/regset.h>
53 #include <sys/privregs.h>
54 #include <sys/reboot.h>
55 #include <sys/psw.h>
56 #include <sys/x86_archext.h>
57
58 #if defined(__lint)
59 #include <sys/types.h>
60 #include <sys/systm.h>
61 #include <sys/thread.h>
62 #include <sys/archsystm.h>
63 #include <sys/byteorder.h>
64 #include <sys/dtrace.h>
65 #include <sys/ftrace.h>
66 #else /* __lint */
67 #include "assym.h"
68 #endif /* __lint */
69 #include <sys/dditypes.h>
70
71 /*
72 * on_fault()
73 * Catch lofault faults. Like setjmp except it returns one
74 * if code following causes uncorrectable fault. Turned off
75 * by calling no_fault().
76 */
77
78 #if defined(__lint)
79
80 /* ARGSUSED */
81 int
82 on_fault(label_t *ljb)
83 { return (0); }
84
85 void
86 no_fault(void)
87 {}
88
89 #else /* __lint */
90
91 #if defined(__amd64)
92
93 ENTRY(on_fault)
94 movq %gs:CPU_THREAD, %rsi
95 leaq catch_fault(%rip), %rdx
96 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
97 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
98 jmp setjmp /* let setjmp do the rest */
99
100 catch_fault:
101 movq %gs:CPU_THREAD, %rsi
102 movq T_ONFAULT(%rsi), %rdi /* address of save area */
103 xorl %eax, %eax
104 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
105 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
106 jmp longjmp /* let longjmp do the rest */
107 SET_SIZE(on_fault)
108
109 ENTRY(no_fault)
110 movq %gs:CPU_THREAD, %rsi
111 xorl %eax, %eax
112 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
113 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
114 ret
115 SET_SIZE(no_fault)
116
117 #elif defined(__i386)
118
119 ENTRY(on_fault)
120 movl %gs:CPU_THREAD, %edx
121 movl 4(%esp), %eax /* jumpbuf address */
122 leal catch_fault, %ecx
123 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
124 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
125 jmp setjmp /* let setjmp do the rest */
126
127 catch_fault:
128 movl %gs:CPU_THREAD, %edx
129 xorl %eax, %eax
130 movl T_ONFAULT(%edx), %ecx /* address of save area */
131 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
132 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
133 pushl %ecx
134 call longjmp /* let longjmp do the rest */
135 SET_SIZE(on_fault)
136
137 ENTRY(no_fault)
138 movl %gs:CPU_THREAD, %edx
139 xorl %eax, %eax
140 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
141 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
142 ret
143 SET_SIZE(no_fault)
144
145 #endif /* __i386 */
146 #endif /* __lint */
147
148 /*
149 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
150 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
151 */
152
153 #if defined(lint)
154
155 void
156 on_trap_trampoline(void)
157 {}
158
159 #else /* __lint */
160
161 #if defined(__amd64)
162
163 ENTRY(on_trap_trampoline)
164 movq %gs:CPU_THREAD, %rsi
165 movq T_ONTRAP(%rsi), %rdi
166 addq $OT_JMPBUF, %rdi
167 jmp longjmp
168 SET_SIZE(on_trap_trampoline)
169
170 #elif defined(__i386)
171
172 ENTRY(on_trap_trampoline)
173 movl %gs:CPU_THREAD, %eax
174 movl T_ONTRAP(%eax), %eax
175 addl $OT_JMPBUF, %eax
176 pushl %eax
177 call longjmp
178 SET_SIZE(on_trap_trampoline)
179
180 #endif /* __i386 */
181 #endif /* __lint */
182
183 /*
184 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
185 * more information about the on_trap() mechanism. If the on_trap_data is the
186 * same as the topmost stack element, we just modify that element.
187 */
188 #if defined(lint)
189
190 /*ARGSUSED*/
191 int
192 on_trap(on_trap_data_t *otp, uint_t prot)
193 { return (0); }
194
195 #else /* __lint */
196
197 #if defined(__amd64)
198
199 ENTRY(on_trap)
200 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
201 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
202 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
203 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
204 xorl %ecx, %ecx
205 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
206 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
207 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
208 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
209 cmpq %rdi, %rcx /* if (otp == %rcx) */
210 je 0f /* don't modify t_ontrap */
211
212 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
213 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
214
215 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
216 jmp setjmp
217 SET_SIZE(on_trap)
218
219 #elif defined(__i386)
220
221 ENTRY(on_trap)
222 movl 4(%esp), %eax /* %eax = otp */
223 movl 8(%esp), %edx /* %edx = prot */
224
225 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
226 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
227 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
228 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
229 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
230 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
231 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
232 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
233 cmpl %eax, %ecx /* if (otp == %ecx) */
234 je 0f /* don't modify t_ontrap */
235
236 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
237 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
238
239 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
240 movl %eax, 4(%esp) /* put %eax back on the stack */
241 jmp setjmp /* let setjmp do the rest */
242 SET_SIZE(on_trap)
243
244 #endif /* __i386 */
245 #endif /* __lint */
246
247 /*
248 * Setjmp and longjmp implement non-local gotos using state vectors
249 * type label_t.
250 */
251
252 #if defined(__lint)
253
254 /* ARGSUSED */
255 int
256 setjmp(label_t *lp)
257 { return (0); }
258
259 /* ARGSUSED */
260 void
261 longjmp(label_t *lp)
262 {}
263
264 #else /* __lint */
265
266 #if LABEL_PC != 0
267 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
268 #endif /* LABEL_PC != 0 */
269
270 #if defined(__amd64)
271
272 ENTRY(setjmp)
273 movq %rsp, LABEL_SP(%rdi)
274 movq %rbp, LABEL_RBP(%rdi)
275 movq %rbx, LABEL_RBX(%rdi)
276 movq %r12, LABEL_R12(%rdi)
277 movq %r13, LABEL_R13(%rdi)
278 movq %r14, LABEL_R14(%rdi)
279 movq %r15, LABEL_R15(%rdi)
280 movq (%rsp), %rdx /* return address */
281 movq %rdx, (%rdi) /* LABEL_PC is 0 */
282 xorl %eax, %eax /* return 0 */
283 ret
284 SET_SIZE(setjmp)
285
286 ENTRY(longjmp)
287 movq LABEL_SP(%rdi), %rsp
288 movq LABEL_RBP(%rdi), %rbp
289 movq LABEL_RBX(%rdi), %rbx
290 movq LABEL_R12(%rdi), %r12
291 movq LABEL_R13(%rdi), %r13
292 movq LABEL_R14(%rdi), %r14
293 movq LABEL_R15(%rdi), %r15
294 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
295 movq %rdx, (%rsp)
296 xorl %eax, %eax
297 incl %eax /* return 1 */
298 ret
299 SET_SIZE(longjmp)
300
301 #elif defined(__i386)
302
303 ENTRY(setjmp)
304 movl 4(%esp), %edx /* address of save area */
305 movl %ebp, LABEL_EBP(%edx)
306 movl %ebx, LABEL_EBX(%edx)
307 movl %esi, LABEL_ESI(%edx)
308 movl %edi, LABEL_EDI(%edx)
309 movl %esp, 4(%edx)
310 movl (%esp), %ecx /* %eip (return address) */
311 movl %ecx, (%edx) /* LABEL_PC is 0 */
312 subl %eax, %eax /* return 0 */
313 ret
314 SET_SIZE(setjmp)
315
316 ENTRY(longjmp)
317 movl 4(%esp), %edx /* address of save area */
318 movl LABEL_EBP(%edx), %ebp
319 movl LABEL_EBX(%edx), %ebx
320 movl LABEL_ESI(%edx), %esi
321 movl LABEL_EDI(%edx), %edi
322 movl 4(%edx), %esp
323 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
324 movl $1, %eax
325 addl $4, %esp /* pop ret adr */
326 jmp *%ecx /* indirect */
327 SET_SIZE(longjmp)
328
329 #endif /* __i386 */
330 #endif /* __lint */
331
332 /*
333 * if a() calls b() calls caller(),
334 * caller() returns return address in a().
335 * (Note: We assume a() and b() are C routines which do the normal entry/exit
336 * sequence.)
337 */
338
339 #if defined(__lint)
340
341 caddr_t
342 caller(void)
343 { return (0); }
344
345 #else /* __lint */
346
347 #if defined(__amd64)
348
349 ENTRY(caller)
350 movq 8(%rbp), %rax /* b()'s return pc, in a() */
351 ret
352 SET_SIZE(caller)
353
354 #elif defined(__i386)
355
356 ENTRY(caller)
357 movl 4(%ebp), %eax /* b()'s return pc, in a() */
358 ret
359 SET_SIZE(caller)
360
361 #endif /* __i386 */
362 #endif /* __lint */
363
364 /*
365 * if a() calls callee(), callee() returns the
366 * return address in a();
367 */
368
369 #if defined(__lint)
370
371 caddr_t
372 callee(void)
373 { return (0); }
374
375 #else /* __lint */
376
377 #if defined(__amd64)
378
379 ENTRY(callee)
380 movq (%rsp), %rax /* callee()'s return pc, in a() */
381 ret
382 SET_SIZE(callee)
383
384 #elif defined(__i386)
385
386 ENTRY(callee)
387 movl (%esp), %eax /* callee()'s return pc, in a() */
388 ret
389 SET_SIZE(callee)
390
391 #endif /* __i386 */
392 #endif /* __lint */
393
394 /*
395 * return the current frame pointer
396 */
397
398 #if defined(__lint)
399
400 greg_t
401 getfp(void)
402 { return (0); }
403
404 #else /* __lint */
405
406 #if defined(__amd64)
407
408 ENTRY(getfp)
409 movq %rbp, %rax
410 ret
411 SET_SIZE(getfp)
412
413 #elif defined(__i386)
414
415 ENTRY(getfp)
416 movl %ebp, %eax
417 ret
418 SET_SIZE(getfp)
419
420 #endif /* __i386 */
421 #endif /* __lint */
422
423 /*
424 * Invalidate a single page table entry in the TLB
425 */
426
427 #if defined(__lint)
428
429 /* ARGSUSED */
430 void
431 mmu_tlbflush_entry(caddr_t m)
432 {}
433
434 #else /* __lint */
435
436 #if defined(__amd64)
437
438 ENTRY(mmu_tlbflush_entry)
439 invlpg (%rdi)
440 ret
441 SET_SIZE(mmu_tlbflush_entry)
442
443 #elif defined(__i386)
444
445 ENTRY(mmu_tlbflush_entry)
446 movl 4(%esp), %eax
447 invlpg (%eax)
448 ret
449 SET_SIZE(mmu_tlbflush_entry)
450
451 #endif /* __i386 */
452 #endif /* __lint */
453
454
455 /*
456 * Get/Set the value of various control registers
457 */
458
459 #if defined(__lint)
460
461 ulong_t
462 getcr0(void)
463 { return (0); }
464
465 /* ARGSUSED */
466 void
467 setcr0(ulong_t value)
468 {}
469
470 ulong_t
471 getcr2(void)
472 { return (0); }
473
474 ulong_t
475 getcr3(void)
476 { return (0); }
477
478 #if !defined(__xpv)
479 /* ARGSUSED */
480 void
481 setcr3(ulong_t val)
482 {}
483
484 void
485 reload_cr3(void)
486 {}
487 #endif
488
489 ulong_t
490 getcr4(void)
491 { return (0); }
492
493 /* ARGSUSED */
494 void
495 setcr4(ulong_t val)
496 {}
497
498 #if defined(__amd64)
499
500 ulong_t
501 getcr8(void)
502 { return (0); }
503
504 /* ARGSUSED */
505 void
506 setcr8(ulong_t val)
507 {}
508
509 #endif /* __amd64 */
510
511 #else /* __lint */
512
513 #if defined(__amd64)
514
515 ENTRY(getcr0)
516 movq %cr0, %rax
517 ret
518 SET_SIZE(getcr0)
519
520 ENTRY(setcr0)
521 movq %rdi, %cr0
522 ret
523 SET_SIZE(setcr0)
524
525 ENTRY(getcr2)
526 #if defined(__xpv)
527 movq %gs:CPU_VCPU_INFO, %rax
528 movq VCPU_INFO_ARCH_CR2(%rax), %rax
529 #else
530 movq %cr2, %rax
531 #endif
532 ret
533 SET_SIZE(getcr2)
534
535 ENTRY(getcr3)
536 movq %cr3, %rax
537 ret
538 SET_SIZE(getcr3)
539
540 #if !defined(__xpv)
541
542 ENTRY(setcr3)
543 movq %rdi, %cr3
544 ret
545 SET_SIZE(setcr3)
546
547 ENTRY(reload_cr3)
548 movq %cr3, %rdi
549 movq %rdi, %cr3
550 ret
551 SET_SIZE(reload_cr3)
552
553 #endif /* __xpv */
554
555 ENTRY(getcr4)
556 movq %cr4, %rax
557 ret
558 SET_SIZE(getcr4)
559
560 ENTRY(setcr4)
561 movq %rdi, %cr4
562 ret
563 SET_SIZE(setcr4)
564
565 ENTRY(getcr8)
566 movq %cr8, %rax
567 ret
568 SET_SIZE(getcr8)
569
570 ENTRY(setcr8)
571 movq %rdi, %cr8
572 ret
573 SET_SIZE(setcr8)
574
575 #elif defined(__i386)
576
577 ENTRY(getcr0)
578 movl %cr0, %eax
579 ret
580 SET_SIZE(getcr0)
581
582 ENTRY(setcr0)
583 movl 4(%esp), %eax
584 movl %eax, %cr0
585 ret
586 SET_SIZE(setcr0)
587
588 /*
589 * "lock mov %cr0" is used on processors which indicate it is
590 * supported via CPUID. Normally the 32 bit TPR is accessed via
591 * the local APIC.
592 */
593 ENTRY(getcr8)
594 lock
595 movl %cr0, %eax
596 ret
597 SET_SIZE(getcr8)
598
599 ENTRY(setcr8)
600 movl 4(%esp), %eax
601 lock
602 movl %eax, %cr0
603 ret
604 SET_SIZE(setcr8)
605
606 ENTRY(getcr2)
607 #if defined(__xpv)
608 movl %gs:CPU_VCPU_INFO, %eax
609 movl VCPU_INFO_ARCH_CR2(%eax), %eax
610 #else
611 movl %cr2, %eax
612 #endif
613 ret
614 SET_SIZE(getcr2)
615
616 ENTRY(getcr3)
617 movl %cr3, %eax
618 ret
619 SET_SIZE(getcr3)
620
621 #if !defined(__xpv)
622
623 ENTRY(setcr3)
624 movl 4(%esp), %eax
625 movl %eax, %cr3
626 ret
627 SET_SIZE(setcr3)
628
629 ENTRY(reload_cr3)
630 movl %cr3, %eax
631 movl %eax, %cr3
632 ret
633 SET_SIZE(reload_cr3)
634
635 #endif /* __xpv */
636
637 ENTRY(getcr4)
638 movl %cr4, %eax
639 ret
640 SET_SIZE(getcr4)
641
642 ENTRY(setcr4)
643 movl 4(%esp), %eax
644 movl %eax, %cr4
645 ret
646 SET_SIZE(setcr4)
647
648 #endif /* __i386 */
649 #endif /* __lint */
650
651 #if defined(__lint)
652
653 /*ARGSUSED*/
654 uint32_t
655 __cpuid_insn(struct cpuid_regs *regs)
656 { return (0); }
657
658 #else /* __lint */
659
660 #if defined(__amd64)
661
662 ENTRY(__cpuid_insn)
663 movq %rbx, %r8
664 movq %rcx, %r9
665 movq %rdx, %r11
666 movl (%rdi), %eax /* %eax = regs->cp_eax */
667 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
668 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
669 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
670 cpuid
671 movl %eax, (%rdi) /* regs->cp_eax = %eax */
672 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
673 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
674 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
675 movq %r8, %rbx
676 movq %r9, %rcx
677 movq %r11, %rdx
678 ret
679 SET_SIZE(__cpuid_insn)
680
681 #elif defined(__i386)
682
683 ENTRY(__cpuid_insn)
684 pushl %ebp
685 movl 0x8(%esp), %ebp /* %ebp = regs */
686 pushl %ebx
687 pushl %ecx
688 pushl %edx
689 movl (%ebp), %eax /* %eax = regs->cp_eax */
690 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
691 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
692 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
693 cpuid
694 movl %eax, (%ebp) /* regs->cp_eax = %eax */
695 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
696 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
697 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
698 popl %edx
699 popl %ecx
700 popl %ebx
701 popl %ebp
702 ret
703 SET_SIZE(__cpuid_insn)
704
705 #endif /* __i386 */
706 #endif /* __lint */
707
708 #if defined(__lint)
709
710 /*ARGSUSED*/
711 void
712 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
713 {}
714
715 #else /* __lint */
716
717 #if defined(__amd64)
718
719 ENTRY_NP(i86_monitor)
720 pushq %rbp
721 movq %rsp, %rbp
722 movq %rdi, %rax /* addr */
723 movq %rsi, %rcx /* extensions */
724 /* rdx contains input arg3: hints */
725 clflush (%rax)
726 .byte 0x0f, 0x01, 0xc8 /* monitor */
727 leave
728 ret
729 SET_SIZE(i86_monitor)
730
731 #elif defined(__i386)
732
733 ENTRY_NP(i86_monitor)
734 pushl %ebp
735 movl %esp, %ebp
736 movl 0x8(%ebp),%eax /* addr */
737 movl 0xc(%ebp),%ecx /* extensions */
738 movl 0x10(%ebp),%edx /* hints */
739 clflush (%eax)
740 .byte 0x0f, 0x01, 0xc8 /* monitor */
741 leave
742 ret
743 SET_SIZE(i86_monitor)
744
745 #endif /* __i386 */
746 #endif /* __lint */
747
748 #if defined(__lint)
749
750 /*ARGSUSED*/
751 void
752 i86_mwait(uint32_t data, uint32_t extensions)
753 {}
754
755 #else /* __lint */
756
757 #if defined(__amd64)
758
759 ENTRY_NP(i86_mwait)
760 pushq %rbp
761 movq %rsp, %rbp
762 movq %rdi, %rax /* data */
763 movq %rsi, %rcx /* extensions */
764 .byte 0x0f, 0x01, 0xc9 /* mwait */
765 leave
766 ret
767 SET_SIZE(i86_mwait)
768
769 #elif defined(__i386)
770
771 ENTRY_NP(i86_mwait)
772 pushl %ebp
773 movl %esp, %ebp
774 movl 0x8(%ebp),%eax /* data */
775 movl 0xc(%ebp),%ecx /* extensions */
776 .byte 0x0f, 0x01, 0xc9 /* mwait */
777 leave
778 ret
779 SET_SIZE(i86_mwait)
780
781 #endif /* __i386 */
782 #endif /* __lint */
783
784 #if defined(__xpv)
785 /*
786 * Defined in C
787 */
788 #else
789
790 #if defined(__lint)
791
792 hrtime_t
793 tsc_read(void)
794 {
795 return (0);
796 }
797
798 #else /* __lint */
799
800 #if defined(__amd64)
801
802 ENTRY_NP(tsc_read)
803 movq %rbx, %r11
804 movl $0, %eax
805 cpuid
806 rdtsc
807 movq %r11, %rbx
808 shlq $32, %rdx
809 orq %rdx, %rax
810 ret
811 .globl _tsc_mfence_start
812 _tsc_mfence_start:
813 mfence
814 rdtsc
815 shlq $32, %rdx
816 orq %rdx, %rax
817 ret
818 .globl _tsc_mfence_end
819 _tsc_mfence_end:
820 .globl _tscp_start
821 _tscp_start:
822 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
823 shlq $32, %rdx
824 orq %rdx, %rax
825 ret
826 .globl _tscp_end
827 _tscp_end:
828 .globl _no_rdtsc_start
829 _no_rdtsc_start:
830 xorl %edx, %edx
831 xorl %eax, %eax
832 ret
833 .globl _no_rdtsc_end
834 _no_rdtsc_end:
835 .globl _tsc_lfence_start
836 _tsc_lfence_start:
837 lfence
838 rdtsc
839 shlq $32, %rdx
840 orq %rdx, %rax
841 ret
842 .globl _tsc_lfence_end
843 _tsc_lfence_end:
844 SET_SIZE(tsc_read)
845
846 #else /* __i386 */
847
848 ENTRY_NP(tsc_read)
849 pushl %ebx
850 movl $0, %eax
851 cpuid
852 rdtsc
853 popl %ebx
854 ret
855 .globl _tsc_mfence_start
856 _tsc_mfence_start:
857 mfence
858 rdtsc
859 ret
860 .globl _tsc_mfence_end
861 _tsc_mfence_end:
862 .globl _tscp_start
863 _tscp_start:
864 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
865 ret
866 .globl _tscp_end
867 _tscp_end:
868 .globl _no_rdtsc_start
869 _no_rdtsc_start:
870 xorl %edx, %edx
871 xorl %eax, %eax
872 ret
873 .globl _no_rdtsc_end
874 _no_rdtsc_end:
875 .globl _tsc_lfence_start
876 _tsc_lfence_start:
877 lfence
878 rdtsc
879 ret
880 .globl _tsc_lfence_end
881 _tsc_lfence_end:
882 SET_SIZE(tsc_read)
883
884 #endif /* __i386 */
885
886 #endif /* __lint */
887
888
889 #endif /* __xpv */
890
891 #ifdef __lint
892 /*
893 * Do not use this function for obtaining clock tick. This
894 * is called by callers who do not need to have a guarenteed
895 * correct tick value. The proper routine to use is tsc_read().
896 */
897 u_longlong_t
898 randtick(void)
899 {
900 return (0);
901 }
902 #else
903 #if defined(__amd64)
904 ENTRY_NP(randtick)
905 rdtsc
906 shlq $32, %rdx
907 orq %rdx, %rax
908 ret
909 SET_SIZE(randtick)
910 #else
911 ENTRY_NP(randtick)
912 rdtsc
913 ret
914 SET_SIZE(randtick)
915 #endif /* __i386 */
916 #endif /* __lint */
917 /*
918 * Insert entryp after predp in a doubly linked list.
919 */
920
921 #if defined(__lint)
922
923 /*ARGSUSED*/
924 void
925 _insque(caddr_t entryp, caddr_t predp)
926 {}
927
928 #else /* __lint */
929
930 #if defined(__amd64)
931
932 ENTRY(_insque)
933 movq (%rsi), %rax /* predp->forw */
934 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
935 movq %rax, (%rdi) /* entryp->forw = predp->forw */
936 movq %rdi, (%rsi) /* predp->forw = entryp */
937 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
938 ret
939 SET_SIZE(_insque)
940
941 #elif defined(__i386)
942
943 ENTRY(_insque)
944 movl 8(%esp), %edx
945 movl 4(%esp), %ecx
946 movl (%edx), %eax /* predp->forw */
947 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
948 movl %eax, (%ecx) /* entryp->forw = predp->forw */
949 movl %ecx, (%edx) /* predp->forw = entryp */
950 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
951 ret
952 SET_SIZE(_insque)
953
954 #endif /* __i386 */
955 #endif /* __lint */
956
957 /*
958 * Remove entryp from a doubly linked list
959 */
960
961 #if defined(__lint)
962
963 /*ARGSUSED*/
964 void
965 _remque(caddr_t entryp)
966 {}
967
968 #else /* __lint */
969
970 #if defined(__amd64)
971
972 ENTRY(_remque)
973 movq (%rdi), %rax /* entry->forw */
974 movq CPTRSIZE(%rdi), %rdx /* entry->back */
975 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
976 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
977 ret
978 SET_SIZE(_remque)
979
980 #elif defined(__i386)
981
982 ENTRY(_remque)
983 movl 4(%esp), %ecx
984 movl (%ecx), %eax /* entry->forw */
985 movl CPTRSIZE(%ecx), %edx /* entry->back */
986 movl %eax, (%edx) /* entry->back->forw = entry->forw */
987 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
988 ret
989 SET_SIZE(_remque)
990
991 #endif /* __i386 */
992 #endif /* __lint */
993
994 /*
995 * Returns the number of
996 * non-NULL bytes in string argument.
997 */
998
999 #if defined(__lint)
1000
1001 /* ARGSUSED */
1002 size_t
1003 strlen(const char *str)
1004 { return (0); }
1005
1006 #else /* __lint */
1007
1008 #if defined(__amd64)
1009
1010 /*
1011 * This is close to a simple transliteration of a C version of this
1012 * routine. We should either just -make- this be a C version, or
1013 * justify having it in assembler by making it significantly faster.
1014 *
1015 * size_t
1016 * strlen(const char *s)
1017 * {
1018 * const char *s0;
1019 * #if defined(DEBUG)
1020 * if ((uintptr_t)s < KERNELBASE)
1021 * panic(.str_panic_msg);
1022 * #endif
1023 * for (s0 = s; *s; s++)
1024 * ;
1025 * return (s - s0);
1026 * }
1027 */
1028
1029 ENTRY(strlen)
1030 #ifdef DEBUG
1031 movq postbootkernelbase(%rip), %rax
1032 cmpq %rax, %rdi
1033 jae str_valid
1034 pushq %rbp
1035 movq %rsp, %rbp
1036 leaq .str_panic_msg(%rip), %rdi
1037 xorl %eax, %eax
1038 call panic
1039 #endif /* DEBUG */
1040 str_valid:
1041 cmpb $0, (%rdi)
1042 movq %rdi, %rax
1043 je .null_found
1044 .align 4
1045 .strlen_loop:
1046 incq %rdi
1047 cmpb $0, (%rdi)
1048 jne .strlen_loop
1049 .null_found:
1050 subq %rax, %rdi
1051 movq %rdi, %rax
1052 ret
1053 SET_SIZE(strlen)
1054
1055 #elif defined(__i386)
1056
1057 ENTRY(strlen)
1058 #ifdef DEBUG
1059 movl postbootkernelbase, %eax
1060 cmpl %eax, 4(%esp)
1061 jae str_valid
1062 pushl %ebp
1063 movl %esp, %ebp
1064 pushl $.str_panic_msg
1065 call panic
1066 #endif /* DEBUG */
1067
1068 str_valid:
1069 movl 4(%esp), %eax /* %eax = string address */
1070 testl $3, %eax /* if %eax not word aligned */
1071 jnz .not_word_aligned /* goto .not_word_aligned */
1072 .align 4
1073 .word_aligned:
1074 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1075 movl $0x7f7f7f7f, %ecx
1076 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1077 addl $4, %eax /* next word */
1078 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1079 orl %edx, %ecx /* %ecx |= %edx */
1080 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1081 cmpl $0x80808080, %ecx /* if no null byte in this word */
1082 je .word_aligned /* goto .word_aligned */
1083 subl $4, %eax /* post-incremented */
1084 .not_word_aligned:
1085 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1086 je .null_found /* goto .null_found */
1087 incl %eax /* next byte */
1088 testl $3, %eax /* if %eax not word aligned */
1089 jnz .not_word_aligned /* goto .not_word_aligned */
1090 jmp .word_aligned /* goto .word_aligned */
1091 .align 4
1092 .null_found:
1093 subl 4(%esp), %eax /* %eax -= string address */
1094 ret
1095 SET_SIZE(strlen)
1096
1097 #endif /* __i386 */
1098
1099 #ifdef DEBUG
1100 .text
1101 .str_panic_msg:
1102 .string "strlen: argument below kernelbase"
1103 #endif /* DEBUG */
1104
1105 #endif /* __lint */
1106
1107 /*
1108 * Berkeley 4.3 introduced symbolically named interrupt levels
1109 * as a way deal with priority in a machine independent fashion.
1110 * Numbered priorities are machine specific, and should be
1111 * discouraged where possible.
1112 *
1113 * Note, for the machine specific priorities there are
1114 * examples listed for devices that use a particular priority.
1115 * It should not be construed that all devices of that
1116 * type should be at that priority. It is currently were
1117 * the current devices fit into the priority scheme based
1118 * upon time criticalness.
1119 *
1120 * The underlying assumption of these assignments is that
1121 * IPL 10 is the highest level from which a device
1122 * routine can call wakeup. Devices that interrupt from higher
1123 * levels are restricted in what they can do. If they need
1124 * kernels services they should schedule a routine at a lower
1125 * level (via software interrupt) to do the required
1126 * processing.
1127 *
1128 * Examples of this higher usage:
1129 * Level Usage
1130 * 14 Profiling clock (and PROM uart polling clock)
1131 * 12 Serial ports
1132 *
1133 * The serial ports request lower level processing on level 6.
1134 *
1135 * Also, almost all splN routines (where N is a number or a
1136 * mnemonic) will do a RAISE(), on the assumption that they are
1137 * never used to lower our priority.
1138 * The exceptions are:
1139 * spl8() Because you can't be above 15 to begin with!
1140 * splzs() Because this is used at boot time to lower our
1141 * priority, to allow the PROM to poll the uart.
1142 * spl0() Used to lower priority to 0.
1143 */
1144
1145 #if defined(__lint)
1146
1147 int spl0(void) { return (0); }
1148 int spl6(void) { return (0); }
1149 int spl7(void) { return (0); }
1150 int spl8(void) { return (0); }
1151 int splhigh(void) { return (0); }
1152 int splhi(void) { return (0); }
1153 int splzs(void) { return (0); }
1154
1155 /* ARGSUSED */
1156 void
1157 splx(int level)
1158 {}
1159
1160 #else /* __lint */
1161
1162 #if defined(__amd64)
1163
1164 #define SETPRI(level) \
1165 movl $/**/level, %edi; /* new priority */ \
1166 jmp do_splx /* redirect to do_splx */
1167
1168 #define RAISE(level) \
1169 movl $/**/level, %edi; /* new priority */ \
1170 jmp splr /* redirect to splr */
1171
1172 #elif defined(__i386)
1173
1174 #define SETPRI(level) \
1175 pushl $/**/level; /* new priority */ \
1176 call do_splx; /* invoke common splx code */ \
1177 addl $4, %esp; /* unstack arg */ \
1178 ret
1179
1180 #define RAISE(level) \
1181 pushl $/**/level; /* new priority */ \
1182 call splr; /* invoke common splr code */ \
1183 addl $4, %esp; /* unstack args */ \
1184 ret
1185
1186 #endif /* __i386 */
1187
1188 /* locks out all interrupts, including memory errors */
1189 ENTRY(spl8)
1190 SETPRI(15)
1191 SET_SIZE(spl8)
1192
1193 /* just below the level that profiling runs */
1194 ENTRY(spl7)
1195 RAISE(13)
1196 SET_SIZE(spl7)
1197
1198 /* sun specific - highest priority onboard serial i/o asy ports */
1199 ENTRY(splzs)
1200 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1201 SET_SIZE(splzs)
1202
1203 ENTRY(splhi)
1204 ALTENTRY(splhigh)
1205 ALTENTRY(spl6)
1206 ALTENTRY(i_ddi_splhigh)
1207
1208 RAISE(DISP_LEVEL)
1209
1210 SET_SIZE(i_ddi_splhigh)
1211 SET_SIZE(spl6)
1212 SET_SIZE(splhigh)
1213 SET_SIZE(splhi)
1214
1215 /* allow all interrupts */
1216 ENTRY(spl0)
1217 SETPRI(0)
1218 SET_SIZE(spl0)
1219
1220
1221 /* splx implementation */
1222 ENTRY(splx)
1223 jmp do_splx /* redirect to common splx code */
1224 SET_SIZE(splx)
1225
1226 #endif /* __lint */
1227
1228 #if defined(__i386)
1229
1230 /*
1231 * Read and write the %gs register
1232 */
1233
1234 #if defined(__lint)
1235
1236 /*ARGSUSED*/
1237 uint16_t
1238 getgs(void)
1239 { return (0); }
1240
1241 /*ARGSUSED*/
1242 void
1243 setgs(uint16_t sel)
1244 {}
1245
1246 #else /* __lint */
1247
1248 ENTRY(getgs)
1249 clr %eax
1250 movw %gs, %ax
1251 ret
1252 SET_SIZE(getgs)
1253
1254 ENTRY(setgs)
1255 movw 4(%esp), %gs
1256 ret
1257 SET_SIZE(setgs)
1258
1259 #endif /* __lint */
1260 #endif /* __i386 */
1261
1262 #if defined(__lint)
1263
1264 void
1265 pc_reset(void)
1266 {}
1267
1268 void
1269 efi_reset(void)
1270 {}
1271
1272 #else /* __lint */
1273
1274 ENTRY(wait_500ms)
1275 #if defined(__amd64)
1276 pushq %rbx
1277 #elif defined(__i386)
1278 push %ebx
1279 #endif
1280 movl $50000, %ebx
1281 1:
1282 call tenmicrosec
1283 decl %ebx
1284 jnz 1b
1285 #if defined(__amd64)
1286 popq %rbx
1287 #elif defined(__i386)
1288 pop %ebx
1289 #endif
1290 ret
1291 SET_SIZE(wait_500ms)
1292
1293 #define RESET_METHOD_KBC 1
1294 #define RESET_METHOD_PORT92 2
1295 #define RESET_METHOD_PCI 4
1296
1297 DGDEF3(pc_reset_methods, 4, 8)
1298 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1299
1300 ENTRY(pc_reset)
1301
1302 #if defined(__i386)
1303 testl $RESET_METHOD_KBC, pc_reset_methods
1304 #elif defined(__amd64)
1305 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1306 #endif
1307 jz 1f
1308
1309 /
1310 / Try the classic keyboard controller-triggered reset.
1311 /
1312 movw $0x64, %dx
1313 movb $0xfe, %al
1314 outb (%dx)
1315
1316 / Wait up to 500 milliseconds here for the keyboard controller
1317 / to pull the reset line. On some systems where the keyboard
1318 / controller is slow to pull the reset line, the next reset method
1319 / may be executed (which may be bad if those systems hang when the
1320 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1321 / and Ferrari 4000 (doesn't like the cf9 reset method))
1322
1323 call wait_500ms
1324
1325 1:
1326 #if defined(__i386)
1327 testl $RESET_METHOD_PORT92, pc_reset_methods
1328 #elif defined(__amd64)
1329 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1330 #endif
1331 jz 3f
1332
1333 /
1334 / Try port 0x92 fast reset
1335 /
1336 movw $0x92, %dx
1337 inb (%dx)
1338 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1339 je 1f
1340 testb $1, %al / If bit 0
1341 jz 2f / is clear, jump to perform the reset
1342 andb $0xfe, %al / otherwise,
1343 outb (%dx) / clear bit 0 first, then
1344 2:
1345 orb $1, %al / Set bit 0
1346 outb (%dx) / and reset the system
1347 1:
1348
1349 call wait_500ms
1350
1351 3:
1352 #if defined(__i386)
1353 testl $RESET_METHOD_PCI, pc_reset_methods
1354 #elif defined(__amd64)
1355 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1356 #endif
1357 jz 4f
1358
1359 / Try the PCI (soft) reset vector (should work on all modern systems,
1360 / but has been shown to cause problems on 450NX systems, and some newer
1361 / systems (e.g. ATI IXP400-equipped systems))
1362 / When resetting via this method, 2 writes are required. The first
1363 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1364 / power cycle).
1365 / The reset occurs on the second write, during bit 2's transition from
1366 / 0->1.
1367 movw $0xcf9, %dx
1368 movb $0x2, %al / Reset mode = hard, no power cycle
1369 outb (%dx)
1370 movb $0x6, %al
1371 outb (%dx)
1372
1373 call wait_500ms
1374
1375 4:
1376 /
1377 / port 0xcf9 failed also. Last-ditch effort is to
1378 / triple-fault the CPU.
1379 / Also, use triple fault for EFI firmware
1380 /
1381 ENTRY(efi_reset)
1382 #if defined(__amd64)
1383 pushq $0x0
1384 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1385 lidt (%rsp)
1386 #elif defined(__i386)
1387 pushl $0x0
1388 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1389 lidt (%esp)
1390 #endif
1391 int $0x0 / Trigger interrupt, generate triple-fault
1392
1393 cli
1394 hlt / Wait forever
1395 /*NOTREACHED*/
1396 SET_SIZE(efi_reset)
1397 SET_SIZE(pc_reset)
1398
1399 #endif /* __lint */
1400
1401 /*
1402 * C callable in and out routines
1403 */
1404
1405 #if defined(__lint)
1406
1407 /* ARGSUSED */
1408 void
1409 outl(int port_address, uint32_t val)
1410 {}
1411
1412 #else /* __lint */
1413
1414 #if defined(__amd64)
1415
1416 ENTRY(outl)
1417 movw %di, %dx
1418 movl %esi, %eax
1419 outl (%dx)
1420 ret
1421 SET_SIZE(outl)
1422
1423 #elif defined(__i386)
1424
1425 .set PORT, 4
1426 .set VAL, 8
1427
1428 ENTRY(outl)
1429 movw PORT(%esp), %dx
1430 movl VAL(%esp), %eax
1431 outl (%dx)
1432 ret
1433 SET_SIZE(outl)
1434
1435 #endif /* __i386 */
1436 #endif /* __lint */
1437
1438 #if defined(__lint)
1439
1440 /* ARGSUSED */
1441 void
1442 outw(int port_address, uint16_t val)
1443 {}
1444
1445 #else /* __lint */
1446
1447 #if defined(__amd64)
1448
1449 ENTRY(outw)
1450 movw %di, %dx
1451 movw %si, %ax
1452 D16 outl (%dx) /* XX64 why not outw? */
1453 ret
1454 SET_SIZE(outw)
1455
1456 #elif defined(__i386)
1457
1458 ENTRY(outw)
1459 movw PORT(%esp), %dx
1460 movw VAL(%esp), %ax
1461 D16 outl (%dx)
1462 ret
1463 SET_SIZE(outw)
1464
1465 #endif /* __i386 */
1466 #endif /* __lint */
1467
1468 #if defined(__lint)
1469
1470 /* ARGSUSED */
1471 void
1472 outb(int port_address, uint8_t val)
1473 {}
1474
1475 #else /* __lint */
1476
1477 #if defined(__amd64)
1478
1479 ENTRY(outb)
1480 movw %di, %dx
1481 movb %sil, %al
1482 outb (%dx)
1483 ret
1484 SET_SIZE(outb)
1485
1486 #elif defined(__i386)
1487
1488 ENTRY(outb)
1489 movw PORT(%esp), %dx
1490 movb VAL(%esp), %al
1491 outb (%dx)
1492 ret
1493 SET_SIZE(outb)
1494
1495 #endif /* __i386 */
1496 #endif /* __lint */
1497
1498 #if defined(__lint)
1499
1500 /* ARGSUSED */
1501 uint32_t
1502 inl(int port_address)
1503 { return (0); }
1504
1505 #else /* __lint */
1506
1507 #if defined(__amd64)
1508
1509 ENTRY(inl)
1510 xorl %eax, %eax
1511 movw %di, %dx
1512 inl (%dx)
1513 ret
1514 SET_SIZE(inl)
1515
1516 #elif defined(__i386)
1517
1518 ENTRY(inl)
1519 movw PORT(%esp), %dx
1520 inl (%dx)
1521 ret
1522 SET_SIZE(inl)
1523
1524 #endif /* __i386 */
1525 #endif /* __lint */
1526
1527 #if defined(__lint)
1528
1529 /* ARGSUSED */
1530 uint16_t
1531 inw(int port_address)
1532 { return (0); }
1533
1534 #else /* __lint */
1535
1536 #if defined(__amd64)
1537
1538 ENTRY(inw)
1539 xorl %eax, %eax
1540 movw %di, %dx
1541 D16 inl (%dx)
1542 ret
1543 SET_SIZE(inw)
1544
1545 #elif defined(__i386)
1546
1547 ENTRY(inw)
1548 subl %eax, %eax
1549 movw PORT(%esp), %dx
1550 D16 inl (%dx)
1551 ret
1552 SET_SIZE(inw)
1553
1554 #endif /* __i386 */
1555 #endif /* __lint */
1556
1557
1558 #if defined(__lint)
1559
1560 /* ARGSUSED */
1561 uint8_t
1562 inb(int port_address)
1563 { return (0); }
1564
1565 #else /* __lint */
1566
1567 #if defined(__amd64)
1568
1569 ENTRY(inb)
1570 xorl %eax, %eax
1571 movw %di, %dx
1572 inb (%dx)
1573 ret
1574 SET_SIZE(inb)
1575
1576 #elif defined(__i386)
1577
1578 ENTRY(inb)
1579 subl %eax, %eax
1580 movw PORT(%esp), %dx
1581 inb (%dx)
1582 ret
1583 SET_SIZE(inb)
1584
1585 #endif /* __i386 */
1586 #endif /* __lint */
1587
1588
1589 #if defined(__lint)
1590
1591 /* ARGSUSED */
1592 void
1593 repoutsw(int port, uint16_t *addr, int cnt)
1594 {}
1595
1596 #else /* __lint */
1597
1598 #if defined(__amd64)
1599
1600 ENTRY(repoutsw)
1601 movl %edx, %ecx
1602 movw %di, %dx
1603 rep
1604 D16 outsl
1605 ret
1606 SET_SIZE(repoutsw)
1607
1608 #elif defined(__i386)
1609
1610 /*
1611 * The arguments and saved registers are on the stack in the
1612 * following order:
1613 * | cnt | +16
1614 * | *addr | +12
1615 * | port | +8
1616 * | eip | +4
1617 * | esi | <-- %esp
1618 * If additional values are pushed onto the stack, make sure
1619 * to adjust the following constants accordingly.
1620 */
1621 .set PORT, 8
1622 .set ADDR, 12
1623 .set COUNT, 16
1624
1625 ENTRY(repoutsw)
1626 pushl %esi
1627 movl PORT(%esp), %edx
1628 movl ADDR(%esp), %esi
1629 movl COUNT(%esp), %ecx
1630 rep
1631 D16 outsl
1632 popl %esi
1633 ret
1634 SET_SIZE(repoutsw)
1635
1636 #endif /* __i386 */
1637 #endif /* __lint */
1638
1639
1640 #if defined(__lint)
1641
1642 /* ARGSUSED */
1643 void
1644 repinsw(int port_addr, uint16_t *addr, int cnt)
1645 {}
1646
1647 #else /* __lint */
1648
1649 #if defined(__amd64)
1650
1651 ENTRY(repinsw)
1652 movl %edx, %ecx
1653 movw %di, %dx
1654 rep
1655 D16 insl
1656 ret
1657 SET_SIZE(repinsw)
1658
1659 #elif defined(__i386)
1660
1661 ENTRY(repinsw)
1662 pushl %edi
1663 movl PORT(%esp), %edx
1664 movl ADDR(%esp), %edi
1665 movl COUNT(%esp), %ecx
1666 rep
1667 D16 insl
1668 popl %edi
1669 ret
1670 SET_SIZE(repinsw)
1671
1672 #endif /* __i386 */
1673 #endif /* __lint */
1674
1675
1676 #if defined(__lint)
1677
1678 /* ARGSUSED */
1679 void
1680 repinsb(int port, uint8_t *addr, int count)
1681 {}
1682
1683 #else /* __lint */
1684
1685 #if defined(__amd64)
1686
1687 ENTRY(repinsb)
1688 movl %edx, %ecx
1689 movw %di, %dx
1690 movq %rsi, %rdi
1691 rep
1692 insb
1693 ret
1694 SET_SIZE(repinsb)
1695
1696 #elif defined(__i386)
1697
1698 /*
1699 * The arguments and saved registers are on the stack in the
1700 * following order:
1701 * | cnt | +16
1702 * | *addr | +12
1703 * | port | +8
1704 * | eip | +4
1705 * | esi | <-- %esp
1706 * If additional values are pushed onto the stack, make sure
1707 * to adjust the following constants accordingly.
1708 */
1709 .set IO_PORT, 8
1710 .set IO_ADDR, 12
1711 .set IO_COUNT, 16
1712
1713 ENTRY(repinsb)
1714 pushl %edi
1715 movl IO_ADDR(%esp), %edi
1716 movl IO_COUNT(%esp), %ecx
1717 movl IO_PORT(%esp), %edx
1718 rep
1719 insb
1720 popl %edi
1721 ret
1722 SET_SIZE(repinsb)
1723
1724 #endif /* __i386 */
1725 #endif /* __lint */
1726
1727
1728 /*
1729 * Input a stream of 32-bit words.
1730 * NOTE: count is a DWORD count.
1731 */
1732 #if defined(__lint)
1733
1734 /* ARGSUSED */
1735 void
1736 repinsd(int port, uint32_t *addr, int count)
1737 {}
1738
1739 #else /* __lint */
1740
1741 #if defined(__amd64)
1742
1743 ENTRY(repinsd)
1744 movl %edx, %ecx
1745 movw %di, %dx
1746 movq %rsi, %rdi
1747 rep
1748 insl
1749 ret
1750 SET_SIZE(repinsd)
1751
1752 #elif defined(__i386)
1753
1754 ENTRY(repinsd)
1755 pushl %edi
1756 movl IO_ADDR(%esp), %edi
1757 movl IO_COUNT(%esp), %ecx
1758 movl IO_PORT(%esp), %edx
1759 rep
1760 insl
1761 popl %edi
1762 ret
1763 SET_SIZE(repinsd)
1764
1765 #endif /* __i386 */
1766 #endif /* __lint */
1767
1768 /*
1769 * Output a stream of bytes
1770 * NOTE: count is a byte count
1771 */
1772 #if defined(__lint)
1773
1774 /* ARGSUSED */
1775 void
1776 repoutsb(int port, uint8_t *addr, int count)
1777 {}
1778
1779 #else /* __lint */
1780
1781 #if defined(__amd64)
1782
1783 ENTRY(repoutsb)
1784 movl %edx, %ecx
1785 movw %di, %dx
1786 rep
1787 outsb
1788 ret
1789 SET_SIZE(repoutsb)
1790
1791 #elif defined(__i386)
1792
1793 ENTRY(repoutsb)
1794 pushl %esi
1795 movl IO_ADDR(%esp), %esi
1796 movl IO_COUNT(%esp), %ecx
1797 movl IO_PORT(%esp), %edx
1798 rep
1799 outsb
1800 popl %esi
1801 ret
1802 SET_SIZE(repoutsb)
1803
1804 #endif /* __i386 */
1805 #endif /* __lint */
1806
1807 /*
1808 * Output a stream of 32-bit words
1809 * NOTE: count is a DWORD count
1810 */
1811 #if defined(__lint)
1812
1813 /* ARGSUSED */
1814 void
1815 repoutsd(int port, uint32_t *addr, int count)
1816 {}
1817
1818 #else /* __lint */
1819
1820 #if defined(__amd64)
1821
1822 ENTRY(repoutsd)
1823 movl %edx, %ecx
1824 movw %di, %dx
1825 rep
1826 outsl
1827 ret
1828 SET_SIZE(repoutsd)
1829
1830 #elif defined(__i386)
1831
1832 ENTRY(repoutsd)
1833 pushl %esi
1834 movl IO_ADDR(%esp), %esi
1835 movl IO_COUNT(%esp), %ecx
1836 movl IO_PORT(%esp), %edx
1837 rep
1838 outsl
1839 popl %esi
1840 ret
1841 SET_SIZE(repoutsd)
1842
1843 #endif /* __i386 */
1844 #endif /* __lint */
1845
1846 /*
1847 * void int3(void)
1848 * void int18(void)
1849 * void int20(void)
1850 * void int_cmci(void)
1851 */
1852
1853 #if defined(__lint)
1854
1855 void
1856 int3(void)
1857 {}
1858
1859 void
1860 int18(void)
1861 {}
1862
1863 void
1864 int20(void)
1865 {}
1866
1867 void
1868 int_cmci(void)
1869 {}
1870
1871 #else /* __lint */
1872
1873 ENTRY(int3)
1874 int $T_BPTFLT
1875 ret
1876 SET_SIZE(int3)
1877
1878 ENTRY(int18)
1879 int $T_MCE
1880 ret
1881 SET_SIZE(int18)
1882
1883 ENTRY(int20)
1884 movl boothowto, %eax
1885 andl $RB_DEBUG, %eax
1886 jz 1f
1887
1888 int $T_DBGENTR
1889 1:
1890 rep; ret /* use 2 byte return instruction when branch target */
1891 /* AMD Software Optimization Guide - Section 6.2 */
1892 SET_SIZE(int20)
1893
1894 ENTRY(int_cmci)
1895 int $T_ENOEXTFLT
1896 ret
1897 SET_SIZE(int_cmci)
1898
1899 #endif /* __lint */
1900
1901 #if defined(__lint)
1902
1903 /* ARGSUSED */
1904 int
1905 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1906 { return (0); }
1907
1908 #else /* __lint */
1909
1910 #if defined(__amd64)
1911
1912 ENTRY(scanc)
1913 /* rdi == size */
1914 /* rsi == cp */
1915 /* rdx == table */
1916 /* rcx == mask */
1917 addq %rsi, %rdi /* end = &cp[size] */
1918 .scanloop:
1919 cmpq %rdi, %rsi /* while (cp < end */
1920 jnb .scandone
1921 movzbq (%rsi), %r8 /* %r8 = *cp */
1922 incq %rsi /* cp++ */
1923 testb %cl, (%r8, %rdx)
1924 jz .scanloop /* && (table[*cp] & mask) == 0) */
1925 decq %rsi /* (fix post-increment) */
1926 .scandone:
1927 movl %edi, %eax
1928 subl %esi, %eax /* return (end - cp) */
1929 ret
1930 SET_SIZE(scanc)
1931
1932 #elif defined(__i386)
1933
1934 ENTRY(scanc)
1935 pushl %edi
1936 pushl %esi
1937 movb 24(%esp), %cl /* mask = %cl */
1938 movl 16(%esp), %esi /* cp = %esi */
1939 movl 20(%esp), %edx /* table = %edx */
1940 movl %esi, %edi
1941 addl 12(%esp), %edi /* end = &cp[size]; */
1942 .scanloop:
1943 cmpl %edi, %esi /* while (cp < end */
1944 jnb .scandone
1945 movzbl (%esi), %eax /* %al = *cp */
1946 incl %esi /* cp++ */
1947 movb (%edx, %eax), %al /* %al = table[*cp] */
1948 testb %al, %cl
1949 jz .scanloop /* && (table[*cp] & mask) == 0) */
1950 dec %esi /* post-incremented */
1951 .scandone:
1952 movl %edi, %eax
1953 subl %esi, %eax /* return (end - cp) */
1954 popl %esi
1955 popl %edi
1956 ret
1957 SET_SIZE(scanc)
1958
1959 #endif /* __i386 */
1960 #endif /* __lint */
1961
1962 /*
1963 * Replacement functions for ones that are normally inlined.
1964 * In addition to the copy in i86.il, they are defined here just in case.
1965 */
1966
1967 #if defined(__lint)
1968
1969 ulong_t
1970 intr_clear(void)
1971 { return (0); }
1972
1973 ulong_t
1974 clear_int_flag(void)
1975 { return (0); }
1976
1977 #else /* __lint */
1978
1979 #if defined(__amd64)
1980
1981 ENTRY(intr_clear)
1982 ENTRY(clear_int_flag)
1983 pushfq
1984 popq %rax
1985 #if defined(__xpv)
1986 leaq xpv_panicking, %rdi
1987 movl (%rdi), %edi
1988 cmpl $0, %edi
1989 jne 2f
1990 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1991 /*
1992 * Synthesize the PS_IE bit from the event mask bit
1993 */
1994 andq $_BITNOT(PS_IE), %rax
1995 testb $1, %dl
1996 jnz 1f
1997 orq $PS_IE, %rax
1998 1:
1999 ret
2000 2:
2001 #endif
2002 CLI(%rdi)
2003 ret
2004 SET_SIZE(clear_int_flag)
2005 SET_SIZE(intr_clear)
2006
2007 #elif defined(__i386)
2008
2009 ENTRY(intr_clear)
2010 ENTRY(clear_int_flag)
2011 pushfl
2012 popl %eax
2013 #if defined(__xpv)
2014 leal xpv_panicking, %edx
2015 movl (%edx), %edx
2016 cmpl $0, %edx
2017 jne 2f
2018 CLIRET(%edx, %cl) /* returns event mask in %cl */
2019 /*
2020 * Synthesize the PS_IE bit from the event mask bit
2021 */
2022 andl $_BITNOT(PS_IE), %eax
2023 testb $1, %cl
2024 jnz 1f
2025 orl $PS_IE, %eax
2026 1:
2027 ret
2028 2:
2029 #endif
2030 CLI(%edx)
2031 ret
2032 SET_SIZE(clear_int_flag)
2033 SET_SIZE(intr_clear)
2034
2035 #endif /* __i386 */
2036 #endif /* __lint */
2037
2038 #if defined(__lint)
2039
2040 struct cpu *
2041 curcpup(void)
2042 { return 0; }
2043
2044 #else /* __lint */
2045
2046 #if defined(__amd64)
2047
2048 ENTRY(curcpup)
2049 movq %gs:CPU_SELF, %rax
2050 ret
2051 SET_SIZE(curcpup)
2052
2053 #elif defined(__i386)
2054
2055 ENTRY(curcpup)
2056 movl %gs:CPU_SELF, %eax
2057 ret
2058 SET_SIZE(curcpup)
2059
2060 #endif /* __i386 */
2061 #endif /* __lint */
2062
2063 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2064 * These functions reverse the byte order of the input parameter and returns
2065 * the result. This is to convert the byte order from host byte order
2066 * (little endian) to network byte order (big endian), or vice versa.
2067 */
2068
2069 #if defined(__lint)
2070
2071 uint64_t
2072 htonll(uint64_t i)
2073 { return (i); }
2074
2075 uint64_t
2076 ntohll(uint64_t i)
2077 { return (i); }
2078
2079 uint32_t
2080 htonl(uint32_t i)
2081 { return (i); }
2082
2083 uint32_t
2084 ntohl(uint32_t i)
2085 { return (i); }
2086
2087 uint16_t
2088 htons(uint16_t i)
2089 { return (i); }
2090
2091 uint16_t
2092 ntohs(uint16_t i)
2093 { return (i); }
2094
2095 #else /* __lint */
2096
2097 #if defined(__amd64)
2098
2099 ENTRY(htonll)
2100 ALTENTRY(ntohll)
2101 movq %rdi, %rax
2102 bswapq %rax
2103 ret
2104 SET_SIZE(ntohll)
2105 SET_SIZE(htonll)
2106
2107 /* XX64 there must be shorter sequences for this */
2108 ENTRY(htonl)
2109 ALTENTRY(ntohl)
2110 movl %edi, %eax
2111 bswap %eax
2112 ret
2113 SET_SIZE(ntohl)
2114 SET_SIZE(htonl)
2115
2116 /* XX64 there must be better sequences for this */
2117 ENTRY(htons)
2118 ALTENTRY(ntohs)
2119 movl %edi, %eax
2120 bswap %eax
2121 shrl $16, %eax
2122 ret
2123 SET_SIZE(ntohs)
2124 SET_SIZE(htons)
2125
2126 #elif defined(__i386)
2127
2128 ENTRY(htonll)
2129 ALTENTRY(ntohll)
2130 movl 4(%esp), %edx
2131 movl 8(%esp), %eax
2132 bswap %edx
2133 bswap %eax
2134 ret
2135 SET_SIZE(ntohll)
2136 SET_SIZE(htonll)
2137
2138 ENTRY(htonl)
2139 ALTENTRY(ntohl)
2140 movl 4(%esp), %eax
2141 bswap %eax
2142 ret
2143 SET_SIZE(ntohl)
2144 SET_SIZE(htonl)
2145
2146 ENTRY(htons)
2147 ALTENTRY(ntohs)
2148 movl 4(%esp), %eax
2149 bswap %eax
2150 shrl $16, %eax
2151 ret
2152 SET_SIZE(ntohs)
2153 SET_SIZE(htons)
2154
2155 #endif /* __i386 */
2156 #endif /* __lint */
2157
2158
2159 #if defined(__lint)
2160
2161 /* ARGSUSED */
2162 void
2163 intr_restore(ulong_t i)
2164 { return; }
2165
2166 /* ARGSUSED */
2167 void
2168 restore_int_flag(ulong_t i)
2169 { return; }
2170
2171 #else /* __lint */
2172
2173 #if defined(__amd64)
2174
2175 ENTRY(intr_restore)
2176 ENTRY(restore_int_flag)
2177 testq $PS_IE, %rdi
2178 jz 1f
2179 #if defined(__xpv)
2180 leaq xpv_panicking, %rsi
2181 movl (%rsi), %esi
2182 cmpl $0, %esi
2183 jne 1f
2184 /*
2185 * Since we're -really- running unprivileged, our attempt
2186 * to change the state of the IF bit will be ignored.
2187 * The virtual IF bit is tweaked by CLI and STI.
2188 */
2189 IE_TO_EVENT_MASK(%rsi, %rdi)
2190 #else
2191 sti
2192 #endif
2193 1:
2194 ret
2195 SET_SIZE(restore_int_flag)
2196 SET_SIZE(intr_restore)
2197
2198 #elif defined(__i386)
2199
2200 ENTRY(intr_restore)
2201 ENTRY(restore_int_flag)
2202 testl $PS_IE, 4(%esp)
2203 jz 1f
2204 #if defined(__xpv)
2205 leal xpv_panicking, %edx
2206 movl (%edx), %edx
2207 cmpl $0, %edx
2208 jne 1f
2209 /*
2210 * Since we're -really- running unprivileged, our attempt
2211 * to change the state of the IF bit will be ignored.
2212 * The virtual IF bit is tweaked by CLI and STI.
2213 */
2214 IE_TO_EVENT_MASK(%edx, 4(%esp))
2215 #else
2216 sti
2217 #endif
2218 1:
2219 ret
2220 SET_SIZE(restore_int_flag)
2221 SET_SIZE(intr_restore)
2222
2223 #endif /* __i386 */
2224 #endif /* __lint */
2225
2226 #if defined(__lint)
2227
2228 void
2229 sti(void)
2230 {}
2231
2232 void
2233 cli(void)
2234 {}
2235
2236 #else /* __lint */
2237
2238 ENTRY(sti)
2239 STI
2240 ret
2241 SET_SIZE(sti)
2242
2243 ENTRY(cli)
2244 #if defined(__amd64)
2245 CLI(%rax)
2246 #elif defined(__i386)
2247 CLI(%eax)
2248 #endif /* __i386 */
2249 ret
2250 SET_SIZE(cli)
2251
2252 #endif /* __lint */
2253
2254 #if defined(__lint)
2255
2256 dtrace_icookie_t
2257 dtrace_interrupt_disable(void)
2258 { return (0); }
2259
2260 #else /* __lint */
2261
2262 #if defined(__amd64)
2263
2264 ENTRY(dtrace_interrupt_disable)
2265 pushfq
2266 popq %rax
2267 #if defined(__xpv)
2268 leaq xpv_panicking, %rdi
2269 movl (%rdi), %edi
2270 cmpl $0, %edi
2271 jne .dtrace_interrupt_disable_done
2272 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2273 /*
2274 * Synthesize the PS_IE bit from the event mask bit
2275 */
2276 andq $_BITNOT(PS_IE), %rax
2277 testb $1, %dl
2278 jnz .dtrace_interrupt_disable_done
2279 orq $PS_IE, %rax
2280 #else
2281 CLI(%rdx)
2282 #endif
2283 .dtrace_interrupt_disable_done:
2284 ret
2285 SET_SIZE(dtrace_interrupt_disable)
2286
2287 #elif defined(__i386)
2288
2289 ENTRY(dtrace_interrupt_disable)
2290 pushfl
2291 popl %eax
2292 #if defined(__xpv)
2293 leal xpv_panicking, %edx
2294 movl (%edx), %edx
2295 cmpl $0, %edx
2296 jne .dtrace_interrupt_disable_done
2297 CLIRET(%edx, %cl) /* returns event mask in %cl */
2298 /*
2299 * Synthesize the PS_IE bit from the event mask bit
2300 */
2301 andl $_BITNOT(PS_IE), %eax
2302 testb $1, %cl
2303 jnz .dtrace_interrupt_disable_done
2304 orl $PS_IE, %eax
2305 #else
2306 CLI(%edx)
2307 #endif
2308 .dtrace_interrupt_disable_done:
2309 ret
2310 SET_SIZE(dtrace_interrupt_disable)
2311
2312 #endif /* __i386 */
2313 #endif /* __lint */
2314
2315 #if defined(__lint)
2316
2317 /*ARGSUSED*/
2318 void
2319 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2320 {}
2321
2322 #else /* __lint */
2323
2324 #if defined(__amd64)
2325
2326 ENTRY(dtrace_interrupt_enable)
2327 pushq %rdi
2328 popfq
2329 #if defined(__xpv)
2330 leaq xpv_panicking, %rdx
2331 movl (%rdx), %edx
2332 cmpl $0, %edx
2333 jne .dtrace_interrupt_enable_done
2334 /*
2335 * Since we're -really- running unprivileged, our attempt
2336 * to change the state of the IF bit will be ignored. The
2337 * virtual IF bit is tweaked by CLI and STI.
2338 */
2339 IE_TO_EVENT_MASK(%rdx, %rdi)
2340 #endif
2341 .dtrace_interrupt_enable_done:
2342 ret
2343 SET_SIZE(dtrace_interrupt_enable)
2344
2345 #elif defined(__i386)
2346
2347 ENTRY(dtrace_interrupt_enable)
2348 movl 4(%esp), %eax
2349 pushl %eax
2350 popfl
2351 #if defined(__xpv)
2352 leal xpv_panicking, %edx
2353 movl (%edx), %edx
2354 cmpl $0, %edx
2355 jne .dtrace_interrupt_enable_done
2356 /*
2357 * Since we're -really- running unprivileged, our attempt
2358 * to change the state of the IF bit will be ignored. The
2359 * virtual IF bit is tweaked by CLI and STI.
2360 */
2361 IE_TO_EVENT_MASK(%edx, %eax)
2362 #endif
2363 .dtrace_interrupt_enable_done:
2364 ret
2365 SET_SIZE(dtrace_interrupt_enable)
2366
2367 #endif /* __i386 */
2368 #endif /* __lint */
2369
2370
2371 #if defined(lint)
2372
2373 void
2374 dtrace_membar_producer(void)
2375 {}
2376
2377 void
2378 dtrace_membar_consumer(void)
2379 {}
2380
2381 #else /* __lint */
2382
2383 ENTRY(dtrace_membar_producer)
2384 rep; ret /* use 2 byte return instruction when branch target */
2385 /* AMD Software Optimization Guide - Section 6.2 */
2386 SET_SIZE(dtrace_membar_producer)
2387
2388 ENTRY(dtrace_membar_consumer)
2389 rep; ret /* use 2 byte return instruction when branch target */
2390 /* AMD Software Optimization Guide - Section 6.2 */
2391 SET_SIZE(dtrace_membar_consumer)
2392
2393 #endif /* __lint */
2394
2395 #if defined(__lint)
2396
2397 kthread_id_t
2398 threadp(void)
2399 { return ((kthread_id_t)0); }
2400
2401 #else /* __lint */
2402
2403 #if defined(__amd64)
2404
2405 ENTRY(threadp)
2406 movq %gs:CPU_THREAD, %rax
2407 ret
2408 SET_SIZE(threadp)
2409
2410 #elif defined(__i386)
2411
2412 ENTRY(threadp)
2413 movl %gs:CPU_THREAD, %eax
2414 ret
2415 SET_SIZE(threadp)
2416
2417 #endif /* __i386 */
2418 #endif /* __lint */
2419
2420 /*
2421 * Checksum routine for Internet Protocol Headers
2422 */
2423
2424 #if defined(__lint)
2425
2426 /* ARGSUSED */
2427 unsigned int
2428 ip_ocsum(
2429 ushort_t *address, /* ptr to 1st message buffer */
2430 int halfword_count, /* length of data */
2431 unsigned int sum) /* partial checksum */
2432 {
2433 int i;
2434 unsigned int psum = 0; /* partial sum */
2435
2436 for (i = 0; i < halfword_count; i++, address++) {
2437 psum += *address;
2438 }
2439
2440 while ((psum >> 16) != 0) {
2441 psum = (psum & 0xffff) + (psum >> 16);
2442 }
2443
2444 psum += sum;
2445
2446 while ((psum >> 16) != 0) {
2447 psum = (psum & 0xffff) + (psum >> 16);
2448 }
2449
2450 return (psum);
2451 }
2452
2453 #else /* __lint */
2454
2455 #if defined(__amd64)
2456
2457 ENTRY(ip_ocsum)
2458 pushq %rbp
2459 movq %rsp, %rbp
2460 #ifdef DEBUG
2461 movq postbootkernelbase(%rip), %rax
2462 cmpq %rax, %rdi
2463 jnb 1f
2464 xorl %eax, %eax
2465 movq %rdi, %rsi
2466 leaq .ip_ocsum_panic_msg(%rip), %rdi
2467 call panic
2468 /*NOTREACHED*/
2469 .ip_ocsum_panic_msg:
2470 .string "ip_ocsum: address 0x%p below kernelbase\n"
2471 1:
2472 #endif
2473 movl %esi, %ecx /* halfword_count */
2474 movq %rdi, %rsi /* address */
2475 /* partial sum in %edx */
2476 xorl %eax, %eax
2477 testl %ecx, %ecx
2478 jz .ip_ocsum_done
2479 testq $3, %rsi
2480 jnz .ip_csum_notaligned
2481 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2482 .next_iter:
2483 /* XX64 opportunities for prefetch? */
2484 /* XX64 compute csum with 64 bit quantities? */
2485 subl $32, %ecx
2486 jl .less_than_32
2487
2488 addl 0(%rsi), %edx
2489 .only60:
2490 adcl 4(%rsi), %eax
2491 .only56:
2492 adcl 8(%rsi), %edx
2493 .only52:
2494 adcl 12(%rsi), %eax
2495 .only48:
2496 adcl 16(%rsi), %edx
2497 .only44:
2498 adcl 20(%rsi), %eax
2499 .only40:
2500 adcl 24(%rsi), %edx
2501 .only36:
2502 adcl 28(%rsi), %eax
2503 .only32:
2504 adcl 32(%rsi), %edx
2505 .only28:
2506 adcl 36(%rsi), %eax
2507 .only24:
2508 adcl 40(%rsi), %edx
2509 .only20:
2510 adcl 44(%rsi), %eax
2511 .only16:
2512 adcl 48(%rsi), %edx
2513 .only12:
2514 adcl 52(%rsi), %eax
2515 .only8:
2516 adcl 56(%rsi), %edx
2517 .only4:
2518 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2519 .only0:
2520 adcl $0, %eax /* could be adding -1 in eax with a carry */
2521 adcl $0, %eax
2522
2523 addq $64, %rsi
2524 testl %ecx, %ecx
2525 jnz .next_iter
2526
2527 .ip_ocsum_done:
2528 addl %eax, %edx
2529 adcl $0, %edx
2530 movl %edx, %eax /* form a 16 bit checksum by */
2531 shrl $16, %eax /* adding two halves of 32 bit checksum */
2532 addw %dx, %ax
2533 adcw $0, %ax
2534 andl $0xffff, %eax
2535 leave
2536 ret
2537
2538 .ip_csum_notaligned:
2539 xorl %edi, %edi
2540 movw (%rsi), %di
2541 addl %edi, %edx
2542 adcl $0, %edx
2543 addq $2, %rsi
2544 decl %ecx
2545 jmp .ip_csum_aligned
2546
2547 .less_than_32:
2548 addl $32, %ecx
2549 testl $1, %ecx
2550 jz .size_aligned
2551 andl $0xfe, %ecx
2552 movzwl (%rsi, %rcx, 2), %edi
2553 addl %edi, %edx
2554 adcl $0, %edx
2555 .size_aligned:
2556 movl %ecx, %edi
2557 shrl $1, %ecx
2558 shl $1, %edi
2559 subq $64, %rdi
2560 addq %rdi, %rsi
2561 leaq .ip_ocsum_jmptbl(%rip), %rdi
2562 leaq (%rdi, %rcx, 8), %rdi
2563 xorl %ecx, %ecx
2564 clc
2565 jmp *(%rdi)
2566
2567 .align 8
2568 .ip_ocsum_jmptbl:
2569 .quad .only0, .only4, .only8, .only12, .only16, .only20
2570 .quad .only24, .only28, .only32, .only36, .only40, .only44
2571 .quad .only48, .only52, .only56, .only60
2572 SET_SIZE(ip_ocsum)
2573
2574 #elif defined(__i386)
2575
2576 ENTRY(ip_ocsum)
2577 pushl %ebp
2578 movl %esp, %ebp
2579 pushl %ebx
2580 pushl %esi
2581 pushl %edi
2582 movl 12(%ebp), %ecx /* count of half words */
2583 movl 16(%ebp), %edx /* partial checksum */
2584 movl 8(%ebp), %esi
2585 xorl %eax, %eax
2586 testl %ecx, %ecx
2587 jz .ip_ocsum_done
2588
2589 testl $3, %esi
2590 jnz .ip_csum_notaligned
2591 .ip_csum_aligned:
2592 .next_iter:
2593 subl $32, %ecx
2594 jl .less_than_32
2595
2596 addl 0(%esi), %edx
2597 .only60:
2598 adcl 4(%esi), %eax
2599 .only56:
2600 adcl 8(%esi), %edx
2601 .only52:
2602 adcl 12(%esi), %eax
2603 .only48:
2604 adcl 16(%esi), %edx
2605 .only44:
2606 adcl 20(%esi), %eax
2607 .only40:
2608 adcl 24(%esi), %edx
2609 .only36:
2610 adcl 28(%esi), %eax
2611 .only32:
2612 adcl 32(%esi), %edx
2613 .only28:
2614 adcl 36(%esi), %eax
2615 .only24:
2616 adcl 40(%esi), %edx
2617 .only20:
2618 adcl 44(%esi), %eax
2619 .only16:
2620 adcl 48(%esi), %edx
2621 .only12:
2622 adcl 52(%esi), %eax
2623 .only8:
2624 adcl 56(%esi), %edx
2625 .only4:
2626 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2627 .only0:
2628 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2629 adcl $0, %eax
2630
2631 addl $64, %esi
2632 andl %ecx, %ecx
2633 jnz .next_iter
2634
2635 .ip_ocsum_done:
2636 addl %eax, %edx
2637 adcl $0, %edx
2638 movl %edx, %eax /* form a 16 bit checksum by */
2639 shrl $16, %eax /* adding two halves of 32 bit checksum */
2640 addw %dx, %ax
2641 adcw $0, %ax
2642 andl $0xffff, %eax
2643 popl %edi /* restore registers */
2644 popl %esi
2645 popl %ebx
2646 leave
2647 ret
2648
2649 .ip_csum_notaligned:
2650 xorl %edi, %edi
2651 movw (%esi), %di
2652 addl %edi, %edx
2653 adcl $0, %edx
2654 addl $2, %esi
2655 decl %ecx
2656 jmp .ip_csum_aligned
2657
2658 .less_than_32:
2659 addl $32, %ecx
2660 testl $1, %ecx
2661 jz .size_aligned
2662 andl $0xfe, %ecx
2663 movzwl (%esi, %ecx, 2), %edi
2664 addl %edi, %edx
2665 adcl $0, %edx
2666 .size_aligned:
2667 movl %ecx, %edi
2668 shrl $1, %ecx
2669 shl $1, %edi
2670 subl $64, %edi
2671 addl %edi, %esi
2672 movl $.ip_ocsum_jmptbl, %edi
2673 lea (%edi, %ecx, 4), %edi
2674 xorl %ecx, %ecx
2675 clc
2676 jmp *(%edi)
2677 SET_SIZE(ip_ocsum)
2678
2679 .data
2680 .align 4
2681
2682 .ip_ocsum_jmptbl:
2683 .long .only0, .only4, .only8, .only12, .only16, .only20
2684 .long .only24, .only28, .only32, .only36, .only40, .only44
2685 .long .only48, .only52, .only56, .only60
2686
2687
2688 #endif /* __i386 */
2689 #endif /* __lint */
2690
2691 /*
2692 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2693 * Provided to manipulate hrtime_t values.
2694 */
2695 #if defined(__lint)
2696
2697 /* result = a * b; */
2698
2699 /* ARGSUSED */
2700 unsigned long long
2701 mul32(uint_t a, uint_t b)
2702 { return (0); }
2703
2704 #else /* __lint */
2705
2706 #if defined(__amd64)
2707
2708 ENTRY(mul32)
2709 xorl %edx, %edx /* XX64 joe, paranoia? */
2710 movl %edi, %eax
2711 mull %esi
2712 shlq $32, %rdx
2713 orq %rdx, %rax
2714 ret
2715 SET_SIZE(mul32)
2716
2717 #elif defined(__i386)
2718
2719 ENTRY(mul32)
2720 movl 8(%esp), %eax
2721 movl 4(%esp), %ecx
2722 mull %ecx
2723 ret
2724 SET_SIZE(mul32)
2725
2726 #endif /* __i386 */
2727 #endif /* __lint */
2728
2729 #if defined(notused)
2730 #if defined(__lint)
2731 /* ARGSUSED */
2732 void
2733 load_pte64(uint64_t *pte, uint64_t pte_value)
2734 {}
2735 #else /* __lint */
2736 .globl load_pte64
2737 load_pte64:
2738 movl 4(%esp), %eax
2739 movl 8(%esp), %ecx
2740 movl 12(%esp), %edx
2741 movl %edx, 4(%eax)
2742 movl %ecx, (%eax)
2743 ret
2744 #endif /* __lint */
2745 #endif /* notused */
2746
2747 #if defined(__lint)
2748
2749 /*ARGSUSED*/
2750 void
2751 scan_memory(caddr_t addr, size_t size)
2752 {}
2753
2754 #else /* __lint */
2755
2756 #if defined(__amd64)
2757
2758 ENTRY(scan_memory)
2759 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2760 jz .scanm_done
2761 movq %rsi, %rcx /* move count into rep control register */
2762 movq %rdi, %rsi /* move addr into lodsq control reg. */
2763 rep lodsq /* scan the memory range */
2764 .scanm_done:
2765 rep; ret /* use 2 byte return instruction when branch target */
2766 /* AMD Software Optimization Guide - Section 6.2 */
2767 SET_SIZE(scan_memory)
2768
2769 #elif defined(__i386)
2770
2771 ENTRY(scan_memory)
2772 pushl %ecx
2773 pushl %esi
2774 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2775 shrl $2, %ecx /* convert from byte count to word count */
2776 jz .scanm_done
2777 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2778 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2779 lodsl
2780 .scanm_done:
2781 popl %esi
2782 popl %ecx
2783 ret
2784 SET_SIZE(scan_memory)
2785
2786 #endif /* __i386 */
2787 #endif /* __lint */
2788
2789
2790 #if defined(__lint)
2791
2792 /*ARGSUSED */
2793 int
2794 lowbit(ulong_t i)
2795 { return (0); }
2796
2797 #else /* __lint */
2798
2799 #if defined(__amd64)
2800
2801 ENTRY(lowbit)
2802 movl $-1, %eax
2803 bsfq %rdi, %rax
2804 incl %eax
2805 ret
2806 SET_SIZE(lowbit)
2807
2808 #elif defined(__i386)
2809
2810 ENTRY(lowbit)
2811 movl $-1, %eax
2812 bsfl 4(%esp), %eax
2813 incl %eax
2814 ret
2815 SET_SIZE(lowbit)
2816
2817 #endif /* __i386 */
2818 #endif /* __lint */
2819
2820 #if defined(__lint)
2821
2822 /*ARGSUSED*/
2823 int
2824 highbit(ulong_t i)
2825 { return (0); }
2826
2827 #else /* __lint */
2828
2829 #if defined(__amd64)
2830
2831 ENTRY(highbit)
2832 movl $-1, %eax
2833 bsrq %rdi, %rax
2834 incl %eax
2835 ret
2836 SET_SIZE(highbit)
2837
2838 #elif defined(__i386)
2839
2840 ENTRY(highbit)
2841 movl $-1, %eax
2842 bsrl 4(%esp), %eax
2843 incl %eax
2844 ret
2845 SET_SIZE(highbit)
2846
2847 #endif /* __i386 */
2848 #endif /* __lint */
2849
2850 #if defined(__lint)
2851
2852 /*ARGSUSED*/
2853 uint64_t
2854 rdmsr(uint_t r)
2855 { return (0); }
2856
2857 /*ARGSUSED*/
2858 void
2859 wrmsr(uint_t r, const uint64_t val)
2860 {}
2861
2862 /*ARGSUSED*/
2863 uint64_t
2864 xrdmsr(uint_t r)
2865 { return (0); }
2866
2867 /*ARGSUSED*/
2868 void
2869 xwrmsr(uint_t r, const uint64_t val)
2870 {}
2871
2872 void
2873 invalidate_cache(void)
2874 {}
2875
2876 /*ARGSUSED*/
2877 uint64_t
2878 get_xcr(uint_t r)
2879 { return (0); }
2880
2881 /*ARGSUSED*/
2882 void
2883 set_xcr(uint_t r, const uint64_t val)
2884 {}
2885
2886 #else /* __lint */
2887
2888 #define XMSR_ACCESS_VAL $0x9c5a203a
2889
2890 #if defined(__amd64)
2891
2892 ENTRY(rdmsr)
2893 movl %edi, %ecx
2894 rdmsr
2895 shlq $32, %rdx
2896 orq %rdx, %rax
2897 ret
2898 SET_SIZE(rdmsr)
2899
2900 ENTRY(wrmsr)
2901 movq %rsi, %rdx
2902 shrq $32, %rdx
2903 movl %esi, %eax
2904 movl %edi, %ecx
2905 wrmsr
2906 ret
2907 SET_SIZE(wrmsr)
2908
2909 ENTRY(xrdmsr)
2910 pushq %rbp
2911 movq %rsp, %rbp
2912 movl %edi, %ecx
2913 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2914 rdmsr
2915 shlq $32, %rdx
2916 orq %rdx, %rax
2917 leave
2918 ret
2919 SET_SIZE(xrdmsr)
2920
2921 ENTRY(xwrmsr)
2922 pushq %rbp
2923 movq %rsp, %rbp
2924 movl %edi, %ecx
2925 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2926 movq %rsi, %rdx
2927 shrq $32, %rdx
2928 movl %esi, %eax
2929 wrmsr
2930 leave
2931 ret
2932 SET_SIZE(xwrmsr)
2933
2934 ENTRY(get_xcr)
2935 movl %edi, %ecx
2936 #xgetbv
2937 .byte 0x0f,0x01,0xd0
2938 shlq $32, %rdx
2939 orq %rdx, %rax
2940 ret
2941 SET_SIZE(get_xcr)
2942
2943 ENTRY(set_xcr)
2944 movq %rsi, %rdx
2945 shrq $32, %rdx
2946 movl %esi, %eax
2947 movl %edi, %ecx
2948 #xsetbv
2949 .byte 0x0f,0x01,0xd1
2950 ret
2951 SET_SIZE(set_xcr)
2952
2953 #elif defined(__i386)
2954
2955 ENTRY(rdmsr)
2956 movl 4(%esp), %ecx
2957 rdmsr
2958 ret
2959 SET_SIZE(rdmsr)
2960
2961 ENTRY(wrmsr)
2962 movl 4(%esp), %ecx
2963 movl 8(%esp), %eax
2964 movl 12(%esp), %edx
2965 wrmsr
2966 ret
2967 SET_SIZE(wrmsr)
2968
2969 ENTRY(xrdmsr)
2970 pushl %ebp
2971 movl %esp, %ebp
2972 movl 8(%esp), %ecx
2973 pushl %edi
2974 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2975 rdmsr
2976 popl %edi
2977 leave
2978 ret
2979 SET_SIZE(xrdmsr)
2980
2981 ENTRY(xwrmsr)
2982 pushl %ebp
2983 movl %esp, %ebp
2984 movl 8(%esp), %ecx
2985 movl 12(%esp), %eax
2986 movl 16(%esp), %edx
2987 pushl %edi
2988 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2989 wrmsr
2990 popl %edi
2991 leave
2992 ret
2993 SET_SIZE(xwrmsr)
2994
2995 ENTRY(get_xcr)
2996 movl 4(%esp), %ecx
2997 #xgetbv
2998 .byte 0x0f,0x01,0xd0
2999 ret
3000 SET_SIZE(get_xcr)
3001
3002 ENTRY(set_xcr)
3003 movl 4(%esp), %ecx
3004 movl 8(%esp), %eax
3005 movl 12(%esp), %edx
3006 #xsetbv
3007 .byte 0x0f,0x01,0xd1
3008 ret
3009 SET_SIZE(set_xcr)
3010
3011 #endif /* __i386 */
3012
3013 ENTRY(invalidate_cache)
3014 wbinvd
3015 ret
3016 SET_SIZE(invalidate_cache)
3017
3018 #endif /* __lint */
3019
3020 #if defined(__lint)
3021
3022 /*ARGSUSED*/
3023 void
3024 getcregs(struct cregs *crp)
3025 {}
3026
3027 #else /* __lint */
3028
3029 #if defined(__amd64)
3030
3031 ENTRY_NP(getcregs)
3032 #if defined(__xpv)
3033 /*
3034 * Only a few of the hardware control registers or descriptor tables
3035 * are directly accessible to us, so just zero the structure.
3036 *
3037 * XXPV Perhaps it would be helpful for the hypervisor to return
3038 * virtualized versions of these for post-mortem use.
3039 * (Need to reevaluate - perhaps it already does!)
3040 */
3041 pushq %rdi /* save *crp */
3042 movq $CREGSZ, %rsi
3043 call bzero
3044 popq %rdi
3045
3046 /*
3047 * Dump what limited information we can
3048 */
3049 movq %cr0, %rax
3050 movq %rax, CREG_CR0(%rdi) /* cr0 */
3051 movq %cr2, %rax
3052 movq %rax, CREG_CR2(%rdi) /* cr2 */
3053 movq %cr3, %rax
3054 movq %rax, CREG_CR3(%rdi) /* cr3 */
3055 movq %cr4, %rax
3056 movq %rax, CREG_CR4(%rdi) /* cr4 */
3057
3058 #else /* __xpv */
3059
3060 #define GETMSR(r, off, d) \
3061 movl $r, %ecx; \
3062 rdmsr; \
3063 movl %eax, off(d); \
3064 movl %edx, off+4(d)
3065
3066 xorl %eax, %eax
3067 movq %rax, CREG_GDT+8(%rdi)
3068 sgdt CREG_GDT(%rdi) /* 10 bytes */
3069 movq %rax, CREG_IDT+8(%rdi)
3070 sidt CREG_IDT(%rdi) /* 10 bytes */
3071 movq %rax, CREG_LDT(%rdi)
3072 sldt CREG_LDT(%rdi) /* 2 bytes */
3073 movq %rax, CREG_TASKR(%rdi)
3074 str CREG_TASKR(%rdi) /* 2 bytes */
3075 movq %cr0, %rax
3076 movq %rax, CREG_CR0(%rdi) /* cr0 */
3077 movq %cr2, %rax
3078 movq %rax, CREG_CR2(%rdi) /* cr2 */
3079 movq %cr3, %rax
3080 movq %rax, CREG_CR3(%rdi) /* cr3 */
3081 movq %cr4, %rax
3082 movq %rax, CREG_CR4(%rdi) /* cr4 */
3083 movq %cr8, %rax
3084 movq %rax, CREG_CR8(%rdi) /* cr8 */
3085 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3086 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3087 #endif /* __xpv */
3088 ret
3089 SET_SIZE(getcregs)
3090
3091 #undef GETMSR
3092
3093 #elif defined(__i386)
3094
3095 ENTRY_NP(getcregs)
3096 #if defined(__xpv)
3097 /*
3098 * Only a few of the hardware control registers or descriptor tables
3099 * are directly accessible to us, so just zero the structure.
3100 *
3101 * XXPV Perhaps it would be helpful for the hypervisor to return
3102 * virtualized versions of these for post-mortem use.
3103 * (Need to reevaluate - perhaps it already does!)
3104 */
3105 movl 4(%esp), %edx
3106 pushl $CREGSZ
3107 pushl %edx
3108 call bzero
3109 addl $8, %esp
3110 movl 4(%esp), %edx
3111
3112 /*
3113 * Dump what limited information we can
3114 */
3115 movl %cr0, %eax
3116 movl %eax, CREG_CR0(%edx) /* cr0 */
3117 movl %cr2, %eax
3118 movl %eax, CREG_CR2(%edx) /* cr2 */
3119 movl %cr3, %eax
3120 movl %eax, CREG_CR3(%edx) /* cr3 */
3121 movl %cr4, %eax
3122 movl %eax, CREG_CR4(%edx) /* cr4 */
3123
3124 #else /* __xpv */
3125
3126 movl 4(%esp), %edx
3127 movw $0, CREG_GDT+6(%edx)
3128 movw $0, CREG_IDT+6(%edx)
3129 sgdt CREG_GDT(%edx) /* gdt */
3130 sidt CREG_IDT(%edx) /* idt */
3131 sldt CREG_LDT(%edx) /* ldt */
3132 str CREG_TASKR(%edx) /* task */
3133 movl %cr0, %eax
3134 movl %eax, CREG_CR0(%edx) /* cr0 */
3135 movl %cr2, %eax
3136 movl %eax, CREG_CR2(%edx) /* cr2 */
3137 movl %cr3, %eax
3138 movl %eax, CREG_CR3(%edx) /* cr3 */
3139 bt $X86FSET_LARGEPAGE, x86_featureset
3140 jnc .nocr4
3141 movl %cr4, %eax
3142 movl %eax, CREG_CR4(%edx) /* cr4 */
3143 jmp .skip
3144 .nocr4:
3145 movl $0, CREG_CR4(%edx)
3146 .skip:
3147 #endif
3148 ret
3149 SET_SIZE(getcregs)
3150
3151 #endif /* __i386 */
3152 #endif /* __lint */
3153
3154
3155 /*
3156 * A panic trigger is a word which is updated atomically and can only be set
3157 * once. We atomically store 0xDEFACEDD and load the old value. If the
3158 * previous value was 0, we succeed and return 1; otherwise return 0.
3159 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3160 * has its own version of this function to allow it to panic correctly from
3161 * probe context.
3162 */
3163 #if defined(__lint)
3164
3165 /*ARGSUSED*/
3166 int
3167 panic_trigger(int *tp)
3168 { return (0); }
3169
3170 /*ARGSUSED*/
3171 int
3172 dtrace_panic_trigger(int *tp)
3173 { return (0); }
3174
3175 #else /* __lint */
3176
3177 #if defined(__amd64)
3178
3179 ENTRY_NP(panic_trigger)
3180 xorl %eax, %eax
3181 movl $0xdefacedd, %edx
3182 lock
3183 xchgl %edx, (%rdi)
3184 cmpl $0, %edx
3185 je 0f
3186 movl $0, %eax
3187 ret
3188 0: movl $1, %eax
3189 ret
3190 SET_SIZE(panic_trigger)
3191
3192 ENTRY_NP(dtrace_panic_trigger)
3193 xorl %eax, %eax
3194 movl $0xdefacedd, %edx
3195 lock
3196 xchgl %edx, (%rdi)
3197 cmpl $0, %edx
3198 je 0f
3199 movl $0, %eax
3200 ret
3201 0: movl $1, %eax
3202 ret
3203 SET_SIZE(dtrace_panic_trigger)
3204
3205 #elif defined(__i386)
3206
3207 ENTRY_NP(panic_trigger)
3208 movl 4(%esp), %edx / %edx = address of trigger
3209 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3210 lock / assert lock
3211 xchgl %eax, (%edx) / exchange %eax and the trigger
3212 cmpl $0, %eax / if (%eax == 0x0)
3213 je 0f / return (1);
3214 movl $0, %eax / else
3215 ret / return (0);
3216 0: movl $1, %eax
3217 ret
3218 SET_SIZE(panic_trigger)
3219
3220 ENTRY_NP(dtrace_panic_trigger)
3221 movl 4(%esp), %edx / %edx = address of trigger
3222 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3223 lock / assert lock
3224 xchgl %eax, (%edx) / exchange %eax and the trigger
3225 cmpl $0, %eax / if (%eax == 0x0)
3226 je 0f / return (1);
3227 movl $0, %eax / else
3228 ret / return (0);
3229 0: movl $1, %eax
3230 ret
3231 SET_SIZE(dtrace_panic_trigger)
3232
3233 #endif /* __i386 */
3234 #endif /* __lint */
3235
3236 /*
3237 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3238 * into the panic code implemented in panicsys(). vpanic() is responsible
3239 * for passing through the format string and arguments, and constructing a
3240 * regs structure on the stack into which it saves the current register
3241 * values. If we are not dying due to a fatal trap, these registers will
3242 * then be preserved in panicbuf as the current processor state. Before
3243 * invoking panicsys(), vpanic() activates the first panic trigger (see
3244 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3245 * DTrace takes a slightly different panic path if it must panic from probe
3246 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3247 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3248 * branches back into vpanic().
3249 */
3250 #if defined(__lint)
3251
3252 /*ARGSUSED*/
3253 void
3254 vpanic(const char *format, va_list alist)
3255 {}
3256
3257 /*ARGSUSED*/
3258 void
3259 dtrace_vpanic(const char *format, va_list alist)
3260 {}
3261
3262 #else /* __lint */
3263
3264 #if defined(__amd64)
3265
3266 ENTRY_NP(vpanic) /* Initial stack layout: */
3267
3268 pushq %rbp /* | %rip | 0x60 */
3269 movq %rsp, %rbp /* | %rbp | 0x58 */
3270 pushfq /* | rfl | 0x50 */
3271 pushq %r11 /* | %r11 | 0x48 */
3272 pushq %r10 /* | %r10 | 0x40 */
3273 pushq %rbx /* | %rbx | 0x38 */
3274 pushq %rax /* | %rax | 0x30 */
3275 pushq %r9 /* | %r9 | 0x28 */
3276 pushq %r8 /* | %r8 | 0x20 */
3277 pushq %rcx /* | %rcx | 0x18 */
3278 pushq %rdx /* | %rdx | 0x10 */
3279 pushq %rsi /* | %rsi | 0x8 alist */
3280 pushq %rdi /* | %rdi | 0x0 format */
3281
3282 movq %rsp, %rbx /* %rbx = current %rsp */
3283
3284 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3285 call panic_trigger /* %eax = panic_trigger() */
3286
3287 vpanic_common:
3288 /*
3289 * The panic_trigger result is in %eax from the call above, and
3290 * dtrace_panic places it in %eax before branching here.
3291 * The rdmsr instructions that follow below will clobber %eax so
3292 * we stash the panic_trigger result in %r11d.
3293 */
3294 movl %eax, %r11d
3295 cmpl $0, %r11d
3296 je 0f
3297
3298 /*
3299 * If panic_trigger() was successful, we are the first to initiate a
3300 * panic: we now switch to the reserved panic_stack before continuing.
3301 */
3302 leaq panic_stack(%rip), %rsp
3303 addq $PANICSTKSIZE, %rsp
3304 0: subq $REGSIZE, %rsp
3305 /*
3306 * Now that we've got everything set up, store the register values as
3307 * they were when we entered vpanic() to the designated location in
3308 * the regs structure we allocated on the stack.
3309 */
3310 movq 0x0(%rbx), %rcx
3311 movq %rcx, REGOFF_RDI(%rsp)
3312 movq 0x8(%rbx), %rcx
3313 movq %rcx, REGOFF_RSI(%rsp)
3314 movq 0x10(%rbx), %rcx
3315 movq %rcx, REGOFF_RDX(%rsp)
3316 movq 0x18(%rbx), %rcx
3317 movq %rcx, REGOFF_RCX(%rsp)
3318 movq 0x20(%rbx), %rcx
3319
3320 movq %rcx, REGOFF_R8(%rsp)
3321 movq 0x28(%rbx), %rcx
3322 movq %rcx, REGOFF_R9(%rsp)
3323 movq 0x30(%rbx), %rcx
3324 movq %rcx, REGOFF_RAX(%rsp)
3325 movq 0x38(%rbx), %rcx
3326 movq %rcx, REGOFF_RBX(%rsp)
3327 movq 0x58(%rbx), %rcx
3328
3329 movq %rcx, REGOFF_RBP(%rsp)
3330 movq 0x40(%rbx), %rcx
3331 movq %rcx, REGOFF_R10(%rsp)
3332 movq 0x48(%rbx), %rcx
3333 movq %rcx, REGOFF_R11(%rsp)
3334 movq %r12, REGOFF_R12(%rsp)
3335
3336 movq %r13, REGOFF_R13(%rsp)
3337 movq %r14, REGOFF_R14(%rsp)
3338 movq %r15, REGOFF_R15(%rsp)
3339
3340 xorl %ecx, %ecx
3341 movw %ds, %cx
3342 movq %rcx, REGOFF_DS(%rsp)
3343 movw %es, %cx
3344 movq %rcx, REGOFF_ES(%rsp)
3345 movw %fs, %cx
3346 movq %rcx, REGOFF_FS(%rsp)
3347 movw %gs, %cx
3348 movq %rcx, REGOFF_GS(%rsp)
3349
3350 movq $0, REGOFF_TRAPNO(%rsp)
3351
3352 movq $0, REGOFF_ERR(%rsp)
3353 leaq vpanic(%rip), %rcx
3354 movq %rcx, REGOFF_RIP(%rsp)
3355 movw %cs, %cx
3356 movzwq %cx, %rcx
3357 movq %rcx, REGOFF_CS(%rsp)
3358 movq 0x50(%rbx), %rcx
3359 movq %rcx, REGOFF_RFL(%rsp)
3360 movq %rbx, %rcx
3361 addq $0x60, %rcx
3362 movq %rcx, REGOFF_RSP(%rsp)
3363 movw %ss, %cx
3364 movzwq %cx, %rcx
3365 movq %rcx, REGOFF_SS(%rsp)
3366
3367 /*
3368 * panicsys(format, alist, rp, on_panic_stack)
3369 */
3370 movq REGOFF_RDI(%rsp), %rdi /* format */
3371 movq REGOFF_RSI(%rsp), %rsi /* alist */
3372 movq %rsp, %rdx /* struct regs */
3373 movl %r11d, %ecx /* on_panic_stack */
3374 call panicsys
3375 addq $REGSIZE, %rsp
3376 popq %rdi
3377 popq %rsi
3378 popq %rdx
3379 popq %rcx
3380 popq %r8
3381 popq %r9
3382 popq %rax
3383 popq %rbx
3384 popq %r10
3385 popq %r11
3386 popfq
3387 leave
3388 ret
3389 SET_SIZE(vpanic)
3390
3391 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3392
3393 pushq %rbp /* | %rip | 0x60 */
3394 movq %rsp, %rbp /* | %rbp | 0x58 */
3395 pushfq /* | rfl | 0x50 */
3396 pushq %r11 /* | %r11 | 0x48 */
3397 pushq %r10 /* | %r10 | 0x40 */
3398 pushq %rbx /* | %rbx | 0x38 */
3399 pushq %rax /* | %rax | 0x30 */
3400 pushq %r9 /* | %r9 | 0x28 */
3401 pushq %r8 /* | %r8 | 0x20 */
3402 pushq %rcx /* | %rcx | 0x18 */
3403 pushq %rdx /* | %rdx | 0x10 */
3404 pushq %rsi /* | %rsi | 0x8 alist */
3405 pushq %rdi /* | %rdi | 0x0 format */
3406
3407 movq %rsp, %rbx /* %rbx = current %rsp */
3408
3409 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3410 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3411 jmp vpanic_common
3412
3413 SET_SIZE(dtrace_vpanic)
3414
3415 #elif defined(__i386)
3416
3417 ENTRY_NP(vpanic) / Initial stack layout:
3418
3419 pushl %ebp / | %eip | 20
3420 movl %esp, %ebp / | %ebp | 16
3421 pushl %eax / | %eax | 12
3422 pushl %ebx / | %ebx | 8
3423 pushl %ecx / | %ecx | 4
3424 pushl %edx / | %edx | 0
3425
3426 movl %esp, %ebx / %ebx = current stack pointer
3427
3428 lea panic_quiesce, %eax / %eax = &panic_quiesce
3429 pushl %eax / push &panic_quiesce
3430 call panic_trigger / %eax = panic_trigger()
3431 addl $4, %esp / reset stack pointer
3432
3433 vpanic_common:
3434 cmpl $0, %eax / if (%eax == 0)
3435 je 0f / goto 0f;
3436
3437 /*
3438 * If panic_trigger() was successful, we are the first to initiate a
3439 * panic: we now switch to the reserved panic_stack before continuing.
3440 */
3441 lea panic_stack, %esp / %esp = panic_stack
3442 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3443
3444 0: subl $REGSIZE, %esp / allocate struct regs
3445
3446 /*
3447 * Now that we've got everything set up, store the register values as
3448 * they were when we entered vpanic() to the designated location in
3449 * the regs structure we allocated on the stack.
3450 */
3451 #if !defined(__GNUC_AS__)
3452 movw %gs, %edx
3453 movl %edx, REGOFF_GS(%esp)
3454 movw %fs, %edx
3455 movl %edx, REGOFF_FS(%esp)
3456 movw %es, %edx
3457 movl %edx, REGOFF_ES(%esp)
3458 movw %ds, %edx
3459 movl %edx, REGOFF_DS(%esp)
3460 #else /* __GNUC_AS__ */
3461 mov %gs, %edx
3462 mov %edx, REGOFF_GS(%esp)
3463 mov %fs, %edx
3464 mov %edx, REGOFF_FS(%esp)
3465 mov %es, %edx
3466 mov %edx, REGOFF_ES(%esp)
3467 mov %ds, %edx
3468 mov %edx, REGOFF_DS(%esp)
3469 #endif /* __GNUC_AS__ */
3470 movl %edi, REGOFF_EDI(%esp)
3471 movl %esi, REGOFF_ESI(%esp)
3472 movl 16(%ebx), %ecx
3473 movl %ecx, REGOFF_EBP(%esp)
3474 movl %ebx, %ecx
3475 addl $20, %ecx
3476 movl %ecx, REGOFF_ESP(%esp)
3477 movl 8(%ebx), %ecx
3478 movl %ecx, REGOFF_EBX(%esp)
3479 movl 0(%ebx), %ecx
3480 movl %ecx, REGOFF_EDX(%esp)
3481 movl 4(%ebx), %ecx
3482 movl %ecx, REGOFF_ECX(%esp)
3483 movl 12(%ebx), %ecx
3484 movl %ecx, REGOFF_EAX(%esp)
3485 movl $0, REGOFF_TRAPNO(%esp)
3486 movl $0, REGOFF_ERR(%esp)
3487 lea vpanic, %ecx
3488 movl %ecx, REGOFF_EIP(%esp)
3489 #if !defined(__GNUC_AS__)
3490 movw %cs, %edx
3491 #else /* __GNUC_AS__ */
3492 mov %cs, %edx
3493 #endif /* __GNUC_AS__ */
3494 movl %edx, REGOFF_CS(%esp)
3495 pushfl
3496 popl %ecx
3497 #if defined(__xpv)
3498 /*
3499 * Synthesize the PS_IE bit from the event mask bit
3500 */
3501 CURTHREAD(%edx)
3502 KPREEMPT_DISABLE(%edx)
3503 EVENT_MASK_TO_IE(%edx, %ecx)
3504 CURTHREAD(%edx)
3505 KPREEMPT_ENABLE_NOKP(%edx)
3506 #endif
3507 movl %ecx, REGOFF_EFL(%esp)
3508 movl $0, REGOFF_UESP(%esp)
3509 #if !defined(__GNUC_AS__)
3510 movw %ss, %edx
3511 #else /* __GNUC_AS__ */
3512 mov %ss, %edx
3513 #endif /* __GNUC_AS__ */
3514 movl %edx, REGOFF_SS(%esp)
3515
3516 movl %esp, %ecx / %ecx = &regs
3517 pushl %eax / push on_panic_stack
3518 pushl %ecx / push &regs
3519 movl 12(%ebp), %ecx / %ecx = alist
3520 pushl %ecx / push alist
3521 movl 8(%ebp), %ecx / %ecx = format
3522 pushl %ecx / push format
3523 call panicsys / panicsys();
3524 addl $16, %esp / pop arguments
3525
3526 addl $REGSIZE, %esp
3527 popl %edx
3528 popl %ecx
3529 popl %ebx
3530 popl %eax
3531 leave
3532 ret
3533 SET_SIZE(vpanic)
3534
3535 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3536
3537 pushl %ebp / | %eip | 20
3538 movl %esp, %ebp / | %ebp | 16
3539 pushl %eax / | %eax | 12
3540 pushl %ebx / | %ebx | 8
3541 pushl %ecx / | %ecx | 4
3542 pushl %edx / | %edx | 0
3543
3544 movl %esp, %ebx / %ebx = current stack pointer
3545
3546 lea panic_quiesce, %eax / %eax = &panic_quiesce
3547 pushl %eax / push &panic_quiesce
3548 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3549 addl $4, %esp / reset stack pointer
3550 jmp vpanic_common / jump back to common code
3551
3552 SET_SIZE(dtrace_vpanic)
3553
3554 #endif /* __i386 */
3555 #endif /* __lint */
3556
3557 #if defined(__lint)
3558
3559 void
3560 hres_tick(void)
3561 {}
3562
3563 int64_t timedelta;
3564 hrtime_t hres_last_tick;
3565 volatile timestruc_t hrestime;
3566 int64_t hrestime_adj;
3567 volatile int hres_lock;
3568 hrtime_t hrtime_base;
3569
3570 #else /* __lint */
3571
3572 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8)
3573 .NWORD 0, 0
3574
3575 DGDEF3(hrestime_adj, 8, 8)
3576 .long 0, 0
3577
3578 DGDEF3(hres_last_tick, 8, 8)
3579 .long 0, 0
3580
3581 DGDEF3(timedelta, 8, 8)
3582 .long 0, 0
3583
3584 DGDEF3(hres_lock, 4, 8)
3585 .long 0
3586
3587 /*
3588 * initialized to a non zero value to make pc_gethrtime()
3589 * work correctly even before clock is initialized
3590 */
3591 DGDEF3(hrtime_base, 8, 8)
3592 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3593
3594 DGDEF3(adj_shift, 4, 4)
3595 .long ADJ_SHIFT
3596
3597 #if defined(__amd64)
3598
3599 ENTRY_NP(hres_tick)
3600 pushq %rbp
3601 movq %rsp, %rbp
3602
3603 /*
3604 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3605 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3606 * At worst, performing this now instead of under CLOCK_LOCK may
3607 * introduce some jitter in pc_gethrestime().
3608 */
3609 call *gethrtimef(%rip)
3610 movq %rax, %r8
3611
3612 leaq hres_lock(%rip), %rax
3613 movb $-1, %dl
3614 .CL1:
3615 xchgb %dl, (%rax)
3616 testb %dl, %dl
3617 jz .CL3 /* got it */
3618 .CL2:
3619 cmpb $0, (%rax) /* possible to get lock? */
3620 pause
3621 jne .CL2
3622 jmp .CL1 /* yes, try again */
3623 .CL3:
3624 /*
3625 * compute the interval since last time hres_tick was called
3626 * and adjust hrtime_base and hrestime accordingly
3627 * hrtime_base is an 8 byte value (in nsec), hrestime is
3628 * a timestruc_t (sec, nsec)
3629 */
3630 leaq hres_last_tick(%rip), %rax
3631 movq %r8, %r11
3632 subq (%rax), %r8
3633 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3634 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3635 /*
3636 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3637 */
3638 movq %r11, (%rax)
3639
3640 call __adj_hrestime
3641
3642 /*
3643 * release the hres_lock
3644 */
3645 incl hres_lock(%rip)
3646 leave
3647 ret
3648 SET_SIZE(hres_tick)
3649
3650 #elif defined(__i386)
3651
3652 ENTRY_NP(hres_tick)
3653 pushl %ebp
3654 movl %esp, %ebp
3655 pushl %esi
3656 pushl %ebx
3657
3658 /*
3659 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3660 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3661 * At worst, performing this now instead of under CLOCK_LOCK may
3662 * introduce some jitter in pc_gethrestime().
3663 */
3664 call *gethrtimef
3665 movl %eax, %ebx
3666 movl %edx, %esi
3667
3668 movl $hres_lock, %eax
3669 movl $-1, %edx
3670 .CL1:
3671 xchgb %dl, (%eax)
3672 testb %dl, %dl
3673 jz .CL3 / got it
3674 .CL2:
3675 cmpb $0, (%eax) / possible to get lock?
3676 pause
3677 jne .CL2
3678 jmp .CL1 / yes, try again
3679 .CL3:
3680 /*
3681 * compute the interval since last time hres_tick was called
3682 * and adjust hrtime_base and hrestime accordingly
3683 * hrtime_base is an 8 byte value (in nsec), hrestime is
3684 * timestruc_t (sec, nsec)
3685 */
3686
3687 lea hres_last_tick, %eax
3688
3689 movl %ebx, %edx
3690 movl %esi, %ecx
3691
3692 subl (%eax), %edx
3693 sbbl 4(%eax), %ecx
3694
3695 addl %edx, hrtime_base / add interval to hrtime_base
3696 adcl %ecx, hrtime_base+4
3697
3698 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3699
3700 /
3701 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3702 /
3703 movl %ebx, (%eax)
3704 movl %esi, 4(%eax)
3705
3706 / get hrestime at this moment. used as base for pc_gethrestime
3707 /
3708 / Apply adjustment, if any
3709 /
3710 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3711 / (max_hres_adj)
3712 /
3713 / void
3714 / adj_hrestime()
3715 / {
3716 / long long adj;
3717 /
3718 / if (hrestime_adj == 0)
3719 / adj = 0;
3720 / else if (hrestime_adj > 0) {
3721 / if (hrestime_adj < HRES_ADJ)
3722 / adj = hrestime_adj;
3723 / else
3724 / adj = HRES_ADJ;
3725 / }
3726 / else {
3727 / if (hrestime_adj < -(HRES_ADJ))
3728 / adj = -(HRES_ADJ);
3729 / else
3730 / adj = hrestime_adj;
3731 / }
3732 /
3733 / timedelta -= adj;
3734 / hrestime_adj = timedelta;
3735 / hrestime.tv_nsec += adj;
3736 /
3737 / while (hrestime.tv_nsec >= NANOSEC) {
3738 / one_sec++;
3739 / hrestime.tv_sec++;
3740 / hrestime.tv_nsec -= NANOSEC;
3741 / }
3742 / }
3743 __adj_hrestime:
3744 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3745 movl hrestime_adj+4, %edx
3746 andl %esi, %esi
3747 jne .CL4 / no
3748 andl %edx, %edx
3749 jne .CL4 / no
3750 subl %ecx, %ecx / yes, adj = 0;
3751 subl %edx, %edx
3752 jmp .CL5
3753 .CL4:
3754 subl %ecx, %ecx
3755 subl %eax, %eax
3756 subl %esi, %ecx
3757 sbbl %edx, %eax
3758 andl %eax, %eax / if (hrestime_adj > 0)
3759 jge .CL6
3760
3761 / In the following comments, HRES_ADJ is used, while in the code
3762 / max_hres_adj is used.
3763 /
3764 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3765 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3766 / on the logical equivalence of:
3767 /
3768 / !(hrestime_adj < HRES_ADJ)
3769 /
3770 / and the two step sequence:
3771 /
3772 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3773 /
3774 / which computes whether or not the least significant 32-bits
3775 / of hrestime_adj is greater than HRES_ADJ, followed by:
3776 /
3777 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3778 /
3779 / which generates a carry whenever step 1 is true or the most
3780 / significant long of the longlong hrestime_adj is non-zero.
3781
3782 movl max_hres_adj, %ecx / hrestime_adj is positive
3783 subl %esi, %ecx
3784 movl %edx, %eax
3785 adcl $-1, %eax
3786 jnc .CL7
3787 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3788 subl %edx, %edx
3789 jmp .CL5
3790
3791 / The following computation is similar to the one above.
3792 /
3793 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3794 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3795 / on the logical equivalence of:
3796 /
3797 / (hrestime_adj > -HRES_ADJ)
3798 /
3799 / and the two step sequence:
3800 /
3801 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3802 /
3803 / which means the least significant 32-bits of hrestime_adj is
3804 / greater than -HRES_ADJ, followed by:
3805 /
3806 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3807 /
3808 / which generates a carry only when step 1 is true and the most
3809 / significant long of the longlong hrestime_adj is -1.
3810
3811 .CL6: / hrestime_adj is negative
3812 movl %esi, %ecx
3813 addl max_hres_adj, %ecx
3814 movl %edx, %eax
3815 adcl $0, %eax
3816 jc .CL7
3817 xor %ecx, %ecx
3818 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3819 movl $-1, %edx
3820 jmp .CL5
3821 .CL7:
3822 movl %esi, %ecx / adj = hrestime_adj;
3823 .CL5:
3824 movl timedelta, %esi
3825 subl %ecx, %esi
3826 movl timedelta+4, %eax
3827 sbbl %edx, %eax
3828 movl %esi, timedelta
3829 movl %eax, timedelta+4 / timedelta -= adj;
3830 movl %esi, hrestime_adj
3831 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3832 addl hrestime+4, %ecx
3833
3834 movl %ecx, %eax / eax = tv_nsec
3835 1:
3836 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3837 jb .CL8 / no
3838 incl one_sec / yes, one_sec++;
3839 incl hrestime / hrestime.tv_sec++;
3840 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3841 jmp 1b / check for more seconds
3842
3843 .CL8:
3844 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3845 incl hres_lock / release the hres_lock
3846
3847 popl %ebx
3848 popl %esi
3849 leave
3850 ret
3851 SET_SIZE(hres_tick)
3852
3853 #endif /* __i386 */
3854 #endif /* __lint */
3855
3856 /*
3857 * void prefetch_smap_w(void *)
3858 *
3859 * Prefetch ahead within a linear list of smap structures.
3860 * Not implemented for ia32. Stub for compatibility.
3861 */
3862
3863 #if defined(__lint)
3864
3865 /*ARGSUSED*/
3866 void prefetch_smap_w(void *smp)
3867 {}
3868
3869 #else /* __lint */
3870
3871 ENTRY(prefetch_smap_w)
3872 rep; ret /* use 2 byte return instruction when branch target */
3873 /* AMD Software Optimization Guide - Section 6.2 */
3874 SET_SIZE(prefetch_smap_w)
3875
3876 #endif /* __lint */
3877
3878 /*
3879 * prefetch_page_r(page_t *)
3880 * issue prefetch instructions for a page_t
3881 */
3882 #if defined(__lint)
3883
3884 /*ARGSUSED*/
3885 void
3886 prefetch_page_r(void *pp)
3887 {}
3888
3889 #else /* __lint */
3890
3891 ENTRY(prefetch_page_r)
3892 rep; ret /* use 2 byte return instruction when branch target */
3893 /* AMD Software Optimization Guide - Section 6.2 */
3894 SET_SIZE(prefetch_page_r)
3895
3896 #endif /* __lint */
3897
3898 #if defined(__lint)
3899
3900 /*ARGSUSED*/
3901 int
3902 bcmp(const void *s1, const void *s2, size_t count)
3903 { return (0); }
3904
3905 #else /* __lint */
3906
3907 #if defined(__amd64)
3908
3909 ENTRY(bcmp)
3910 pushq %rbp
3911 movq %rsp, %rbp
3912 #ifdef DEBUG
3913 testq %rdx,%rdx
3914 je 1f
3915 movq postbootkernelbase(%rip), %r11
3916 cmpq %r11, %rdi
3917 jb 0f
3918 cmpq %r11, %rsi
3919 jnb 1f
3920 0: leaq .bcmp_panic_msg(%rip), %rdi
3921 xorl %eax, %eax
3922 call panic
3923 1:
3924 #endif /* DEBUG */
3925 call memcmp
3926 testl %eax, %eax
3927 setne %dl
3928 leave
3929 movzbl %dl, %eax
3930 ret
3931 SET_SIZE(bcmp)
3932
3933 #elif defined(__i386)
3934
3935 #define ARG_S1 8
3936 #define ARG_S2 12
3937 #define ARG_LENGTH 16
3938
3939 ENTRY(bcmp)
3940 pushl %ebp
3941 movl %esp, %ebp / create new stack frame
3942 #ifdef DEBUG
3943 cmpl $0, ARG_LENGTH(%ebp)
3944 je 1f
3945 movl postbootkernelbase, %eax
3946 cmpl %eax, ARG_S1(%ebp)
3947 jb 0f
3948 cmpl %eax, ARG_S2(%ebp)
3949 jnb 1f
3950 0: pushl $.bcmp_panic_msg
3951 call panic
3952 1:
3953 #endif /* DEBUG */
3954
3955 pushl %edi / save register variable
3956 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3957 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3958 cmpl %eax, %ecx / if the same string
3959 je .equal / goto .equal
3960 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3961 cmpl $4, %edi / if %edi < 4
3962 jb .byte_check / goto .byte_check
3963 .align 4
3964 .word_loop:
3965 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3966 leal -4(%edi), %edi / %edi -= 4
3967 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3968 jne .word_not_equal / if not equal, goto .word_not_equal
3969 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3970 leal 4(%eax), %eax / %eax += 4 (next word)
3971 cmpl $4, %edi / if %edi >= 4
3972 jae .word_loop / goto .word_loop
3973 .byte_check:
3974 cmpl $0, %edi / if %edi == 0
3975 je .equal / goto .equal
3976 jmp .byte_loop / goto .byte_loop (checks in bytes)
3977 .word_not_equal:
3978 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3979 .align 4
3980 .byte_loop:
3981 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3982 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3983 jne .not_equal / if not equal, goto .not_equal
3984 incl %ecx / %ecx++ (next byte)
3985 incl %eax / %eax++ (next byte)
3986 decl %edi / %edi--
3987 jnz .byte_loop / if not zero, goto .byte_loop
3988 .equal:
3989 xorl %eax, %eax / %eax = 0
3990 popl %edi / restore register variable
3991 leave / restore old stack frame
3992 ret / return (NULL)
3993 .align 4
3994 .not_equal:
3995 movl $1, %eax / return 1
3996 popl %edi / restore register variable
3997 leave / restore old stack frame
3998 ret / return (NULL)
3999 SET_SIZE(bcmp)
4000
4001 #endif /* __i386 */
4002
4003 #ifdef DEBUG
4004 .text
4005 .bcmp_panic_msg:
4006 .string "bcmp: arguments below kernelbase"
4007 #endif /* DEBUG */
4008
4009 #endif /* __lint */
4010
4011 #if defined(__lint)
4012
4013 uint_t
4014 bsrw_insn(uint16_t mask)
4015 {
4016 uint_t index = sizeof (mask) * NBBY - 1;
4017
4018 while ((mask & (1 << index)) == 0)
4019 index--;
4020 return (index);
4021 }
4022
4023 #else /* __lint */
4024
4025 #if defined(__amd64)
4026
4027 ENTRY_NP(bsrw_insn)
4028 xorl %eax, %eax
4029 bsrw %di, %ax
4030 ret
4031 SET_SIZE(bsrw_insn)
4032
4033 #elif defined(__i386)
4034
4035 ENTRY_NP(bsrw_insn)
4036 movw 4(%esp), %cx
4037 xorl %eax, %eax
4038 bsrw %cx, %ax
4039 ret
4040 SET_SIZE(bsrw_insn)
4041
4042 #endif /* __i386 */
4043 #endif /* __lint */
4044
4045 #if defined(__lint)
4046
4047 uint_t
4048 atomic_btr32(uint32_t *pending, uint_t pil)
4049 {
4050 return (*pending &= ~(1 << pil));
4051 }
4052
4053 #else /* __lint */
4054
4055 #if defined(__i386)
4056
4057 ENTRY_NP(atomic_btr32)
4058 movl 4(%esp), %ecx
4059 movl 8(%esp), %edx
4060 xorl %eax, %eax
4061 lock
4062 btrl %edx, (%ecx)
4063 setc %al
4064 ret
4065 SET_SIZE(atomic_btr32)
4066
4067 #endif /* __i386 */
4068 #endif /* __lint */
4069
4070 #if defined(__lint)
4071
4072 /*ARGSUSED*/
4073 void
4074 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4075 uint_t arg2)
4076 {}
4077
4078 #else /* __lint */
4079
4080 #if defined(__amd64)
4081
4082 ENTRY_NP(switch_sp_and_call)
4083 pushq %rbp
4084 movq %rsp, %rbp /* set up stack frame */
4085 movq %rdi, %rsp /* switch stack pointer */
4086 movq %rdx, %rdi /* pass func arg 1 */
4087 movq %rsi, %r11 /* save function to call */
4088 movq %rcx, %rsi /* pass func arg 2 */
4089 call *%r11 /* call function */
4090 leave /* restore stack */
4091 ret
4092 SET_SIZE(switch_sp_and_call)
4093
4094 #elif defined(__i386)
4095
4096 ENTRY_NP(switch_sp_and_call)
4097 pushl %ebp
4098 mov %esp, %ebp /* set up stack frame */
4099 movl 8(%ebp), %esp /* switch stack pointer */
4100 pushl 20(%ebp) /* push func arg 2 */
4101 pushl 16(%ebp) /* push func arg 1 */
4102 call *12(%ebp) /* call function */
4103 addl $8, %esp /* pop arguments */
4104 leave /* restore stack */
4105 ret
4106 SET_SIZE(switch_sp_and_call)
4107
4108 #endif /* __i386 */
4109 #endif /* __lint */
4110
4111 #if defined(__lint)
4112
4113 void
4114 kmdb_enter(void)
4115 {}
4116
4117 #else /* __lint */
4118
4119 #if defined(__amd64)
4120
4121 ENTRY_NP(kmdb_enter)
4122 pushq %rbp
4123 movq %rsp, %rbp
4124
4125 /*
4126 * Save flags, do a 'cli' then return the saved flags
4127 */
4128 call intr_clear
4129
4130 int $T_DBGENTR
4131
4132 /*
4133 * Restore the saved flags
4134 */
4135 movq %rax, %rdi
4136 call intr_restore
4137
4138 leave
4139 ret
4140 SET_SIZE(kmdb_enter)
4141
4142 #elif defined(__i386)
4143
4144 ENTRY_NP(kmdb_enter)
4145 pushl %ebp
4146 movl %esp, %ebp
4147
4148 /*
4149 * Save flags, do a 'cli' then return the saved flags
4150 */
4151 call intr_clear
4152
4153 int $T_DBGENTR
4154
4155 /*
4156 * Restore the saved flags
4157 */
4158 pushl %eax
4159 call intr_restore
4160 addl $4, %esp
4161
4162 leave
4163 ret
4164 SET_SIZE(kmdb_enter)
4165
4166 #endif /* __i386 */
4167 #endif /* __lint */
4168
4169 #if defined(__lint)
4170
4171 void
4172 return_instr(void)
4173 {}
4174
4175 #else /* __lint */
4176
4177 ENTRY_NP(return_instr)
4178 rep; ret /* use 2 byte instruction when branch target */
4179 /* AMD Software Optimization Guide - Section 6.2 */
4180 SET_SIZE(return_instr)
4181
4182 #endif /* __lint */
4183
4184 #if defined(__lint)
4185
4186 ulong_t
4187 getflags(void)
4188 {
4189 return (0);
4190 }
4191
4192 #else /* __lint */
4193
4194 #if defined(__amd64)
4195
4196 ENTRY(getflags)
4197 pushfq
4198 popq %rax
4199 #if defined(__xpv)
4200 CURTHREAD(%rdi)
4201 KPREEMPT_DISABLE(%rdi)
4202 /*
4203 * Synthesize the PS_IE bit from the event mask bit
4204 */
4205 CURVCPU(%r11)
4206 andq $_BITNOT(PS_IE), %rax
4207 XEN_TEST_UPCALL_MASK(%r11)
4208 jnz 1f
4209 orq $PS_IE, %rax
4210 1:
4211 KPREEMPT_ENABLE_NOKP(%rdi)
4212 #endif
4213 ret
4214 SET_SIZE(getflags)
4215
4216 #elif defined(__i386)
4217
4218 ENTRY(getflags)
4219 pushfl
4220 popl %eax
4221 #if defined(__xpv)
4222 CURTHREAD(%ecx)
4223 KPREEMPT_DISABLE(%ecx)
4224 /*
4225 * Synthesize the PS_IE bit from the event mask bit
4226 */
4227 CURVCPU(%edx)
4228 andl $_BITNOT(PS_IE), %eax
4229 XEN_TEST_UPCALL_MASK(%edx)
4230 jnz 1f
4231 orl $PS_IE, %eax
4232 1:
4233 KPREEMPT_ENABLE_NOKP(%ecx)
4234 #endif
4235 ret
4236 SET_SIZE(getflags)
4237
4238 #endif /* __i386 */
4239
4240 #endif /* __lint */
4241
4242 #if defined(__lint)
4243
4244 ftrace_icookie_t
4245 ftrace_interrupt_disable(void)
4246 { return (0); }
4247
4248 #else /* __lint */
4249
4250 #if defined(__amd64)
4251
4252 ENTRY(ftrace_interrupt_disable)
4253 pushfq
4254 popq %rax
4255 CLI(%rdx)
4256 ret
4257 SET_SIZE(ftrace_interrupt_disable)
4258
4259 #elif defined(__i386)
4260
4261 ENTRY(ftrace_interrupt_disable)
4262 pushfl
4263 popl %eax
4264 CLI(%edx)
4265 ret
4266 SET_SIZE(ftrace_interrupt_disable)
4267
4268 #endif /* __i386 */
4269 #endif /* __lint */
4270
4271 #if defined(__lint)
4272
4273 /*ARGSUSED*/
4274 void
4275 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4276 {}
4277
4278 #else /* __lint */
4279
4280 #if defined(__amd64)
4281
4282 ENTRY(ftrace_interrupt_enable)
4283 pushq %rdi
4284 popfq
4285 ret
4286 SET_SIZE(ftrace_interrupt_enable)
4287
4288 #elif defined(__i386)
4289
4290 ENTRY(ftrace_interrupt_enable)
4291 movl 4(%esp), %eax
4292 pushl %eax
4293 popfl
4294 ret
4295 SET_SIZE(ftrace_interrupt_enable)
4296
4297 #endif /* __i386 */
4298 #endif /* __lint */
4299
4300 #if defined (__lint)
4301
4302 /*ARGSUSED*/
4303 void
4304 clflush_insn(caddr_t addr)
4305 {}
4306
4307 #else /* __lint */
4308
4309 #if defined (__amd64)
4310 ENTRY(clflush_insn)
4311 clflush (%rdi)
4312 ret
4313 SET_SIZE(clflush_insn)
4314 #elif defined (__i386)
4315 ENTRY(clflush_insn)
4316 movl 4(%esp), %eax
4317 clflush (%eax)
4318 ret
4319 SET_SIZE(clflush_insn)
4320
4321 #endif /* __i386 */
4322 #endif /* __lint */
4323
4324 #if defined (__lint)
4325 /*ARGSUSED*/
4326 void
4327 mfence_insn(void)
4328 {}
4329
4330 #else /* __lint */
4331
4332 #if defined (__amd64)
4333 ENTRY(mfence_insn)
4334 mfence
4335 ret
4336 SET_SIZE(mfence_insn)
4337 #elif defined (__i386)
4338 ENTRY(mfence_insn)
4339 mfence
4340 ret
4341 SET_SIZE(mfence_insn)
4342
4343 #endif /* __i386 */
4344 #endif /* __lint */
4345
4346 /*
4347 * VMware implements an I/O port that programs can query to detect if software
4348 * is running in a VMware hypervisor. This hypervisor port behaves differently
4349 * depending on magic values in certain registers and modifies some registers
4350 * as a side effect.
4351 *
4352 * References: http://kb.vmware.com/kb/1009458
4353 */
4354
4355 #if defined(__lint)
4356
4357 /* ARGSUSED */
4358 void
4359 vmware_port(int cmd, uint32_t *regs) { return; }
4360
4361 #else
4362
4363 #if defined(__amd64)
4364
4365 ENTRY(vmware_port)
4366 pushq %rbx
4367 movl $VMWARE_HVMAGIC, %eax
4368 movl $0xffffffff, %ebx
4369 movl %edi, %ecx
4370 movl $VMWARE_HVPORT, %edx
4371 inl (%dx)
4372 movl %eax, (%rsi)
4373 movl %ebx, 4(%rsi)
4374 movl %ecx, 8(%rsi)
4375 movl %edx, 12(%rsi)
4376 popq %rbx
4377 ret
4378 SET_SIZE(vmware_port)
4379
4380 #elif defined(__i386)
4381
4382 ENTRY(vmware_port)
4383 pushl %ebx
4384 pushl %esi
4385 movl $VMWARE_HVMAGIC, %eax
4386 movl $0xffffffff, %ebx
4387 movl 12(%esp), %ecx
4388 movl $VMWARE_HVPORT, %edx
4389 inl (%dx)
4390 movl 16(%esp), %esi
4391 movl %eax, (%esi)
4392 movl %ebx, 4(%esi)
4393 movl %ecx, 8(%esi)
4394 movl %edx, 12(%esi)
4395 popl %esi
4396 popl %ebx
4397 ret
4398 SET_SIZE(vmware_port)
4399
4400 #endif /* __i386 */
4401 #endif /* __lint */
Unleashed OS