?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Unleashed v1.4
[unleashed.git] / arch / x86 / kernel / ml / i86_subr.s
blob42e7bda40d8cf05888623913582afcf135c7d879
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 * Copyright 2018 Joyent, Inc.
27 */
28
29 /*
30 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
32 * All Rights Reserved
33 */
34
35 /*
36 * Copyright (c) 2009, Intel Corporation.
37 * All rights reserved.
38 */
39
40 /*
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
48 */
49
50 #include <sys/asm_linkage.h>
51 #include <sys/asm_misc.h>
52 #include <sys/panic.h>
53 #include <sys/ontrap.h>
54 #include <sys/regset.h>
55 #include <sys/privregs.h>
56 #include <sys/reboot.h>
57 #include <sys/psw.h>
58 #include <sys/x86_archext.h>
59
60 #include "assym.h"
61 #include <sys/dditypes.h>
62
63 /*
64 * on_fault()
65 *
66 * Catch lofault faults. Like setjmp except it returns one
67 * if code following causes uncorrectable fault. Turned off
68 * by calling no_fault(). Note that while under on_fault(),
69 * SMAP is disabled. For more information see
70 * uts/intel/ia32/ml/copy.s.
71 */
72
73
74 #if defined(__amd64)
75
76 ENTRY(on_fault)
77 movq %gs:CPU_THREAD, %rsi
78 leaq catch_fault(%rip), %rdx
79 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
80 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
81 call smap_disable /* allow user accesses */
82 jmp setjmp /* let setjmp do the rest */
83
84 catch_fault:
85 movq %gs:CPU_THREAD, %rsi
86 movq T_ONFAULT(%rsi), %rdi /* address of save area */
87 xorl %eax, %eax
88 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
89 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
90 call smap_enable /* disallow user accesses */
91 jmp longjmp /* let longjmp do the rest */
92 SET_SIZE(on_fault)
93
94 ENTRY(no_fault)
95 movq %gs:CPU_THREAD, %rsi
96 xorl %eax, %eax
97 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
98 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
99 call smap_enable /* disallow user accesses */
100 ret
101 SET_SIZE(no_fault)
102
103 #elif defined(__i386)
104
105 ENTRY(on_fault)
106 movl %gs:CPU_THREAD, %edx
107 movl 4(%esp), %eax /* jumpbuf address */
108 leal catch_fault, %ecx
109 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
110 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
111 jmp setjmp /* let setjmp do the rest */
112
113 catch_fault:
114 movl %gs:CPU_THREAD, %edx
115 xorl %eax, %eax
116 movl T_ONFAULT(%edx), %ecx /* address of save area */
117 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
118 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
119 pushl %ecx
120 call longjmp /* let longjmp do the rest */
121 SET_SIZE(on_fault)
122
123 ENTRY(no_fault)
124 movl %gs:CPU_THREAD, %edx
125 xorl %eax, %eax
126 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
127 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
128 ret
129 SET_SIZE(no_fault)
130
131 #endif /* __i386 */
132
133 /*
134 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
135 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
136 */
137
138
139 #if defined(__amd64)
140
141 ENTRY(on_trap_trampoline)
142 movq %gs:CPU_THREAD, %rsi
143 movq T_ONTRAP(%rsi), %rdi
144 addq $OT_JMPBUF, %rdi
145 jmp longjmp
146 SET_SIZE(on_trap_trampoline)
147
148 #elif defined(__i386)
149
150 ENTRY(on_trap_trampoline)
151 movl %gs:CPU_THREAD, %eax
152 movl T_ONTRAP(%eax), %eax
153 addl $OT_JMPBUF, %eax
154 pushl %eax
155 call longjmp
156 SET_SIZE(on_trap_trampoline)
157
158 #endif /* __i386 */
159
160 /*
161 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
162 * more information about the on_trap() mechanism. If the on_trap_data is the
163 * same as the topmost stack element, we just modify that element.
164 */
165
166 #if defined(__amd64)
167
168 ENTRY(on_trap)
169 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
170 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
171 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
172 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
173 xorl %ecx, %ecx
174 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
175 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
176 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
177 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
178 cmpq %rdi, %rcx /* if (otp == %rcx) */
179 je 0f /* don't modify t_ontrap */
180
181 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
182 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
183
184 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
185 jmp setjmp
186 SET_SIZE(on_trap)
187
188 #elif defined(__i386)
189
190 ENTRY(on_trap)
191 movl 4(%esp), %eax /* %eax = otp */
192 movl 8(%esp), %edx /* %edx = prot */
193
194 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
195 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
196 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
197 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
198 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
199 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
200 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
201 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
202 cmpl %eax, %ecx /* if (otp == %ecx) */
203 je 0f /* don't modify t_ontrap */
204
205 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
206 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
207
208 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
209 movl %eax, 4(%esp) /* put %eax back on the stack */
210 jmp setjmp /* let setjmp do the rest */
211 SET_SIZE(on_trap)
212
213 #endif /* __i386 */
214
215 /*
216 * Setjmp and longjmp implement non-local gotos using state vectors
217 * type label_t.
218 */
219
220
221 #if LABEL_PC != 0
222 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
223 #endif /* LABEL_PC != 0 */
224
225 #if defined(__amd64)
226
227 ENTRY(setjmp)
228 movq %rsp, LABEL_SP(%rdi)
229 movq %rbp, LABEL_RBP(%rdi)
230 movq %rbx, LABEL_RBX(%rdi)
231 movq %r12, LABEL_R12(%rdi)
232 movq %r13, LABEL_R13(%rdi)
233 movq %r14, LABEL_R14(%rdi)
234 movq %r15, LABEL_R15(%rdi)
235 movq (%rsp), %rdx /* return address */
236 movq %rdx, (%rdi) /* LABEL_PC is 0 */
237 xorl %eax, %eax /* return 0 */
238 ret
239 SET_SIZE(setjmp)
240
241 ENTRY(longjmp)
242 movq LABEL_SP(%rdi), %rsp
243 movq LABEL_RBP(%rdi), %rbp
244 movq LABEL_RBX(%rdi), %rbx
245 movq LABEL_R12(%rdi), %r12
246 movq LABEL_R13(%rdi), %r13
247 movq LABEL_R14(%rdi), %r14
248 movq LABEL_R15(%rdi), %r15
249 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
250 movq %rdx, (%rsp)
251 xorl %eax, %eax
252 incl %eax /* return 1 */
253 ret
254 SET_SIZE(longjmp)
255
256 #elif defined(__i386)
257
258 ENTRY(setjmp)
259 movl 4(%esp), %edx /* address of save area */
260 movl %ebp, LABEL_EBP(%edx)
261 movl %ebx, LABEL_EBX(%edx)
262 movl %esi, LABEL_ESI(%edx)
263 movl %edi, LABEL_EDI(%edx)
264 movl %esp, 4(%edx)
265 movl (%esp), %ecx /* %eip (return address) */
266 movl %ecx, (%edx) /* LABEL_PC is 0 */
267 subl %eax, %eax /* return 0 */
268 ret
269 SET_SIZE(setjmp)
270
271 ENTRY(longjmp)
272 movl 4(%esp), %edx /* address of save area */
273 movl LABEL_EBP(%edx), %ebp
274 movl LABEL_EBX(%edx), %ebx
275 movl LABEL_ESI(%edx), %esi
276 movl LABEL_EDI(%edx), %edi
277 movl 4(%edx), %esp
278 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
279 movl $1, %eax
280 addl $4, %esp /* pop ret adr */
281 jmp *%ecx /* indirect */
282 SET_SIZE(longjmp)
283
284 #endif /* __i386 */
285
286 /*
287 * if a() calls b() calls caller(),
288 * caller() returns return address in a().
289 * (Note: We assume a() and b() are C routines which do the normal entry/exit
290 * sequence.)
291 */
292
293
294 #if defined(__amd64)
295
296 ENTRY(caller)
297 movq 8(%rbp), %rax /* b()'s return pc, in a() */
298 ret
299 SET_SIZE(caller)
300
301 #elif defined(__i386)
302
303 ENTRY(caller)
304 movl 4(%ebp), %eax /* b()'s return pc, in a() */
305 ret
306 SET_SIZE(caller)
307
308 #endif /* __i386 */
309
310 /*
311 * if a() calls callee(), callee() returns the
312 * return address in a();
313 */
314
315
316 #if defined(__amd64)
317
318 ENTRY(callee)
319 movq (%rsp), %rax /* callee()'s return pc, in a() */
320 ret
321 SET_SIZE(callee)
322
323 #elif defined(__i386)
324
325 ENTRY(callee)
326 movl (%esp), %eax /* callee()'s return pc, in a() */
327 ret
328 SET_SIZE(callee)
329
330 #endif /* __i386 */
331
332 /*
333 * return the current frame pointer
334 */
335
336
337 #if defined(__amd64)
338
339 ENTRY(getfp)
340 movq %rbp, %rax
341 ret
342 SET_SIZE(getfp)
343
344 #elif defined(__i386)
345
346 ENTRY(getfp)
347 movl %ebp, %eax
348 ret
349 SET_SIZE(getfp)
350
351 #endif /* __i386 */
352
353 /*
354 * Invalidate a single page table entry in the TLB
355 */
356
357 ENTRY(mmu_invlpg)
358 invlpg (%rdi)
359 ret
360 SET_SIZE(mmu_invlpg)
361
362
363 /*
364 * Get/Set the value of various control registers
365 */
366
367
368 #if defined(__amd64)
369
370 ENTRY(getcr0)
371 movq %cr0, %rax
372 ret
373 SET_SIZE(getcr0)
374
375 ENTRY(setcr0)
376 movq %rdi, %cr0
377 ret
378 SET_SIZE(setcr0)
379
380 ENTRY(getcr2)
381 #if defined(__xpv)
382 movq %gs:CPU_VCPU_INFO, %rax
383 movq VCPU_INFO_ARCH_CR2(%rax), %rax
384 #else
385 movq %cr2, %rax
386 #endif
387 ret
388 SET_SIZE(getcr2)
389
390 ENTRY(getcr3)
391 movq %cr3, %rax
392 ret
393 SET_SIZE(getcr3)
394
395 #if !defined(__xpv)
396
397 ENTRY(setcr3)
398 movq %rdi, %cr3
399 ret
400 SET_SIZE(setcr3)
401
402 ENTRY(reload_cr3)
403 movq %cr3, %rdi
404 movq %rdi, %cr3
405 ret
406 SET_SIZE(reload_cr3)
407
408 #endif /* __xpv */
409
410 ENTRY(getcr4)
411 movq %cr4, %rax
412 ret
413 SET_SIZE(getcr4)
414
415 ENTRY(setcr4)
416 movq %rdi, %cr4
417 ret
418 SET_SIZE(setcr4)
419
420 ENTRY(getcr8)
421 movq %cr8, %rax
422 ret
423 SET_SIZE(getcr8)
424
425 ENTRY(setcr8)
426 movq %rdi, %cr8
427 ret
428 SET_SIZE(setcr8)
429
430 #elif defined(__i386)
431
432 ENTRY(getcr0)
433 movl %cr0, %eax
434 ret
435 SET_SIZE(getcr0)
436
437 ENTRY(setcr0)
438 movl 4(%esp), %eax
439 movl %eax, %cr0
440 ret
441 SET_SIZE(setcr0)
442
443 /*
444 * "lock mov %cr0" is used on processors which indicate it is
445 * supported via CPUID. Normally the 32 bit TPR is accessed via
446 * the local APIC.
447 */
448 ENTRY(getcr8)
449 lock
450 movl %cr0, %eax
451 ret
452 SET_SIZE(getcr8)
453
454 ENTRY(setcr8)
455 movl 4(%esp), %eax
456 lock
457 movl %eax, %cr0
458 ret
459 SET_SIZE(setcr8)
460
461 ENTRY(getcr2)
462 #if defined(__xpv)
463 movl %gs:CPU_VCPU_INFO, %eax
464 movl VCPU_INFO_ARCH_CR2(%eax), %eax
465 #else
466 movl %cr2, %eax
467 #endif
468 ret
469 SET_SIZE(getcr2)
470
471 ENTRY(getcr3)
472 movl %cr3, %eax
473 ret
474 SET_SIZE(getcr3)
475
476 #if !defined(__xpv)
477
478 ENTRY(setcr3)
479 movl 4(%esp), %eax
480 movl %eax, %cr3
481 ret
482 SET_SIZE(setcr3)
483
484 ENTRY(reload_cr3)
485 movl %cr3, %eax
486 movl %eax, %cr3
487 ret
488 SET_SIZE(reload_cr3)
489
490 #endif /* __xpv */
491
492 ENTRY(getcr4)
493 movl %cr4, %eax
494 ret
495 SET_SIZE(getcr4)
496
497 ENTRY(setcr4)
498 movl 4(%esp), %eax
499 movl %eax, %cr4
500 ret
501 SET_SIZE(setcr4)
502
503 #endif /* __i386 */
504
505
506 #if defined(__amd64)
507
508 ENTRY(__cpuid_insn)
509 movq %rbx, %r8
510 movq %rcx, %r9
511 movq %rdx, %r11
512 movl (%rdi), %eax /* %eax = regs->cp_eax */
513 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
514 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
515 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
516 cpuid
517 movl %eax, (%rdi) /* regs->cp_eax = %eax */
518 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
519 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
520 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
521 movq %r8, %rbx
522 movq %r9, %rcx
523 movq %r11, %rdx
524 ret
525 SET_SIZE(__cpuid_insn)
526
527 #elif defined(__i386)
528
529 ENTRY(__cpuid_insn)
530 pushl %ebp
531 movl 0x8(%esp), %ebp /* %ebp = regs */
532 pushl %ebx
533 pushl %ecx
534 pushl %edx
535 movl (%ebp), %eax /* %eax = regs->cp_eax */
536 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
537 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
538 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
539 cpuid
540 movl %eax, (%ebp) /* regs->cp_eax = %eax */
541 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
542 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
543 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
544 popl %edx
545 popl %ecx
546 popl %ebx
547 popl %ebp
548 ret
549 SET_SIZE(__cpuid_insn)
550
551 #endif /* __i386 */
552
553
554 #if defined(__amd64)
555
556 ENTRY_NP(i86_monitor)
557 pushq %rbp
558 movq %rsp, %rbp
559 movq %rdi, %rax /* addr */
560 movq %rsi, %rcx /* extensions */
561 /* rdx contains input arg3: hints */
562 clflush (%rax)
563 .byte 0x0f, 0x01, 0xc8 /* monitor */
564 leave
565 ret
566 SET_SIZE(i86_monitor)
567
568 #elif defined(__i386)
569
570 ENTRY_NP(i86_monitor)
571 pushl %ebp
572 movl %esp, %ebp
573 movl 0x8(%ebp),%eax /* addr */
574 movl 0xc(%ebp),%ecx /* extensions */
575 movl 0x10(%ebp),%edx /* hints */
576 clflush (%eax)
577 .byte 0x0f, 0x01, 0xc8 /* monitor */
578 leave
579 ret
580 SET_SIZE(i86_monitor)
581
582 #endif /* __i386 */
583
584
585 #if defined(__amd64)
586
587 ENTRY_NP(i86_mwait)
588 pushq %rbp
589 movq %rsp, %rbp
590 movq %rdi, %rax /* data */
591 movq %rsi, %rcx /* extensions */
592 .byte 0x0f, 0x01, 0xc9 /* mwait */
593 leave
594 ret
595 SET_SIZE(i86_mwait)
596
597 #elif defined(__i386)
598
599 ENTRY_NP(i86_mwait)
600 pushl %ebp
601 movl %esp, %ebp
602 movl 0x8(%ebp),%eax /* data */
603 movl 0xc(%ebp),%ecx /* extensions */
604 .byte 0x0f, 0x01, 0xc9 /* mwait */
605 leave
606 ret
607 SET_SIZE(i86_mwait)
608
609 #endif /* __i386 */
610
611
612
613 #if defined(__amd64)
614
615 ENTRY_NP(tsc_read)
616 movq %rbx, %r11
617 movl $0, %eax
618 cpuid
619 rdtsc
620 movq %r11, %rbx
621 shlq $32, %rdx
622 orq %rdx, %rax
623 ret
624 .globl _tsc_mfence_start
625 _tsc_mfence_start:
626 mfence
627 rdtsc
628 shlq $32, %rdx
629 orq %rdx, %rax
630 ret
631 .globl _tsc_mfence_end
632 _tsc_mfence_end:
633 .globl _tscp_start
634 _tscp_start:
635 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
636 shlq $32, %rdx
637 orq %rdx, %rax
638 ret
639 .globl _tscp_end
640 _tscp_end:
641 .globl _no_rdtsc_start
642 _no_rdtsc_start:
643 xorl %edx, %edx
644 xorl %eax, %eax
645 ret
646 .globl _no_rdtsc_end
647 _no_rdtsc_end:
648 .globl _tsc_lfence_start
649 _tsc_lfence_start:
650 lfence
651 rdtsc
652 shlq $32, %rdx
653 orq %rdx, %rax
654 ret
655 .globl _tsc_lfence_end
656 _tsc_lfence_end:
657 SET_SIZE(tsc_read)
658
659 #else /* __i386 */
660
661 ENTRY_NP(tsc_read)
662 pushl %ebx
663 movl $0, %eax
664 cpuid
665 rdtsc
666 popl %ebx
667 ret
668 .globl _tsc_mfence_start
669 _tsc_mfence_start:
670 mfence
671 rdtsc
672 ret
673 .globl _tsc_mfence_end
674 _tsc_mfence_end:
675 .globl _tscp_start
676 _tscp_start:
677 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
678 ret
679 .globl _tscp_end
680 _tscp_end:
681 .globl _no_rdtsc_start
682 _no_rdtsc_start:
683 xorl %edx, %edx
684 xorl %eax, %eax
685 ret
686 .globl _no_rdtsc_end
687 _no_rdtsc_end:
688 .globl _tsc_lfence_start
689 _tsc_lfence_start:
690 lfence
691 rdtsc
692 ret
693 .globl _tsc_lfence_end
694 _tsc_lfence_end:
695 SET_SIZE(tsc_read)
696
697 #endif /* __i386 */
698
699
700 /*
701 * Do not use this function for obtaining clock tick. This
702 * is called by callers who do not need to have a guarenteed
703 * correct tick value. The proper routine to use is tsc_read().
704 */
705
706 #if defined(__amd64)
707 ENTRY_NP(randtick)
708 rdtsc
709 shlq $32, %rdx
710 orq %rdx, %rax
711 ret
712 SET_SIZE(randtick)
713 #else
714 ENTRY_NP(randtick)
715 rdtsc
716 ret
717 SET_SIZE(randtick)
718 #endif /* __i386 */
719
720 /*
721 * Insert entryp after predp in a doubly linked list.
722 */
723
724 #if defined(__amd64)
725
726 ENTRY(_insque)
727 movq (%rsi), %rax /* predp->forw */
728 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
729 movq %rax, (%rdi) /* entryp->forw = predp->forw */
730 movq %rdi, (%rsi) /* predp->forw = entryp */
731 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
732 ret
733 SET_SIZE(_insque)
734
735 #elif defined(__i386)
736
737 ENTRY(_insque)
738 movl 8(%esp), %edx
739 movl 4(%esp), %ecx
740 movl (%edx), %eax /* predp->forw */
741 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
742 movl %eax, (%ecx) /* entryp->forw = predp->forw */
743 movl %ecx, (%edx) /* predp->forw = entryp */
744 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
745 ret
746 SET_SIZE(_insque)
747
748 #endif /* __i386 */
749
750 /*
751 * Remove entryp from a doubly linked list
752 */
753
754
755 #if defined(__amd64)
756
757 ENTRY(_remque)
758 movq (%rdi), %rax /* entry->forw */
759 movq CPTRSIZE(%rdi), %rdx /* entry->back */
760 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
761 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
762 ret
763 SET_SIZE(_remque)
764
765 #elif defined(__i386)
766
767 ENTRY(_remque)
768 movl 4(%esp), %ecx
769 movl (%ecx), %eax /* entry->forw */
770 movl CPTRSIZE(%ecx), %edx /* entry->back */
771 movl %eax, (%edx) /* entry->back->forw = entry->forw */
772 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
773 ret
774 SET_SIZE(_remque)
775
776 #endif /* __i386 */
777
778 /*
779 * Returns the number of
780 * non-NULL bytes in string argument.
781 */
782
783
784 #if defined(__amd64)
785
786 /*
787 * This is close to a simple transliteration of a C version of this
788 * routine. We should either just -make- this be a C version, or
789 * justify having it in assembler by making it significantly faster.
790 *
791 * size_t
792 * strlen(const char *s)
793 * {
794 * const char *s0;
795 * #if defined(DEBUG)
796 * if ((uintptr_t)s < KERNELBASE)
797 * panic(.str_panic_msg);
798 * #endif
799 * for (s0 = s; *s; s++)
800 * ;
801 * return (s - s0);
802 * }
803 */
804
805 ENTRY(strlen)
806 #ifdef DEBUG
807 movq postbootkernelbase(%rip), %rax
808 cmpq %rax, %rdi
809 jae str_valid
810 pushq %rbp
811 movq %rsp, %rbp
812 leaq .str_panic_msg(%rip), %rdi
813 xorl %eax, %eax
814 call panic
815 #endif /* DEBUG */
816 str_valid:
817 cmpb $0, (%rdi)
818 movq %rdi, %rax
819 je .null_found
820 .align 4
821 .strlen_loop:
822 incq %rdi
823 cmpb $0, (%rdi)
824 jne .strlen_loop
825 .null_found:
826 subq %rax, %rdi
827 movq %rdi, %rax
828 ret
829 SET_SIZE(strlen)
830
831 #elif defined(__i386)
832
833 ENTRY(strlen)
834 #ifdef DEBUG
835 movl postbootkernelbase, %eax
836 cmpl %eax, 4(%esp)
837 jae str_valid
838 pushl %ebp
839 movl %esp, %ebp
840 pushl $.str_panic_msg
841 call panic
842 #endif /* DEBUG */
843
844 str_valid:
845 movl 4(%esp), %eax /* %eax = string address */
846 testl $3, %eax /* if %eax not word aligned */
847 jnz .not_word_aligned /* goto .not_word_aligned */
848 .align 4
849 .word_aligned:
850 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
851 movl $0x7f7f7f7f, %ecx
852 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
853 addl $4, %eax /* next word */
854 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
855 orl %edx, %ecx /* %ecx |= %edx */
856 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
857 cmpl $0x80808080, %ecx /* if no null byte in this word */
858 je .word_aligned /* goto .word_aligned */
859 subl $4, %eax /* post-incremented */
860 .not_word_aligned:
861 cmpb $0, (%eax) /* if a byte in (%eax) is null */
862 je .null_found /* goto .null_found */
863 incl %eax /* next byte */
864 testl $3, %eax /* if %eax not word aligned */
865 jnz .not_word_aligned /* goto .not_word_aligned */
866 jmp .word_aligned /* goto .word_aligned */
867 .align 4
868 .null_found:
869 subl 4(%esp), %eax /* %eax -= string address */
870 ret
871 SET_SIZE(strlen)
872
873 #endif /* __i386 */
874
875 #ifdef DEBUG
876 .text
877 .str_panic_msg:
878 .string "strlen: argument below kernelbase"
879 #endif /* DEBUG */
880
881
882 /*
883 * Berkeley 4.3 introduced symbolically named interrupt levels
884 * as a way deal with priority in a machine independent fashion.
885 * Numbered priorities are machine specific, and should be
886 * discouraged where possible.
887 *
888 * Note, for the machine specific priorities there are
889 * examples listed for devices that use a particular priority.
890 * It should not be construed that all devices of that
891 * type should be at that priority. It is currently were
892 * the current devices fit into the priority scheme based
893 * upon time criticalness.
894 *
895 * The underlying assumption of these assignments is that
896 * IPL 10 is the highest level from which a device
897 * routine can call wakeup. Devices that interrupt from higher
898 * levels are restricted in what they can do. If they need
899 * kernels services they should schedule a routine at a lower
900 * level (via software interrupt) to do the required
901 * processing.
902 *
903 * Examples of this higher usage:
904 * Level Usage
905 * 14 Profiling clock (and PROM uart polling clock)
906 * 12 Serial ports
907 *
908 * The serial ports request lower level processing on level 6.
909 *
910 * Also, almost all splN routines (where N is a number or a
911 * mnemonic) will do a RAISE(), on the assumption that they are
912 * never used to lower our priority.
913 * The exceptions are:
914 * spl8() Because you can't be above 15 to begin with!
915 * splzs() Because this is used at boot time to lower our
916 * priority, to allow the PROM to poll the uart.
917 * spl0() Used to lower priority to 0.
918 */
919
920
921 #if defined(__amd64)
922
923 #define SETPRI(level) \
924 movl $/**/level, %edi; /* new priority */ \
925 jmp do_splx /* redirect to do_splx */
926
927 #define RAISE(level) \
928 movl $/**/level, %edi; /* new priority */ \
929 jmp splr /* redirect to splr */
930
931 #elif defined(__i386)
932
933 #define SETPRI(level) \
934 pushl $/**/level; /* new priority */ \
935 call do_splx; /* invoke common splx code */ \
936 addl $4, %esp; /* unstack arg */ \
937 ret
938
939 #define RAISE(level) \
940 pushl $/**/level; /* new priority */ \
941 call splr; /* invoke common splr code */ \
942 addl $4, %esp; /* unstack args */ \
943 ret
944
945 #endif /* __i386 */
946
947 /* locks out all interrupts, including memory errors */
948 ENTRY(spl8)
949 SETPRI(15)
950 SET_SIZE(spl8)
951
952 /* just below the level that profiling runs */
953 ENTRY(spl7)
954 RAISE(13)
955 SET_SIZE(spl7)
956
957 /* sun specific - highest priority onboard serial i/o asy ports */
958 ENTRY(splzs)
959 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
960 SET_SIZE(splzs)
961
962 ENTRY(splhi)
963 ALTENTRY(splhigh)
964 ALTENTRY(spl6)
965 ALTENTRY(i_ddi_splhigh)
966
967 RAISE(DISP_LEVEL)
968
969 SET_SIZE(i_ddi_splhigh)
970 SET_SIZE(spl6)
971 SET_SIZE(splhigh)
972 SET_SIZE(splhi)
973
974 /* allow all interrupts */
975 ENTRY(spl0)
976 SETPRI(0)
977 SET_SIZE(spl0)
978
979
980 /* splx implementation */
981 ENTRY(splx)
982 jmp do_splx /* redirect to common splx code */
983 SET_SIZE(splx)
984
985
986 #if defined(__i386)
987
988 /*
989 * Read and write the %gs register
990 */
991
992
993 ENTRY(getgs)
994 clr %eax
995 movw %gs, %ax
996 ret
997 SET_SIZE(getgs)
998
999 ENTRY(setgs)
1000 movw 4(%esp), %gs
1001 ret
1002 SET_SIZE(setgs)
1003
1004 #endif /* __i386 */
1005
1006
1007 ENTRY(wait_500ms)
1008 #if defined(__amd64)
1009 pushq %rbx
1010 #elif defined(__i386)
1011 push %ebx
1012 #endif
1013 movl $50000, %ebx
1014 1:
1015 call tenmicrosec
1016 decl %ebx
1017 jnz 1b
1018 #if defined(__amd64)
1019 popq %rbx
1020 #elif defined(__i386)
1021 pop %ebx
1022 #endif
1023 ret
1024 SET_SIZE(wait_500ms)
1025
1026 #define RESET_METHOD_KBC 1
1027 #define RESET_METHOD_PORT92 2
1028 #define RESET_METHOD_PCI 4
1029
1030 DGDEF3(pc_reset_methods, 4, 8)
1031 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1032
1033 ENTRY(pc_reset)
1034
1035 #if defined(__i386)
1036 testl $RESET_METHOD_KBC, pc_reset_methods
1037 #elif defined(__amd64)
1038 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1039 #endif
1040 jz 1f
1041
1042 /
1043 / Try the classic keyboard controller-triggered reset.
1044 /
1045 movw $0x64, %dx
1046 movb $0xfe, %al
1047 outb (%dx)
1048
1049 / Wait up to 500 milliseconds here for the keyboard controller
1050 / to pull the reset line. On some systems where the keyboard
1051 / controller is slow to pull the reset line, the next reset method
1052 / may be executed (which may be bad if those systems hang when the
1053 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1054 / and Ferrari 4000 (doesn't like the cf9 reset method))
1055
1056 call wait_500ms
1057
1058 1:
1059 #if defined(__i386)
1060 testl $RESET_METHOD_PORT92, pc_reset_methods
1061 #elif defined(__amd64)
1062 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1063 #endif
1064 jz 3f
1065
1066 /
1067 / Try port 0x92 fast reset
1068 /
1069 movw $0x92, %dx
1070 inb (%dx)
1071 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1072 je 1f
1073 testb $1, %al / If bit 0
1074 jz 2f / is clear, jump to perform the reset
1075 andb $0xfe, %al / otherwise,
1076 outb (%dx) / clear bit 0 first, then
1077 2:
1078 orb $1, %al / Set bit 0
1079 outb (%dx) / and reset the system
1080 1:
1081
1082 call wait_500ms
1083
1084 3:
1085 #if defined(__i386)
1086 testl $RESET_METHOD_PCI, pc_reset_methods
1087 #elif defined(__amd64)
1088 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1089 #endif
1090 jz 4f
1091
1092 / Try the PCI (soft) reset vector (should work on all modern systems,
1093 / but has been shown to cause problems on 450NX systems, and some newer
1094 / systems (e.g. ATI IXP400-equipped systems))
1095 / When resetting via this method, 2 writes are required. The first
1096 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1097 / power cycle).
1098 / The reset occurs on the second write, during bit 2's transition from
1099 / 0->1.
1100 movw $0xcf9, %dx
1101 movb $0x2, %al / Reset mode = hard, no power cycle
1102 outb (%dx)
1103 movb $0x6, %al
1104 outb (%dx)
1105
1106 call wait_500ms
1107
1108 4:
1109 /
1110 / port 0xcf9 failed also. Last-ditch effort is to
1111 / triple-fault the CPU.
1112 / Also, use triple fault for EFI firmware
1113 /
1114 ENTRY(efi_reset)
1115 #if defined(__amd64)
1116 pushq $0x0
1117 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1118 lidt (%rsp)
1119 #elif defined(__i386)
1120 pushl $0x0
1121 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1122 lidt (%esp)
1123 #endif
1124 int $0x0 / Trigger interrupt, generate triple-fault
1125
1126 cli
1127 hlt / Wait forever
1128 /*NOTREACHED*/
1129 SET_SIZE(efi_reset)
1130 SET_SIZE(pc_reset)
1131
1132
1133 /*
1134 * C callable in and out routines
1135 */
1136
1137
1138 #if defined(__amd64)
1139
1140 ENTRY(outl)
1141 movw %di, %dx
1142 movl %esi, %eax
1143 outl (%dx)
1144 ret
1145 SET_SIZE(outl)
1146
1147 #elif defined(__i386)
1148
1149 .set PORT, 4
1150 .set VAL, 8
1151
1152 ENTRY(outl)
1153 movw PORT(%esp), %dx
1154 movl VAL(%esp), %eax
1155 outl (%dx)
1156 ret
1157 SET_SIZE(outl)
1158
1159 #endif /* __i386 */
1160
1161
1162 #if defined(__amd64)
1163
1164 ENTRY(outw)
1165 movw %di, %dx
1166 movw %si, %ax
1167 D16 outl (%dx) /* XX64 why not outw? */
1168 ret
1169 SET_SIZE(outw)
1170
1171 #elif defined(__i386)
1172
1173 ENTRY(outw)
1174 movw PORT(%esp), %dx
1175 movw VAL(%esp), %ax
1176 D16 outl (%dx)
1177 ret
1178 SET_SIZE(outw)
1179
1180 #endif /* __i386 */
1181
1182
1183 #if defined(__amd64)
1184
1185 ENTRY(outb)
1186 movw %di, %dx
1187 movb %sil, %al
1188 outb (%dx)
1189 ret
1190 SET_SIZE(outb)
1191
1192 #elif defined(__i386)
1193
1194 ENTRY(outb)
1195 movw PORT(%esp), %dx
1196 movb VAL(%esp), %al
1197 outb (%dx)
1198 ret
1199 SET_SIZE(outb)
1200
1201 #endif /* __i386 */
1202
1203
1204 #if defined(__amd64)
1205
1206 ENTRY(inl)
1207 xorl %eax, %eax
1208 movw %di, %dx
1209 inl (%dx)
1210 ret
1211 SET_SIZE(inl)
1212
1213 #elif defined(__i386)
1214
1215 ENTRY(inl)
1216 movw PORT(%esp), %dx
1217 inl (%dx)
1218 ret
1219 SET_SIZE(inl)
1220
1221 #endif /* __i386 */
1222
1223
1224 #if defined(__amd64)
1225
1226 ENTRY(inw)
1227 xorl %eax, %eax
1228 movw %di, %dx
1229 D16 inl (%dx)
1230 ret
1231 SET_SIZE(inw)
1232
1233 #elif defined(__i386)
1234
1235 ENTRY(inw)
1236 subl %eax, %eax
1237 movw PORT(%esp), %dx
1238 D16 inl (%dx)
1239 ret
1240 SET_SIZE(inw)
1241
1242 #endif /* __i386 */
1243
1244
1245
1246 #if defined(__amd64)
1247
1248 ENTRY(inb)
1249 xorl %eax, %eax
1250 movw %di, %dx
1251 inb (%dx)
1252 ret
1253 SET_SIZE(inb)
1254
1255 #elif defined(__i386)
1256
1257 ENTRY(inb)
1258 subl %eax, %eax
1259 movw PORT(%esp), %dx
1260 inb (%dx)
1261 ret
1262 SET_SIZE(inb)
1263
1264 #endif /* __i386 */
1265
1266
1267
1268 #if defined(__amd64)
1269
1270 ENTRY(repoutsw)
1271 movl %edx, %ecx
1272 movw %di, %dx
1273 rep
1274 D16 outsl
1275 ret
1276 SET_SIZE(repoutsw)
1277
1278 #elif defined(__i386)
1279
1280 /*
1281 * The arguments and saved registers are on the stack in the
1282 * following order:
1283 * | cnt | +16
1284 * | *addr | +12
1285 * | port | +8
1286 * | eip | +4
1287 * | esi | <-- %esp
1288 * If additional values are pushed onto the stack, make sure
1289 * to adjust the following constants accordingly.
1290 */
1291 .set PORT, 8
1292 .set ADDR, 12
1293 .set COUNT, 16
1294
1295 ENTRY(repoutsw)
1296 pushl %esi
1297 movl PORT(%esp), %edx
1298 movl ADDR(%esp), %esi
1299 movl COUNT(%esp), %ecx
1300 rep
1301 D16 outsl
1302 popl %esi
1303 ret
1304 SET_SIZE(repoutsw)
1305
1306 #endif /* __i386 */
1307
1308
1309
1310 #if defined(__amd64)
1311
1312 ENTRY(repinsw)
1313 movl %edx, %ecx
1314 movw %di, %dx
1315 rep
1316 D16 insl
1317 ret
1318 SET_SIZE(repinsw)
1319
1320 #elif defined(__i386)
1321
1322 ENTRY(repinsw)
1323 pushl %edi
1324 movl PORT(%esp), %edx
1325 movl ADDR(%esp), %edi
1326 movl COUNT(%esp), %ecx
1327 rep
1328 D16 insl
1329 popl %edi
1330 ret
1331 SET_SIZE(repinsw)
1332
1333 #endif /* __i386 */
1334
1335
1336
1337 #if defined(__amd64)
1338
1339 ENTRY(repinsb)
1340 movl %edx, %ecx
1341 movw %di, %dx
1342 movq %rsi, %rdi
1343 rep
1344 insb
1345 ret
1346 SET_SIZE(repinsb)
1347
1348 #elif defined(__i386)
1349
1350 /*
1351 * The arguments and saved registers are on the stack in the
1352 * following order:
1353 * | cnt | +16
1354 * | *addr | +12
1355 * | port | +8
1356 * | eip | +4
1357 * | esi | <-- %esp
1358 * If additional values are pushed onto the stack, make sure
1359 * to adjust the following constants accordingly.
1360 */
1361 .set IO_PORT, 8
1362 .set IO_ADDR, 12
1363 .set IO_COUNT, 16
1364
1365 ENTRY(repinsb)
1366 pushl %edi
1367 movl IO_ADDR(%esp), %edi
1368 movl IO_COUNT(%esp), %ecx
1369 movl IO_PORT(%esp), %edx
1370 rep
1371 insb
1372 popl %edi
1373 ret
1374 SET_SIZE(repinsb)
1375
1376 #endif /* __i386 */
1377
1378
1379 /*
1380 * Input a stream of 32-bit words.
1381 * NOTE: count is a DWORD count.
1382 */
1383
1384 #if defined(__amd64)
1385
1386 ENTRY(repinsd)
1387 movl %edx, %ecx
1388 movw %di, %dx
1389 movq %rsi, %rdi
1390 rep
1391 insl
1392 ret
1393 SET_SIZE(repinsd)
1394
1395 #elif defined(__i386)
1396
1397 ENTRY(repinsd)
1398 pushl %edi
1399 movl IO_ADDR(%esp), %edi
1400 movl IO_COUNT(%esp), %ecx
1401 movl IO_PORT(%esp), %edx
1402 rep
1403 insl
1404 popl %edi
1405 ret
1406 SET_SIZE(repinsd)
1407
1408 #endif /* __i386 */
1409
1410 /*
1411 * Output a stream of bytes
1412 * NOTE: count is a byte count
1413 */
1414
1415 #if defined(__amd64)
1416
1417 ENTRY(repoutsb)
1418 movl %edx, %ecx
1419 movw %di, %dx
1420 rep
1421 outsb
1422 ret
1423 SET_SIZE(repoutsb)
1424
1425 #elif defined(__i386)
1426
1427 ENTRY(repoutsb)
1428 pushl %esi
1429 movl IO_ADDR(%esp), %esi
1430 movl IO_COUNT(%esp), %ecx
1431 movl IO_PORT(%esp), %edx
1432 rep
1433 outsb
1434 popl %esi
1435 ret
1436 SET_SIZE(repoutsb)
1437
1438 #endif /* __i386 */
1439
1440 /*
1441 * Output a stream of 32-bit words
1442 * NOTE: count is a DWORD count
1443 */
1444
1445 #if defined(__amd64)
1446
1447 ENTRY(repoutsd)
1448 movl %edx, %ecx
1449 movw %di, %dx
1450 rep
1451 outsl
1452 ret
1453 SET_SIZE(repoutsd)
1454
1455 #elif defined(__i386)
1456
1457 ENTRY(repoutsd)
1458 pushl %esi
1459 movl IO_ADDR(%esp), %esi
1460 movl IO_COUNT(%esp), %ecx
1461 movl IO_PORT(%esp), %edx
1462 rep
1463 outsl
1464 popl %esi
1465 ret
1466 SET_SIZE(repoutsd)
1467
1468 #endif /* __i386 */
1469
1470 /*
1471 * void int3(void)
1472 * void int18(void)
1473 * void int20(void)
1474 * void int_cmci(void)
1475 */
1476
1477
1478 ENTRY(int3)
1479 int $T_BPTFLT
1480 ret
1481 SET_SIZE(int3)
1482
1483 ENTRY(int18)
1484 int $T_MCE
1485 ret
1486 SET_SIZE(int18)
1487
1488 ENTRY(int20)
1489 movl boothowto, %eax
1490 andl $RB_DEBUG, %eax
1491 jz 1f
1492
1493 int $T_DBGENTR
1494 1:
1495 rep; ret /* use 2 byte return instruction when branch target */
1496 /* AMD Software Optimization Guide - Section 6.2 */
1497 SET_SIZE(int20)
1498
1499 ENTRY(int_cmci)
1500 int $T_ENOEXTFLT
1501 ret
1502 SET_SIZE(int_cmci)
1503
1504
1505
1506 #if defined(__amd64)
1507
1508 ENTRY(scanc)
1509 /* rdi == size */
1510 /* rsi == cp */
1511 /* rdx == table */
1512 /* rcx == mask */
1513 addq %rsi, %rdi /* end = &cp[size] */
1514 .scanloop:
1515 cmpq %rdi, %rsi /* while (cp < end */
1516 jnb .scandone
1517 movzbq (%rsi), %r8 /* %r8 = *cp */
1518 incq %rsi /* cp++ */
1519 testb %cl, (%r8, %rdx)
1520 jz .scanloop /* && (table[*cp] & mask) == 0) */
1521 decq %rsi /* (fix post-increment) */
1522 .scandone:
1523 movl %edi, %eax
1524 subl %esi, %eax /* return (end - cp) */
1525 ret
1526 SET_SIZE(scanc)
1527
1528 #elif defined(__i386)
1529
1530 ENTRY(scanc)
1531 pushl %edi
1532 pushl %esi
1533 movb 24(%esp), %cl /* mask = %cl */
1534 movl 16(%esp), %esi /* cp = %esi */
1535 movl 20(%esp), %edx /* table = %edx */
1536 movl %esi, %edi
1537 addl 12(%esp), %edi /* end = &cp[size]; */
1538 .scanloop:
1539 cmpl %edi, %esi /* while (cp < end */
1540 jnb .scandone
1541 movzbl (%esi), %eax /* %al = *cp */
1542 incl %esi /* cp++ */
1543 movb (%edx, %eax), %al /* %al = table[*cp] */
1544 testb %al, %cl
1545 jz .scanloop /* && (table[*cp] & mask) == 0) */
1546 dec %esi /* post-incremented */
1547 .scandone:
1548 movl %edi, %eax
1549 subl %esi, %eax /* return (end - cp) */
1550 popl %esi
1551 popl %edi
1552 ret
1553 SET_SIZE(scanc)
1554
1555 #endif /* __i386 */
1556
1557 /*
1558 * Replacement functions for ones that are normally inlined.
1559 * In addition to the copy in i86.il, they are defined here just in case.
1560 */
1561
1562
1563 #if defined(__amd64)
1564
1565 ENTRY(intr_clear)
1566 ENTRY(clear_int_flag)
1567 pushfq
1568 popq %rax
1569 #if defined(__xpv)
1570 leaq xpv_panicking, %rdi
1571 movl (%rdi), %edi
1572 cmpl $0, %edi
1573 jne 2f
1574 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1575 /*
1576 * Synthesize the PS_IE bit from the event mask bit
1577 */
1578 andq $_BITNOT(PS_IE), %rax
1579 testb $1, %dl
1580 jnz 1f
1581 orq $PS_IE, %rax
1582 1:
1583 ret
1584 2:
1585 #endif
1586 CLI(%rdi)
1587 ret
1588 SET_SIZE(clear_int_flag)
1589 SET_SIZE(intr_clear)
1590
1591 #elif defined(__i386)
1592
1593 ENTRY(intr_clear)
1594 ENTRY(clear_int_flag)
1595 pushfl
1596 popl %eax
1597 #if defined(__xpv)
1598 leal xpv_panicking, %edx
1599 movl (%edx), %edx
1600 cmpl $0, %edx
1601 jne 2f
1602 CLIRET(%edx, %cl) /* returns event mask in %cl */
1603 /*
1604 * Synthesize the PS_IE bit from the event mask bit
1605 */
1606 andl $_BITNOT(PS_IE), %eax
1607 testb $1, %cl
1608 jnz 1f
1609 orl $PS_IE, %eax
1610 1:
1611 ret
1612 2:
1613 #endif
1614 CLI(%edx)
1615 ret
1616 SET_SIZE(clear_int_flag)
1617 SET_SIZE(intr_clear)
1618
1619 #endif /* __i386 */
1620
1621
1622 #if defined(__amd64)
1623
1624 ENTRY(curcpup)
1625 movq %gs:CPU_SELF, %rax
1626 ret
1627 SET_SIZE(curcpup)
1628
1629 #elif defined(__i386)
1630
1631 ENTRY(curcpup)
1632 movl %gs:CPU_SELF, %eax
1633 ret
1634 SET_SIZE(curcpup)
1635
1636 #endif /* __i386 */
1637
1638 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
1639 * These functions reverse the byte order of the input parameter and returns
1640 * the result. This is to convert the byte order from host byte order
1641 * (little endian) to network byte order (big endian), or vice versa.
1642 */
1643
1644
1645 #if defined(__amd64)
1646
1647 ENTRY(htonll)
1648 ALTENTRY(ntohll)
1649 movq %rdi, %rax
1650 bswapq %rax
1651 ret
1652 SET_SIZE(ntohll)
1653 SET_SIZE(htonll)
1654
1655 /* XX64 there must be shorter sequences for this */
1656 ENTRY(htonl)
1657 ALTENTRY(ntohl)
1658 movl %edi, %eax
1659 bswap %eax
1660 ret
1661 SET_SIZE(ntohl)
1662 SET_SIZE(htonl)
1663
1664 /* XX64 there must be better sequences for this */
1665 ENTRY(htons)
1666 ALTENTRY(ntohs)
1667 movl %edi, %eax
1668 bswap %eax
1669 shrl $16, %eax
1670 ret
1671 SET_SIZE(ntohs)
1672 SET_SIZE(htons)
1673
1674 #elif defined(__i386)
1675
1676 ENTRY(htonll)
1677 ALTENTRY(ntohll)
1678 movl 4(%esp), %edx
1679 movl 8(%esp), %eax
1680 bswap %edx
1681 bswap %eax
1682 ret
1683 SET_SIZE(ntohll)
1684 SET_SIZE(htonll)
1685
1686 ENTRY(htonl)
1687 ALTENTRY(ntohl)
1688 movl 4(%esp), %eax
1689 bswap %eax
1690 ret
1691 SET_SIZE(ntohl)
1692 SET_SIZE(htonl)
1693
1694 ENTRY(htons)
1695 ALTENTRY(ntohs)
1696 movl 4(%esp), %eax
1697 bswap %eax
1698 shrl $16, %eax
1699 ret
1700 SET_SIZE(ntohs)
1701 SET_SIZE(htons)
1702
1703 #endif /* __i386 */
1704
1705
1706
1707 #if defined(__amd64)
1708
1709 ENTRY(intr_restore)
1710 ENTRY(restore_int_flag)
1711 testq $PS_IE, %rdi
1712 jz 1f
1713 #if defined(__xpv)
1714 leaq xpv_panicking, %rsi
1715 movl (%rsi), %esi
1716 cmpl $0, %esi
1717 jne 1f
1718 /*
1719 * Since we're -really- running unprivileged, our attempt
1720 * to change the state of the IF bit will be ignored.
1721 * The virtual IF bit is tweaked by CLI and STI.
1722 */
1723 IE_TO_EVENT_MASK(%rsi, %rdi)
1724 #else
1725 sti
1726 #endif
1727 1:
1728 ret
1729 SET_SIZE(restore_int_flag)
1730 SET_SIZE(intr_restore)
1731
1732 #elif defined(__i386)
1733
1734 ENTRY(intr_restore)
1735 ENTRY(restore_int_flag)
1736 testl $PS_IE, 4(%esp)
1737 jz 1f
1738 #if defined(__xpv)
1739 leal xpv_panicking, %edx
1740 movl (%edx), %edx
1741 cmpl $0, %edx
1742 jne 1f
1743 /*
1744 * Since we're -really- running unprivileged, our attempt
1745 * to change the state of the IF bit will be ignored.
1746 * The virtual IF bit is tweaked by CLI and STI.
1747 */
1748 IE_TO_EVENT_MASK(%edx, 4(%esp))
1749 #else
1750 sti
1751 #endif
1752 1:
1753 ret
1754 SET_SIZE(restore_int_flag)
1755 SET_SIZE(intr_restore)
1756
1757 #endif /* __i386 */
1758
1759
1760 ENTRY(sti)
1761 STI
1762 ret
1763 SET_SIZE(sti)
1764
1765 ENTRY(cli)
1766 #if defined(__amd64)
1767 CLI(%rax)
1768 #elif defined(__i386)
1769 CLI(%eax)
1770 #endif /* __i386 */
1771 ret
1772 SET_SIZE(cli)
1773
1774
1775
1776 #if defined(__amd64)
1777
1778 ENTRY(dtrace_interrupt_disable)
1779 pushfq
1780 popq %rax
1781 #if defined(__xpv)
1782 leaq xpv_panicking, %rdi
1783 movl (%rdi), %edi
1784 cmpl $0, %edi
1785 jne .dtrace_interrupt_disable_done
1786 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1787 /*
1788 * Synthesize the PS_IE bit from the event mask bit
1789 */
1790 andq $_BITNOT(PS_IE), %rax
1791 testb $1, %dl
1792 jnz .dtrace_interrupt_disable_done
1793 orq $PS_IE, %rax
1794 #else
1795 CLI(%rdx)
1796 #endif
1797 .dtrace_interrupt_disable_done:
1798 ret
1799 SET_SIZE(dtrace_interrupt_disable)
1800
1801 #elif defined(__i386)
1802
1803 ENTRY(dtrace_interrupt_disable)
1804 pushfl
1805 popl %eax
1806 #if defined(__xpv)
1807 leal xpv_panicking, %edx
1808 movl (%edx), %edx
1809 cmpl $0, %edx
1810 jne .dtrace_interrupt_disable_done
1811 CLIRET(%edx, %cl) /* returns event mask in %cl */
1812 /*
1813 * Synthesize the PS_IE bit from the event mask bit
1814 */
1815 andl $_BITNOT(PS_IE), %eax
1816 testb $1, %cl
1817 jnz .dtrace_interrupt_disable_done
1818 orl $PS_IE, %eax
1819 #else
1820 CLI(%edx)
1821 #endif
1822 .dtrace_interrupt_disable_done:
1823 ret
1824 SET_SIZE(dtrace_interrupt_disable)
1825
1826 #endif /* __i386 */
1827
1828
1829 #if defined(__amd64)
1830
1831 ENTRY(dtrace_interrupt_enable)
1832 pushq %rdi
1833 popfq
1834 #if defined(__xpv)
1835 leaq xpv_panicking, %rdx
1836 movl (%rdx), %edx
1837 cmpl $0, %edx
1838 jne .dtrace_interrupt_enable_done
1839 /*
1840 * Since we're -really- running unprivileged, our attempt
1841 * to change the state of the IF bit will be ignored. The
1842 * virtual IF bit is tweaked by CLI and STI.
1843 */
1844 IE_TO_EVENT_MASK(%rdx, %rdi)
1845 #endif
1846 .dtrace_interrupt_enable_done:
1847 ret
1848 SET_SIZE(dtrace_interrupt_enable)
1849
1850 #elif defined(__i386)
1851
1852 ENTRY(dtrace_interrupt_enable)
1853 movl 4(%esp), %eax
1854 pushl %eax
1855 popfl
1856 #if defined(__xpv)
1857 leal xpv_panicking, %edx
1858 movl (%edx), %edx
1859 cmpl $0, %edx
1860 jne .dtrace_interrupt_enable_done
1861 /*
1862 * Since we're -really- running unprivileged, our attempt
1863 * to change the state of the IF bit will be ignored. The
1864 * virtual IF bit is tweaked by CLI and STI.
1865 */
1866 IE_TO_EVENT_MASK(%edx, %eax)
1867 #endif
1868 .dtrace_interrupt_enable_done:
1869 ret
1870 SET_SIZE(dtrace_interrupt_enable)
1871
1872 #endif /* __i386 */
1873
1874
1875
1876 ENTRY(dtrace_membar_producer)
1877 rep; ret /* use 2 byte return instruction when branch target */
1878 /* AMD Software Optimization Guide - Section 6.2 */
1879 SET_SIZE(dtrace_membar_producer)
1880
1881 ENTRY(dtrace_membar_consumer)
1882 rep; ret /* use 2 byte return instruction when branch target */
1883 /* AMD Software Optimization Guide - Section 6.2 */
1884 SET_SIZE(dtrace_membar_consumer)
1885
1886
1887
1888 #if defined(__amd64)
1889
1890 ENTRY(threadp)
1891 movq %gs:CPU_THREAD, %rax
1892 ret
1893 SET_SIZE(threadp)
1894
1895 #elif defined(__i386)
1896
1897 ENTRY(threadp)
1898 movl %gs:CPU_THREAD, %eax
1899 ret
1900 SET_SIZE(threadp)
1901
1902 #endif /* __i386 */
1903
1904 /*
1905 * Checksum routine for Internet Protocol Headers
1906 */
1907
1908
1909 #if defined(__amd64)
1910
1911 ENTRY(ip_ocsum)
1912 pushq %rbp
1913 movq %rsp, %rbp
1914 #ifdef DEBUG
1915 movq postbootkernelbase(%rip), %rax
1916 cmpq %rax, %rdi
1917 jnb 1f
1918 xorl %eax, %eax
1919 movq %rdi, %rsi
1920 leaq .ip_ocsum_panic_msg(%rip), %rdi
1921 call panic
1922 /*NOTREACHED*/
1923 .ip_ocsum_panic_msg:
1924 .string "ip_ocsum: address 0x%p below kernelbase\n"
1925 1:
1926 #endif
1927 movl %esi, %ecx /* halfword_count */
1928 movq %rdi, %rsi /* address */
1929 /* partial sum in %edx */
1930 xorl %eax, %eax
1931 testl %ecx, %ecx
1932 jz .ip_ocsum_done
1933 testq $3, %rsi
1934 jnz .ip_csum_notaligned
1935 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
1936 .next_iter:
1937 /* XX64 opportunities for prefetch? */
1938 /* XX64 compute csum with 64 bit quantities? */
1939 subl $32, %ecx
1940 jl .less_than_32
1941
1942 addl 0(%rsi), %edx
1943 .only60:
1944 adcl 4(%rsi), %eax
1945 .only56:
1946 adcl 8(%rsi), %edx
1947 .only52:
1948 adcl 12(%rsi), %eax
1949 .only48:
1950 adcl 16(%rsi), %edx
1951 .only44:
1952 adcl 20(%rsi), %eax
1953 .only40:
1954 adcl 24(%rsi), %edx
1955 .only36:
1956 adcl 28(%rsi), %eax
1957 .only32:
1958 adcl 32(%rsi), %edx
1959 .only28:
1960 adcl 36(%rsi), %eax
1961 .only24:
1962 adcl 40(%rsi), %edx
1963 .only20:
1964 adcl 44(%rsi), %eax
1965 .only16:
1966 adcl 48(%rsi), %edx
1967 .only12:
1968 adcl 52(%rsi), %eax
1969 .only8:
1970 adcl 56(%rsi), %edx
1971 .only4:
1972 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
1973 .only0:
1974 adcl $0, %eax /* could be adding -1 in eax with a carry */
1975 adcl $0, %eax
1976
1977 addq $64, %rsi
1978 testl %ecx, %ecx
1979 jnz .next_iter
1980
1981 .ip_ocsum_done:
1982 addl %eax, %edx
1983 adcl $0, %edx
1984 movl %edx, %eax /* form a 16 bit checksum by */
1985 shrl $16, %eax /* adding two halves of 32 bit checksum */
1986 addw %dx, %ax
1987 adcw $0, %ax
1988 andl $0xffff, %eax
1989 leave
1990 ret
1991
1992 .ip_csum_notaligned:
1993 xorl %edi, %edi
1994 movw (%rsi), %di
1995 addl %edi, %edx
1996 adcl $0, %edx
1997 addq $2, %rsi
1998 decl %ecx
1999 jmp .ip_csum_aligned
2000
2001 .less_than_32:
2002 addl $32, %ecx
2003 testl $1, %ecx
2004 jz .size_aligned
2005 andl $0xfe, %ecx
2006 movzwl (%rsi, %rcx, 2), %edi
2007 addl %edi, %edx
2008 adcl $0, %edx
2009 .size_aligned:
2010 movl %ecx, %edi
2011 shrl $1, %ecx
2012 shl $1, %edi
2013 subq $64, %rdi
2014 addq %rdi, %rsi
2015 leaq .ip_ocsum_jmptbl(%rip), %rdi
2016 leaq (%rdi, %rcx, 8), %rdi
2017 xorl %ecx, %ecx
2018 clc
2019 jmp *(%rdi)
2020
2021 .align 8
2022 .ip_ocsum_jmptbl:
2023 .quad .only0, .only4, .only8, .only12, .only16, .only20
2024 .quad .only24, .only28, .only32, .only36, .only40, .only44
2025 .quad .only48, .only52, .only56, .only60
2026 SET_SIZE(ip_ocsum)
2027
2028 #elif defined(__i386)
2029
2030 ENTRY(ip_ocsum)
2031 pushl %ebp
2032 movl %esp, %ebp
2033 pushl %ebx
2034 pushl %esi
2035 pushl %edi
2036 movl 12(%ebp), %ecx /* count of half words */
2037 movl 16(%ebp), %edx /* partial checksum */
2038 movl 8(%ebp), %esi
2039 xorl %eax, %eax
2040 testl %ecx, %ecx
2041 jz .ip_ocsum_done
2042
2043 testl $3, %esi
2044 jnz .ip_csum_notaligned
2045 .ip_csum_aligned:
2046 .next_iter:
2047 subl $32, %ecx
2048 jl .less_than_32
2049
2050 addl 0(%esi), %edx
2051 .only60:
2052 adcl 4(%esi), %eax
2053 .only56:
2054 adcl 8(%esi), %edx
2055 .only52:
2056 adcl 12(%esi), %eax
2057 .only48:
2058 adcl 16(%esi), %edx
2059 .only44:
2060 adcl 20(%esi), %eax
2061 .only40:
2062 adcl 24(%esi), %edx
2063 .only36:
2064 adcl 28(%esi), %eax
2065 .only32:
2066 adcl 32(%esi), %edx
2067 .only28:
2068 adcl 36(%esi), %eax
2069 .only24:
2070 adcl 40(%esi), %edx
2071 .only20:
2072 adcl 44(%esi), %eax
2073 .only16:
2074 adcl 48(%esi), %edx
2075 .only12:
2076 adcl 52(%esi), %eax
2077 .only8:
2078 adcl 56(%esi), %edx
2079 .only4:
2080 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2081 .only0:
2082 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2083 adcl $0, %eax
2084
2085 addl $64, %esi
2086 andl %ecx, %ecx
2087 jnz .next_iter
2088
2089 .ip_ocsum_done:
2090 addl %eax, %edx
2091 adcl $0, %edx
2092 movl %edx, %eax /* form a 16 bit checksum by */
2093 shrl $16, %eax /* adding two halves of 32 bit checksum */
2094 addw %dx, %ax
2095 adcw $0, %ax
2096 andl $0xffff, %eax
2097 popl %edi /* restore registers */
2098 popl %esi
2099 popl %ebx
2100 leave
2101 ret
2102
2103 .ip_csum_notaligned:
2104 xorl %edi, %edi
2105 movw (%esi), %di
2106 addl %edi, %edx
2107 adcl $0, %edx
2108 addl $2, %esi
2109 decl %ecx
2110 jmp .ip_csum_aligned
2111
2112 .less_than_32:
2113 addl $32, %ecx
2114 testl $1, %ecx
2115 jz .size_aligned
2116 andl $0xfe, %ecx
2117 movzwl (%esi, %ecx, 2), %edi
2118 addl %edi, %edx
2119 adcl $0, %edx
2120 .size_aligned:
2121 movl %ecx, %edi
2122 shrl $1, %ecx
2123 shl $1, %edi
2124 subl $64, %edi
2125 addl %edi, %esi
2126 movl $.ip_ocsum_jmptbl, %edi
2127 lea (%edi, %ecx, 4), %edi
2128 xorl %ecx, %ecx
2129 clc
2130 jmp *(%edi)
2131 SET_SIZE(ip_ocsum)
2132
2133 .data
2134 .align 4
2135
2136 .ip_ocsum_jmptbl:
2137 .long .only0, .only4, .only8, .only12, .only16, .only20
2138 .long .only24, .only28, .only32, .only36, .only40, .only44
2139 .long .only48, .only52, .only56, .only60
2140
2141
2142 #endif /* __i386 */
2143
2144 /*
2145 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2146 * Provided to manipulate hrtime_t values.
2147 */
2148
2149 #if defined(__amd64)
2150
2151 ENTRY(mul32)
2152 xorl %edx, %edx /* XX64 joe, paranoia? */
2153 movl %edi, %eax
2154 mull %esi
2155 shlq $32, %rdx
2156 orq %rdx, %rax
2157 ret
2158 SET_SIZE(mul32)
2159
2160 #elif defined(__i386)
2161
2162 ENTRY(mul32)
2163 movl 8(%esp), %eax
2164 movl 4(%esp), %ecx
2165 mull %ecx
2166 ret
2167 SET_SIZE(mul32)
2168
2169 #endif /* __i386 */
2170
2171 #if defined(notused)
2172 .globl load_pte64
2173 load_pte64:
2174 movl 4(%esp), %eax
2175 movl 8(%esp), %ecx
2176 movl 12(%esp), %edx
2177 movl %edx, 4(%eax)
2178 movl %ecx, (%eax)
2179 ret
2180 #endif /* notused */
2181
2182
2183 #if defined(__amd64)
2184
2185 ENTRY(scan_memory)
2186 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2187 jz .scanm_done
2188 movq %rsi, %rcx /* move count into rep control register */
2189 movq %rdi, %rsi /* move addr into lodsq control reg. */
2190 rep lodsq /* scan the memory range */
2191 .scanm_done:
2192 rep; ret /* use 2 byte return instruction when branch target */
2193 /* AMD Software Optimization Guide - Section 6.2 */
2194 SET_SIZE(scan_memory)
2195
2196 #elif defined(__i386)
2197
2198 ENTRY(scan_memory)
2199 pushl %ecx
2200 pushl %esi
2201 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2202 shrl $2, %ecx /* convert from byte count to word count */
2203 jz .scanm_done
2204 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2205 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2206 lodsl
2207 .scanm_done:
2208 popl %esi
2209 popl %ecx
2210 ret
2211 SET_SIZE(scan_memory)
2212
2213 #endif /* __i386 */
2214
2215
2216
2217 #if defined(__amd64)
2218
2219 ENTRY(lowbit)
2220 movl $-1, %eax
2221 bsfq %rdi, %rdi
2222 cmovnz %edi, %eax
2223 incl %eax
2224 ret
2225 SET_SIZE(lowbit)
2226
2227 #elif defined(__i386)
2228
2229 ENTRY(lowbit)
2230 bsfl 4(%esp), %eax
2231 jz 0f
2232 incl %eax
2233 ret
2234 0:
2235 xorl %eax, %eax
2236 ret
2237 SET_SIZE(lowbit)
2238
2239 #endif /* __i386 */
2240
2241
2242 #if defined(__amd64)
2243
2244 ENTRY(highbit)
2245 ALTENTRY(highbit64)
2246 movl $-1, %eax
2247 bsrq %rdi, %rdi
2248 cmovnz %edi, %eax
2249 incl %eax
2250 ret
2251 SET_SIZE(highbit64)
2252 SET_SIZE(highbit)
2253
2254 #elif defined(__i386)
2255
2256 ENTRY(highbit)
2257 bsrl 4(%esp), %eax
2258 jz 0f
2259 incl %eax
2260 ret
2261 0:
2262 xorl %eax, %eax
2263 ret
2264 SET_SIZE(highbit)
2265
2266 ENTRY(highbit64)
2267 bsrl 8(%esp), %eax
2268 jz highbit
2269 addl $33, %eax
2270 ret
2271 SET_SIZE(highbit64)
2272
2273 #endif /* __i386 */
2274
2275
2276 #define XMSR_ACCESS_VAL $0x9c5a203a
2277
2278 #if defined(__amd64)
2279
2280 ENTRY(rdmsr)
2281 movl %edi, %ecx
2282 rdmsr
2283 shlq $32, %rdx
2284 orq %rdx, %rax
2285 ret
2286 SET_SIZE(rdmsr)
2287
2288 ENTRY(wrmsr)
2289 movq %rsi, %rdx
2290 shrq $32, %rdx
2291 movl %esi, %eax
2292 movl %edi, %ecx
2293 wrmsr
2294 ret
2295 SET_SIZE(wrmsr)
2296
2297 ENTRY(xrdmsr)
2298 pushq %rbp
2299 movq %rsp, %rbp
2300 movl %edi, %ecx
2301 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2302 rdmsr
2303 shlq $32, %rdx
2304 orq %rdx, %rax
2305 leave
2306 ret
2307 SET_SIZE(xrdmsr)
2308
2309 ENTRY(xwrmsr)
2310 pushq %rbp
2311 movq %rsp, %rbp
2312 movl %edi, %ecx
2313 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2314 movq %rsi, %rdx
2315 shrq $32, %rdx
2316 movl %esi, %eax
2317 wrmsr
2318 leave
2319 ret
2320 SET_SIZE(xwrmsr)
2321
2322 ENTRY(get_xcr)
2323 movl %edi, %ecx
2324 #xgetbv
2325 .byte 0x0f,0x01,0xd0
2326 shlq $32, %rdx
2327 orq %rdx, %rax
2328 ret
2329 SET_SIZE(get_xcr)
2330
2331 ENTRY(set_xcr)
2332 movq %rsi, %rdx
2333 shrq $32, %rdx
2334 movl %esi, %eax
2335 movl %edi, %ecx
2336 #xsetbv
2337 .byte 0x0f,0x01,0xd1
2338 ret
2339 SET_SIZE(set_xcr)
2340
2341 #elif defined(__i386)
2342
2343 ENTRY(rdmsr)
2344 movl 4(%esp), %ecx
2345 rdmsr
2346 ret
2347 SET_SIZE(rdmsr)
2348
2349 ENTRY(wrmsr)
2350 movl 4(%esp), %ecx
2351 movl 8(%esp), %eax
2352 movl 12(%esp), %edx
2353 wrmsr
2354 ret
2355 SET_SIZE(wrmsr)
2356
2357 ENTRY(xrdmsr)
2358 pushl %ebp
2359 movl %esp, %ebp
2360 movl 8(%esp), %ecx
2361 pushl %edi
2362 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2363 rdmsr
2364 popl %edi
2365 leave
2366 ret
2367 SET_SIZE(xrdmsr)
2368
2369 ENTRY(xwrmsr)
2370 pushl %ebp
2371 movl %esp, %ebp
2372 movl 8(%esp), %ecx
2373 movl 12(%esp), %eax
2374 movl 16(%esp), %edx
2375 pushl %edi
2376 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2377 wrmsr
2378 popl %edi
2379 leave
2380 ret
2381 SET_SIZE(xwrmsr)
2382
2383 ENTRY(get_xcr)
2384 movl 4(%esp), %ecx
2385 #xgetbv
2386 .byte 0x0f,0x01,0xd0
2387 ret
2388 SET_SIZE(get_xcr)
2389
2390 ENTRY(set_xcr)
2391 movl 4(%esp), %ecx
2392 movl 8(%esp), %eax
2393 movl 12(%esp), %edx
2394 #xsetbv
2395 .byte 0x0f,0x01,0xd1
2396 ret
2397 SET_SIZE(set_xcr)
2398
2399 #endif /* __i386 */
2400
2401 ENTRY(invalidate_cache)
2402 wbinvd
2403 ret
2404 SET_SIZE(invalidate_cache)
2405
2406
2407
2408 #if defined(__amd64)
2409
2410 ENTRY_NP(getcregs)
2411 #if defined(__xpv)
2412 /*
2413 * Only a few of the hardware control registers or descriptor tables
2414 * are directly accessible to us, so just zero the structure.
2415 *
2416 * XXPV Perhaps it would be helpful for the hypervisor to return
2417 * virtualized versions of these for post-mortem use.
2418 * (Need to reevaluate - perhaps it already does!)
2419 */
2420 pushq %rdi /* save *crp */
2421 movq $CREGSZ, %rsi
2422 call bzero
2423 popq %rdi
2424
2425 /*
2426 * Dump what limited information we can
2427 */
2428 movq %cr0, %rax
2429 movq %rax, CREG_CR0(%rdi) /* cr0 */
2430 movq %cr2, %rax
2431 movq %rax, CREG_CR2(%rdi) /* cr2 */
2432 movq %cr3, %rax
2433 movq %rax, CREG_CR3(%rdi) /* cr3 */
2434 movq %cr4, %rax
2435 movq %rax, CREG_CR4(%rdi) /* cr4 */
2436
2437 #else /* __xpv */
2438
2439 #define GETMSR(r, off, d) \
2440 movl $r, %ecx; \
2441 rdmsr; \
2442 movl %eax, off(d); \
2443 movl %edx, off+4(d)
2444
2445 xorl %eax, %eax
2446 movq %rax, CREG_GDT+8(%rdi)
2447 sgdt CREG_GDT(%rdi) /* 10 bytes */
2448 movq %rax, CREG_IDT+8(%rdi)
2449 sidt CREG_IDT(%rdi) /* 10 bytes */
2450 movq %rax, CREG_LDT(%rdi)
2451 sldt CREG_LDT(%rdi) /* 2 bytes */
2452 movq %rax, CREG_TASKR(%rdi)
2453 str CREG_TASKR(%rdi) /* 2 bytes */
2454 movq %cr0, %rax
2455 movq %rax, CREG_CR0(%rdi) /* cr0 */
2456 movq %cr2, %rax
2457 movq %rax, CREG_CR2(%rdi) /* cr2 */
2458 movq %cr3, %rax
2459 movq %rax, CREG_CR3(%rdi) /* cr3 */
2460 movq %cr4, %rax
2461 movq %rax, CREG_CR4(%rdi) /* cr4 */
2462 movq %cr8, %rax
2463 movq %rax, CREG_CR8(%rdi) /* cr8 */
2464 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
2465 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
2466 #endif /* __xpv */
2467 ret
2468 SET_SIZE(getcregs)
2469
2470 #undef GETMSR
2471
2472 #elif defined(__i386)
2473
2474 ENTRY_NP(getcregs)
2475 #if defined(__xpv)
2476 /*
2477 * Only a few of the hardware control registers or descriptor tables
2478 * are directly accessible to us, so just zero the structure.
2479 *
2480 * XXPV Perhaps it would be helpful for the hypervisor to return
2481 * virtualized versions of these for post-mortem use.
2482 * (Need to reevaluate - perhaps it already does!)
2483 */
2484 movl 4(%esp), %edx
2485 pushl $CREGSZ
2486 pushl %edx
2487 call bzero
2488 addl $8, %esp
2489 movl 4(%esp), %edx
2490
2491 /*
2492 * Dump what limited information we can
2493 */
2494 movl %cr0, %eax
2495 movl %eax, CREG_CR0(%edx) /* cr0 */
2496 movl %cr2, %eax
2497 movl %eax, CREG_CR2(%edx) /* cr2 */
2498 movl %cr3, %eax
2499 movl %eax, CREG_CR3(%edx) /* cr3 */
2500 movl %cr4, %eax
2501 movl %eax, CREG_CR4(%edx) /* cr4 */
2502
2503 #else /* __xpv */
2504
2505 movl 4(%esp), %edx
2506 movw $0, CREG_GDT+6(%edx)
2507 movw $0, CREG_IDT+6(%edx)
2508 sgdt CREG_GDT(%edx) /* gdt */
2509 sidt CREG_IDT(%edx) /* idt */
2510 sldt CREG_LDT(%edx) /* ldt */
2511 str CREG_TASKR(%edx) /* task */
2512 movl %cr0, %eax
2513 movl %eax, CREG_CR0(%edx) /* cr0 */
2514 movl %cr2, %eax
2515 movl %eax, CREG_CR2(%edx) /* cr2 */
2516 movl %cr3, %eax
2517 movl %eax, CREG_CR3(%edx) /* cr3 */
2518 bt $X86FSET_LARGEPAGE, x86_featureset
2519 jnc .nocr4
2520 movl %cr4, %eax
2521 movl %eax, CREG_CR4(%edx) /* cr4 */
2522 jmp .skip
2523 .nocr4:
2524 movl $0, CREG_CR4(%edx)
2525 .skip:
2526 #endif
2527 ret
2528 SET_SIZE(getcregs)
2529
2530 #endif /* __i386 */
2531
2532
2533 /*
2534 * A panic trigger is a word which is updated atomically and can only be set
2535 * once. We atomically store 0xDEFACEDD and load the old value. If the
2536 * previous value was 0, we succeed and return 1; otherwise return 0.
2537 * This allows a partially corrupt trigger to still trigger correctly. DTrace
2538 * has its own version of this function to allow it to panic correctly from
2539 * probe context.
2540 */
2541
2542 #if defined(__amd64)
2543
2544 ENTRY_NP(panic_trigger)
2545 xorl %eax, %eax
2546 movl $0xdefacedd, %edx
2547 lock
2548 xchgl %edx, (%rdi)
2549 cmpl $0, %edx
2550 je 0f
2551 movl $0, %eax
2552 ret
2553 0: movl $1, %eax
2554 ret
2555 SET_SIZE(panic_trigger)
2556
2557 ENTRY_NP(dtrace_panic_trigger)
2558 xorl %eax, %eax
2559 movl $0xdefacedd, %edx
2560 lock
2561 xchgl %edx, (%rdi)
2562 cmpl $0, %edx
2563 je 0f
2564 movl $0, %eax
2565 ret
2566 0: movl $1, %eax
2567 ret
2568 SET_SIZE(dtrace_panic_trigger)
2569
2570 #elif defined(__i386)
2571
2572 ENTRY_NP(panic_trigger)
2573 movl 4(%esp), %edx / %edx = address of trigger
2574 movl $0xdefacedd, %eax / %eax = 0xdefacedd
2575 lock / assert lock
2576 xchgl %eax, (%edx) / exchange %eax and the trigger
2577 cmpl $0, %eax / if (%eax == 0x0)
2578 je 0f / return (1);
2579 movl $0, %eax / else
2580 ret / return (0);
2581 0: movl $1, %eax
2582 ret
2583 SET_SIZE(panic_trigger)
2584
2585 ENTRY_NP(dtrace_panic_trigger)
2586 movl 4(%esp), %edx / %edx = address of trigger
2587 movl $0xdefacedd, %eax / %eax = 0xdefacedd
2588 lock / assert lock
2589 xchgl %eax, (%edx) / exchange %eax and the trigger
2590 cmpl $0, %eax / if (%eax == 0x0)
2591 je 0f / return (1);
2592 movl $0, %eax / else
2593 ret / return (0);
2594 0: movl $1, %eax
2595 ret
2596 SET_SIZE(dtrace_panic_trigger)
2597
2598 #endif /* __i386 */
2599
2600 /*
2601 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
2602 * into the panic code implemented in panicsys(). vpanic() is responsible
2603 * for passing through the format string and arguments, and constructing a
2604 * regs structure on the stack into which it saves the current register
2605 * values. If we are not dying due to a fatal trap, these registers will
2606 * then be preserved in panicbuf as the current processor state. Before
2607 * invoking panicsys(), vpanic() activates the first panic trigger (see
2608 * kernel/os/panic.c) and switches to the panic_stack if successful. Note that
2609 * DTrace takes a slightly different panic path if it must panic from probe
2610 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
2611 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
2612 * branches back into vpanic().
2613 */
2614
2615 #if defined(__amd64)
2616
2617 ENTRY_NP(vpanic) /* Initial stack layout: */
2618
2619 pushq %rbp /* | %rip | 0x60 */
2620 movq %rsp, %rbp /* | %rbp | 0x58 */
2621 pushfq /* | rfl | 0x50 */
2622 pushq %r11 /* | %r11 | 0x48 */
2623 pushq %r10 /* | %r10 | 0x40 */
2624 pushq %rbx /* | %rbx | 0x38 */
2625 pushq %rax /* | %rax | 0x30 */
2626 pushq %r9 /* | %r9 | 0x28 */
2627 pushq %r8 /* | %r8 | 0x20 */
2628 pushq %rcx /* | %rcx | 0x18 */
2629 pushq %rdx /* | %rdx | 0x10 */
2630 pushq %rsi /* | %rsi | 0x8 alist */
2631 pushq %rdi /* | %rdi | 0x0 format */
2632
2633 movq %rsp, %rbx /* %rbx = current %rsp */
2634
2635 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
2636 call panic_trigger /* %eax = panic_trigger() */
2637
2638 vpanic_common:
2639 /*
2640 * The panic_trigger result is in %eax from the call above, and
2641 * dtrace_panic places it in %eax before branching here.
2642 * The rdmsr instructions that follow below will clobber %eax so
2643 * we stash the panic_trigger result in %r11d.
2644 */
2645 movl %eax, %r11d
2646 cmpl $0, %r11d
2647 je 0f
2648
2649 /*
2650 * If panic_trigger() was successful, we are the first to initiate a
2651 * panic: we now switch to the reserved panic_stack before continuing.
2652 */
2653 leaq panic_stack(%rip), %rsp
2654 addq $PANICSTKSIZE, %rsp
2655 0: subq $REGSIZE, %rsp
2656 /*
2657 * Now that we've got everything set up, store the register values as
2658 * they were when we entered vpanic() to the designated location in
2659 * the regs structure we allocated on the stack.
2660 */
2661 movq 0x0(%rbx), %rcx
2662 movq %rcx, REGOFF_RDI(%rsp)
2663 movq 0x8(%rbx), %rcx
2664 movq %rcx, REGOFF_RSI(%rsp)
2665 movq 0x10(%rbx), %rcx
2666 movq %rcx, REGOFF_RDX(%rsp)
2667 movq 0x18(%rbx), %rcx
2668 movq %rcx, REGOFF_RCX(%rsp)
2669 movq 0x20(%rbx), %rcx
2670
2671 movq %rcx, REGOFF_R8(%rsp)
2672 movq 0x28(%rbx), %rcx
2673 movq %rcx, REGOFF_R9(%rsp)
2674 movq 0x30(%rbx), %rcx
2675 movq %rcx, REGOFF_RAX(%rsp)
2676 movq 0x38(%rbx), %rcx
2677 movq %rcx, REGOFF_RBX(%rsp)
2678 movq 0x58(%rbx), %rcx
2679
2680 movq %rcx, REGOFF_RBP(%rsp)
2681 movq 0x40(%rbx), %rcx
2682 movq %rcx, REGOFF_R10(%rsp)
2683 movq 0x48(%rbx), %rcx
2684 movq %rcx, REGOFF_R11(%rsp)
2685 movq %r12, REGOFF_R12(%rsp)
2686
2687 movq %r13, REGOFF_R13(%rsp)
2688 movq %r14, REGOFF_R14(%rsp)
2689 movq %r15, REGOFF_R15(%rsp)
2690
2691 xorl %ecx, %ecx
2692 movw %ds, %cx
2693 movq %rcx, REGOFF_DS(%rsp)
2694 movw %es, %cx
2695 movq %rcx, REGOFF_ES(%rsp)
2696 movw %fs, %cx
2697 movq %rcx, REGOFF_FS(%rsp)
2698 movw %gs, %cx
2699 movq %rcx, REGOFF_GS(%rsp)
2700
2701 movq $0, REGOFF_TRAPNO(%rsp)
2702
2703 movq $0, REGOFF_ERR(%rsp)
2704 leaq vpanic(%rip), %rcx
2705 movq %rcx, REGOFF_RIP(%rsp)
2706 movw %cs, %cx
2707 movzwq %cx, %rcx
2708 movq %rcx, REGOFF_CS(%rsp)
2709 movq 0x50(%rbx), %rcx
2710 movq %rcx, REGOFF_RFL(%rsp)
2711 movq %rbx, %rcx
2712 addq $0x60, %rcx
2713 movq %rcx, REGOFF_RSP(%rsp)
2714 movw %ss, %cx
2715 movzwq %cx, %rcx
2716 movq %rcx, REGOFF_SS(%rsp)
2717
2718 /*
2719 * panicsys(format, alist, rp, on_panic_stack)
2720 */
2721 movq REGOFF_RDI(%rsp), %rdi /* format */
2722 movq REGOFF_RSI(%rsp), %rsi /* alist */
2723 movq %rsp, %rdx /* struct regs */
2724 movl %r11d, %ecx /* on_panic_stack */
2725 call panicsys
2726 addq $REGSIZE, %rsp
2727 popq %rdi
2728 popq %rsi
2729 popq %rdx
2730 popq %rcx
2731 popq %r8
2732 popq %r9
2733 popq %rax
2734 popq %rbx
2735 popq %r10
2736 popq %r11
2737 popfq
2738 leave
2739 ret
2740 SET_SIZE(vpanic)
2741
2742 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
2743
2744 pushq %rbp /* | %rip | 0x60 */
2745 movq %rsp, %rbp /* | %rbp | 0x58 */
2746 pushfq /* | rfl | 0x50 */
2747 pushq %r11 /* | %r11 | 0x48 */
2748 pushq %r10 /* | %r10 | 0x40 */
2749 pushq %rbx /* | %rbx | 0x38 */
2750 pushq %rax /* | %rax | 0x30 */
2751 pushq %r9 /* | %r9 | 0x28 */
2752 pushq %r8 /* | %r8 | 0x20 */
2753 pushq %rcx /* | %rcx | 0x18 */
2754 pushq %rdx /* | %rdx | 0x10 */
2755 pushq %rsi /* | %rsi | 0x8 alist */
2756 pushq %rdi /* | %rdi | 0x0 format */
2757
2758 movq %rsp, %rbx /* %rbx = current %rsp */
2759
2760 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
2761 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
2762 jmp vpanic_common
2763
2764 SET_SIZE(dtrace_vpanic)
2765
2766 #elif defined(__i386)
2767
2768 ENTRY_NP(vpanic) / Initial stack layout:
2769
2770 pushl %ebp / | %eip | 20
2771 movl %esp, %ebp / | %ebp | 16
2772 pushl %eax / | %eax | 12
2773 pushl %ebx / | %ebx | 8
2774 pushl %ecx / | %ecx | 4
2775 pushl %edx / | %edx | 0
2776
2777 movl %esp, %ebx / %ebx = current stack pointer
2778
2779 lea panic_quiesce, %eax / %eax = &panic_quiesce
2780 pushl %eax / push &panic_quiesce
2781 call panic_trigger / %eax = panic_trigger()
2782 addl $4, %esp / reset stack pointer
2783
2784 vpanic_common:
2785 cmpl $0, %eax / if (%eax == 0)
2786 je 0f / goto 0f;
2787
2788 /*
2789 * If panic_trigger() was successful, we are the first to initiate a
2790 * panic: we now switch to the reserved panic_stack before continuing.
2791 */
2792 lea panic_stack, %esp / %esp = panic_stack
2793 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
2794
2795 0: subl $REGSIZE, %esp / allocate struct regs
2796
2797 /*
2798 * Now that we've got everything set up, store the register values as
2799 * they were when we entered vpanic() to the designated location in
2800 * the regs structure we allocated on the stack.
2801 */
2802 #if !defined(__GNUC_AS__)
2803 movw %gs, %edx
2804 movl %edx, REGOFF_GS(%esp)
2805 movw %fs, %edx
2806 movl %edx, REGOFF_FS(%esp)
2807 movw %es, %edx
2808 movl %edx, REGOFF_ES(%esp)
2809 movw %ds, %edx
2810 movl %edx, REGOFF_DS(%esp)
2811 #else /* __GNUC_AS__ */
2812 mov %gs, %edx
2813 mov %edx, REGOFF_GS(%esp)
2814 mov %fs, %edx
2815 mov %edx, REGOFF_FS(%esp)
2816 mov %es, %edx
2817 mov %edx, REGOFF_ES(%esp)
2818 mov %ds, %edx
2819 mov %edx, REGOFF_DS(%esp)
2820 #endif /* __GNUC_AS__ */
2821 movl %edi, REGOFF_EDI(%esp)
2822 movl %esi, REGOFF_ESI(%esp)
2823 movl 16(%ebx), %ecx
2824 movl %ecx, REGOFF_EBP(%esp)
2825 movl %ebx, %ecx
2826 addl $20, %ecx
2827 movl %ecx, REGOFF_ESP(%esp)
2828 movl 8(%ebx), %ecx
2829 movl %ecx, REGOFF_EBX(%esp)
2830 movl 0(%ebx), %ecx
2831 movl %ecx, REGOFF_EDX(%esp)
2832 movl 4(%ebx), %ecx
2833 movl %ecx, REGOFF_ECX(%esp)
2834 movl 12(%ebx), %ecx
2835 movl %ecx, REGOFF_EAX(%esp)
2836 movl $0, REGOFF_TRAPNO(%esp)
2837 movl $0, REGOFF_ERR(%esp)
2838 lea vpanic, %ecx
2839 movl %ecx, REGOFF_EIP(%esp)
2840 #if !defined(__GNUC_AS__)
2841 movw %cs, %edx
2842 #else /* __GNUC_AS__ */
2843 mov %cs, %edx
2844 #endif /* __GNUC_AS__ */
2845 movl %edx, REGOFF_CS(%esp)
2846 pushfl
2847 popl %ecx
2848 #if defined(__xpv)
2849 /*
2850 * Synthesize the PS_IE bit from the event mask bit
2851 */
2852 CURTHREAD(%edx)
2853 KPREEMPT_DISABLE(%edx)
2854 EVENT_MASK_TO_IE(%edx, %ecx)
2855 CURTHREAD(%edx)
2856 KPREEMPT_ENABLE_NOKP(%edx)
2857 #endif
2858 movl %ecx, REGOFF_EFL(%esp)
2859 movl $0, REGOFF_UESP(%esp)
2860 #if !defined(__GNUC_AS__)
2861 movw %ss, %edx
2862 #else /* __GNUC_AS__ */
2863 mov %ss, %edx
2864 #endif /* __GNUC_AS__ */
2865 movl %edx, REGOFF_SS(%esp)
2866
2867 movl %esp, %ecx / %ecx = &regs
2868 pushl %eax / push on_panic_stack
2869 pushl %ecx / push &regs
2870 movl 12(%ebp), %ecx / %ecx = alist
2871 pushl %ecx / push alist
2872 movl 8(%ebp), %ecx / %ecx = format
2873 pushl %ecx / push format
2874 call panicsys / panicsys();
2875 addl $16, %esp / pop arguments
2876
2877 addl $REGSIZE, %esp
2878 popl %edx
2879 popl %ecx
2880 popl %ebx
2881 popl %eax
2882 leave
2883 ret
2884 SET_SIZE(vpanic)
2885
2886 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
2887
2888 pushl %ebp / | %eip | 20
2889 movl %esp, %ebp / | %ebp | 16
2890 pushl %eax / | %eax | 12
2891 pushl %ebx / | %ebx | 8
2892 pushl %ecx / | %ecx | 4
2893 pushl %edx / | %edx | 0
2894
2895 movl %esp, %ebx / %ebx = current stack pointer
2896
2897 lea panic_quiesce, %eax / %eax = &panic_quiesce
2898 pushl %eax / push &panic_quiesce
2899 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
2900 addl $4, %esp / reset stack pointer
2901 jmp vpanic_common / jump back to common code
2902
2903 SET_SIZE(dtrace_vpanic)
2904
2905 #endif /* __i386 */
2906
2907
2908 DGDEF3(timedelta, 8, 8)
2909 .long 0, 0
2910
2911 /*
2912 * initialized to a non zero value to make pc_gethrtime()
2913 * work correctly even before clock is initialized
2914 */
2915 DGDEF3(hrtime_base, 8, 8)
2916 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
2917
2918 DGDEF3(adj_shift, 4, 4)
2919 .long ADJ_SHIFT
2920
2921 #if defined(__amd64)
2922
2923 ENTRY_NP(hres_tick)
2924 pushq %rbp
2925 movq %rsp, %rbp
2926
2927 /*
2928 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
2929 * hres_last_tick can only be modified while holding CLOCK_LOCK).
2930 * At worst, performing this now instead of under CLOCK_LOCK may
2931 * introduce some jitter in pc_gethrestime().
2932 */
2933 call *gethrtimef(%rip)
2934 movq %rax, %r8
2935
2936 leaq hres_lock(%rip), %rax
2937 movb $-1, %dl
2938 .CL1:
2939 xchgb %dl, (%rax)
2940 testb %dl, %dl
2941 jz .CL3 /* got it */
2942 .CL2:
2943 cmpb $0, (%rax) /* possible to get lock? */
2944 pause
2945 jne .CL2
2946 jmp .CL1 /* yes, try again */
2947 .CL3:
2948 /*
2949 * compute the interval since last time hres_tick was called
2950 * and adjust hrtime_base and hrestime accordingly
2951 * hrtime_base is an 8 byte value (in nsec), hrestime is
2952 * a timestruc_t (sec, nsec)
2953 */
2954 leaq hres_last_tick(%rip), %rax
2955 movq %r8, %r11
2956 subq (%rax), %r8
2957 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
2958 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
2959 /*
2960 * Now that we have CLOCK_LOCK, we can update hres_last_tick
2961 */
2962 movq %r11, (%rax)
2963
2964 call __adj_hrestime
2965
2966 /*
2967 * release the hres_lock
2968 */
2969 incl hres_lock(%rip)
2970 leave
2971 ret
2972 SET_SIZE(hres_tick)
2973
2974 #elif defined(__i386)
2975
2976 ENTRY_NP(hres_tick)
2977 pushl %ebp
2978 movl %esp, %ebp
2979 pushl %esi
2980 pushl %ebx
2981
2982 /*
2983 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
2984 * hres_last_tick can only be modified while holding CLOCK_LOCK).
2985 * At worst, performing this now instead of under CLOCK_LOCK may
2986 * introduce some jitter in pc_gethrestime().
2987 */
2988 call *gethrtimef
2989 movl %eax, %ebx
2990 movl %edx, %esi
2991
2992 movl $hres_lock, %eax
2993 movl $-1, %edx
2994 .CL1:
2995 xchgb %dl, (%eax)
2996 testb %dl, %dl
2997 jz .CL3 / got it
2998 .CL2:
2999 cmpb $0, (%eax) / possible to get lock?
3000 pause
3001 jne .CL2
3002 jmp .CL1 / yes, try again
3003 .CL3:
3004 /*
3005 * compute the interval since last time hres_tick was called
3006 * and adjust hrtime_base and hrestime accordingly
3007 * hrtime_base is an 8 byte value (in nsec), hrestime is
3008 * timestruc_t (sec, nsec)
3009 */
3010
3011 lea hres_last_tick, %eax
3012
3013 movl %ebx, %edx
3014 movl %esi, %ecx
3015
3016 subl (%eax), %edx
3017 sbbl 4(%eax), %ecx
3018
3019 addl %edx, hrtime_base / add interval to hrtime_base
3020 adcl %ecx, hrtime_base+4
3021
3022 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3023
3024 /
3025 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3026 /
3027 movl %ebx, (%eax)
3028 movl %esi, 4(%eax)
3029
3030 / get hrestime at this moment. used as base for pc_gethrestime
3031 /
3032 / Apply adjustment, if any
3033 /
3034 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3035 / (max_hres_adj)
3036 /
3037 / void
3038 / adj_hrestime()
3039 / {
3040 / long long adj;
3041 /
3042 / if (hrestime_adj == 0)
3043 / adj = 0;
3044 / else if (hrestime_adj > 0) {
3045 / if (hrestime_adj < HRES_ADJ)
3046 / adj = hrestime_adj;
3047 / else
3048 / adj = HRES_ADJ;
3049 / }
3050 / else {
3051 / if (hrestime_adj < -(HRES_ADJ))
3052 / adj = -(HRES_ADJ);
3053 / else
3054 / adj = hrestime_adj;
3055 / }
3056 /
3057 / timedelta -= adj;
3058 / hrestime_adj = timedelta;
3059 / hrestime.tv_nsec += adj;
3060 /
3061 / while (hrestime.tv_nsec >= NANOSEC) {
3062 / one_sec++;
3063 / hrestime.tv_sec++;
3064 / hrestime.tv_nsec -= NANOSEC;
3065 / }
3066 / }
3067 __adj_hrestime:
3068 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3069 movl hrestime_adj+4, %edx
3070 andl %esi, %esi
3071 jne .CL4 / no
3072 andl %edx, %edx
3073 jne .CL4 / no
3074 subl %ecx, %ecx / yes, adj = 0;
3075 subl %edx, %edx
3076 jmp .CL5
3077 .CL4:
3078 subl %ecx, %ecx
3079 subl %eax, %eax
3080 subl %esi, %ecx
3081 sbbl %edx, %eax
3082 andl %eax, %eax / if (hrestime_adj > 0)
3083 jge .CL6
3084
3085 / In the following comments, HRES_ADJ is used, while in the code
3086 / max_hres_adj is used.
3087 /
3088 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3089 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3090 / on the logical equivalence of:
3091 /
3092 / !(hrestime_adj < HRES_ADJ)
3093 /
3094 / and the two step sequence:
3095 /
3096 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3097 /
3098 / which computes whether or not the least significant 32-bits
3099 / of hrestime_adj is greater than HRES_ADJ, followed by:
3100 /
3101 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3102 /
3103 / which generates a carry whenever step 1 is true or the most
3104 / significant long of the longlong hrestime_adj is non-zero.
3105
3106 movl max_hres_adj, %ecx / hrestime_adj is positive
3107 subl %esi, %ecx
3108 movl %edx, %eax
3109 adcl $-1, %eax
3110 jnc .CL7
3111 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3112 subl %edx, %edx
3113 jmp .CL5
3114
3115 / The following computation is similar to the one above.
3116 /
3117 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3118 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3119 / on the logical equivalence of:
3120 /
3121 / (hrestime_adj > -HRES_ADJ)
3122 /
3123 / and the two step sequence:
3124 /
3125 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3126 /
3127 / which means the least significant 32-bits of hrestime_adj is
3128 / greater than -HRES_ADJ, followed by:
3129 /
3130 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3131 /
3132 / which generates a carry only when step 1 is true and the most
3133 / significant long of the longlong hrestime_adj is -1.
3134
3135 .CL6: / hrestime_adj is negative
3136 movl %esi, %ecx
3137 addl max_hres_adj, %ecx
3138 movl %edx, %eax
3139 adcl $0, %eax
3140 jc .CL7
3141 xor %ecx, %ecx
3142 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3143 movl $-1, %edx
3144 jmp .CL5
3145 .CL7:
3146 movl %esi, %ecx / adj = hrestime_adj;
3147 .CL5:
3148 movl timedelta, %esi
3149 subl %ecx, %esi
3150 movl timedelta+4, %eax
3151 sbbl %edx, %eax
3152 movl %esi, timedelta
3153 movl %eax, timedelta+4 / timedelta -= adj;
3154 movl %esi, hrestime_adj
3155 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3156 addl hrestime+4, %ecx
3157
3158 movl %ecx, %eax / eax = tv_nsec
3159 1:
3160 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3161 jb .CL8 / no
3162 incl one_sec / yes, one_sec++;
3163 incl hrestime / hrestime.tv_sec++;
3164 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3165 jmp 1b / check for more seconds
3166
3167 .CL8:
3168 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3169 incl hres_lock / release the hres_lock
3170
3171 popl %ebx
3172 popl %esi
3173 leave
3174 ret
3175 SET_SIZE(hres_tick)
3176
3177 #endif /* __i386 */
3178
3179 /*
3180 * void prefetch_smap_w(void *)
3181 *
3182 * Prefetch ahead within a linear list of smap structures.
3183 * Not implemented for ia32. Stub for compatibility.
3184 */
3185
3186
3187 ENTRY(prefetch_smap_w)
3188 rep; ret /* use 2 byte return instruction when branch target */
3189 /* AMD Software Optimization Guide - Section 6.2 */
3190 SET_SIZE(prefetch_smap_w)
3191
3192
3193 /*
3194 * prefetch_page_r(page_t *)
3195 * issue prefetch instructions for a page_t
3196 */
3197
3198 ENTRY(prefetch_page_r)
3199 rep; ret /* use 2 byte return instruction when branch target */
3200 /* AMD Software Optimization Guide - Section 6.2 */
3201 SET_SIZE(prefetch_page_r)
3202
3203
3204
3205 #if defined(__amd64)
3206
3207 ENTRY(bcmp)
3208 pushq %rbp
3209 movq %rsp, %rbp
3210 #ifdef DEBUG
3211 testq %rdx,%rdx
3212 je 1f
3213 movq postbootkernelbase(%rip), %r11
3214 cmpq %r11, %rdi
3215 jb 0f
3216 cmpq %r11, %rsi
3217 jnb 1f
3218 0: leaq .bcmp_panic_msg(%rip), %rdi
3219 xorl %eax, %eax
3220 call panic
3221 1:
3222 #endif /* DEBUG */
3223 call memcmp
3224 testl %eax, %eax
3225 setne %dl
3226 leave
3227 movzbl %dl, %eax
3228 ret
3229 SET_SIZE(bcmp)
3230
3231 #elif defined(__i386)
3232
3233 #define ARG_S1 8
3234 #define ARG_S2 12
3235 #define ARG_LENGTH 16
3236
3237 ENTRY(bcmp)
3238 pushl %ebp
3239 movl %esp, %ebp / create new stack frame
3240 #ifdef DEBUG
3241 cmpl $0, ARG_LENGTH(%ebp)
3242 je 1f
3243 movl postbootkernelbase, %eax
3244 cmpl %eax, ARG_S1(%ebp)
3245 jb 0f
3246 cmpl %eax, ARG_S2(%ebp)
3247 jnb 1f
3248 0: pushl $.bcmp_panic_msg
3249 call panic
3250 1:
3251 #endif /* DEBUG */
3252
3253 pushl %edi / save register variable
3254 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3255 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3256 cmpl %eax, %ecx / if the same string
3257 je .equal / goto .equal
3258 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3259 cmpl $4, %edi / if %edi < 4
3260 jb .byte_check / goto .byte_check
3261 .align 4
3262 .word_loop:
3263 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3264 leal -4(%edi), %edi / %edi -= 4
3265 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3266 jne .word_not_equal / if not equal, goto .word_not_equal
3267 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3268 leal 4(%eax), %eax / %eax += 4 (next word)
3269 cmpl $4, %edi / if %edi >= 4
3270 jae .word_loop / goto .word_loop
3271 .byte_check:
3272 cmpl $0, %edi / if %edi == 0
3273 je .equal / goto .equal
3274 jmp .byte_loop / goto .byte_loop (checks in bytes)
3275 .word_not_equal:
3276 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3277 .align 4
3278 .byte_loop:
3279 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3280 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3281 jne .not_equal / if not equal, goto .not_equal
3282 incl %ecx / %ecx++ (next byte)
3283 incl %eax / %eax++ (next byte)
3284 decl %edi / %edi--
3285 jnz .byte_loop / if not zero, goto .byte_loop
3286 .equal:
3287 xorl %eax, %eax / %eax = 0
3288 popl %edi / restore register variable
3289 leave / restore old stack frame
3290 ret / return (NULL)
3291 .align 4
3292 .not_equal:
3293 movl $1, %eax / return 1
3294 popl %edi / restore register variable
3295 leave / restore old stack frame
3296 ret / return (NULL)
3297 SET_SIZE(bcmp)
3298
3299 #endif /* __i386 */
3300
3301 #ifdef DEBUG
3302 .text
3303 .bcmp_panic_msg:
3304 .string "bcmp: arguments below kernelbase"
3305 #endif /* DEBUG */
3306
3307
3308
3309 #if defined(__amd64)
3310
3311 ENTRY_NP(bsrw_insn)
3312 xorl %eax, %eax
3313 bsrw %di, %ax
3314 ret
3315 SET_SIZE(bsrw_insn)
3316
3317 #elif defined(__i386)
3318
3319 ENTRY_NP(bsrw_insn)
3320 movw 4(%esp), %cx
3321 xorl %eax, %eax
3322 bsrw %cx, %ax
3323 ret
3324 SET_SIZE(bsrw_insn)
3325
3326 #endif /* __i386 */
3327
3328
3329 #if defined(__i386)
3330
3331 ENTRY_NP(atomic_btr32)
3332 movl 4(%esp), %ecx
3333 movl 8(%esp), %edx
3334 xorl %eax, %eax
3335 lock
3336 btrl %edx, (%ecx)
3337 setc %al
3338 ret
3339 SET_SIZE(atomic_btr32)
3340
3341 #endif /* __i386 */
3342
3343
3344 #if defined(__amd64)
3345
3346 ENTRY_NP(switch_sp_and_call)
3347 pushq %rbp
3348 movq %rsp, %rbp /* set up stack frame */
3349 movq %rdi, %rsp /* switch stack pointer */
3350 movq %rdx, %rdi /* pass func arg 1 */
3351 movq %rsi, %r11 /* save function to call */
3352 movq %rcx, %rsi /* pass func arg 2 */
3353 call *%r11 /* call function */
3354 leave /* restore stack */
3355 ret
3356 SET_SIZE(switch_sp_and_call)
3357
3358 #elif defined(__i386)
3359
3360 ENTRY_NP(switch_sp_and_call)
3361 pushl %ebp
3362 mov %esp, %ebp /* set up stack frame */
3363 movl 8(%ebp), %esp /* switch stack pointer */
3364 pushl 20(%ebp) /* push func arg 2 */
3365 pushl 16(%ebp) /* push func arg 1 */
3366 call *12(%ebp) /* call function */
3367 addl $8, %esp /* pop arguments */
3368 leave /* restore stack */
3369 ret
3370 SET_SIZE(switch_sp_and_call)
3371
3372 #endif /* __i386 */
3373
3374
3375 #if defined(__amd64)
3376
3377 ENTRY_NP(kmdb_enter)
3378 pushq %rbp
3379 movq %rsp, %rbp
3380
3381 /*
3382 * Save flags, do a 'cli' then return the saved flags
3383 */
3384 call intr_clear
3385
3386 int $T_DBGENTR
3387
3388 /*
3389 * Restore the saved flags
3390 */
3391 movq %rax, %rdi
3392 call intr_restore
3393
3394 leave
3395 ret
3396 SET_SIZE(kmdb_enter)
3397
3398 #elif defined(__i386)
3399
3400 ENTRY_NP(kmdb_enter)
3401 pushl %ebp
3402 movl %esp, %ebp
3403
3404 /*
3405 * Save flags, do a 'cli' then return the saved flags
3406 */
3407 call intr_clear
3408
3409 int $T_DBGENTR
3410
3411 /*
3412 * Restore the saved flags
3413 */
3414 pushl %eax
3415 call intr_restore
3416 addl $4, %esp
3417
3418 leave
3419 ret
3420 SET_SIZE(kmdb_enter)
3421
3422 #endif /* __i386 */
3423
3424
3425 ENTRY_NP(return_instr)
3426 rep; ret /* use 2 byte instruction when branch target */
3427 /* AMD Software Optimization Guide - Section 6.2 */
3428 SET_SIZE(return_instr)
3429
3430
3431
3432 #if defined(__amd64)
3433
3434 ENTRY(getflags)
3435 pushfq
3436 popq %rax
3437 #if defined(__xpv)
3438 CURTHREAD(%rdi)
3439 KPREEMPT_DISABLE(%rdi)
3440 /*
3441 * Synthesize the PS_IE bit from the event mask bit
3442 */
3443 CURVCPU(%r11)
3444 andq $_BITNOT(PS_IE), %rax
3445 XEN_TEST_UPCALL_MASK(%r11)
3446 jnz 1f
3447 orq $PS_IE, %rax
3448 1:
3449 KPREEMPT_ENABLE_NOKP(%rdi)
3450 #endif
3451 ret
3452 SET_SIZE(getflags)
3453
3454 #elif defined(__i386)
3455
3456 ENTRY(getflags)
3457 pushfl
3458 popl %eax
3459 #if defined(__xpv)
3460 CURTHREAD(%ecx)
3461 KPREEMPT_DISABLE(%ecx)
3462 /*
3463 * Synthesize the PS_IE bit from the event mask bit
3464 */
3465 CURVCPU(%edx)
3466 andl $_BITNOT(PS_IE), %eax
3467 XEN_TEST_UPCALL_MASK(%edx)
3468 jnz 1f
3469 orl $PS_IE, %eax
3470 1:
3471 KPREEMPT_ENABLE_NOKP(%ecx)
3472 #endif
3473 ret
3474 SET_SIZE(getflags)
3475
3476 #endif /* __i386 */
3477
3478
3479
3480 #if defined(__amd64)
3481
3482 ENTRY(ftrace_interrupt_disable)
3483 pushfq
3484 popq %rax
3485 CLI(%rdx)
3486 ret
3487 SET_SIZE(ftrace_interrupt_disable)
3488
3489 #elif defined(__i386)
3490
3491 ENTRY(ftrace_interrupt_disable)
3492 pushfl
3493 popl %eax
3494 CLI(%edx)
3495 ret
3496 SET_SIZE(ftrace_interrupt_disable)
3497
3498 #endif /* __i386 */
3499
3500
3501 #if defined(__amd64)
3502
3503 ENTRY(ftrace_interrupt_enable)
3504 pushq %rdi
3505 popfq
3506 ret
3507 SET_SIZE(ftrace_interrupt_enable)
3508
3509 #elif defined(__i386)
3510
3511 ENTRY(ftrace_interrupt_enable)
3512 movl 4(%esp), %eax
3513 pushl %eax
3514 popfl
3515 ret
3516 SET_SIZE(ftrace_interrupt_enable)
3517
3518 #endif /* __i386 */
3519
3520
3521 #if defined (__amd64)
3522 ENTRY(clflush_insn)
3523 clflush (%rdi)
3524 ret
3525 SET_SIZE(clflush_insn)
3526 #elif defined (__i386)
3527 ENTRY(clflush_insn)
3528 movl 4(%esp), %eax
3529 clflush (%eax)
3530 ret
3531 SET_SIZE(clflush_insn)
3532
3533 #endif /* __i386 */
3534
3535
3536 #if defined (__amd64)
3537 ENTRY(mfence_insn)
3538 mfence
3539 ret
3540 SET_SIZE(mfence_insn)
3541 #elif defined (__i386)
3542 ENTRY(mfence_insn)
3543 mfence
3544 ret
3545 SET_SIZE(mfence_insn)
3546
3547 #endif /* __i386 */
3548
3549 /*
3550 * VMware implements an I/O port that programs can query to detect if software
3551 * is running in a VMware hypervisor. This hypervisor port behaves differently
3552 * depending on magic values in certain registers and modifies some registers
3553 * as a side effect.
3554 *
3555 * References: http://kb.vmware.com/kb/1009458
3556 */
3557
3558
3559 #if defined(__amd64)
3560
3561 ENTRY(vmware_port)
3562 pushq %rbx
3563 movl $VMWARE_HVMAGIC, %eax
3564 movl $0xffffffff, %ebx
3565 movl %edi, %ecx
3566 movl $VMWARE_HVPORT, %edx
3567 inl (%dx)
3568 movl %eax, (%rsi)
3569 movl %ebx, 4(%rsi)
3570 movl %ecx, 8(%rsi)
3571 movl %edx, 12(%rsi)
3572 popq %rbx
3573 ret
3574 SET_SIZE(vmware_port)
3575
3576 #elif defined(__i386)
3577
3578 ENTRY(vmware_port)
3579 pushl %ebx
3580 pushl %esi
3581 movl $VMWARE_HVMAGIC, %eax
3582 movl $0xffffffff, %ebx
3583 movl 12(%esp), %ecx
3584 movl $VMWARE_HVPORT, %edx
3585 inl (%dx)
3586 movl 16(%esp), %esi
3587 movl %eax, (%esi)
3588 movl %ebx, 4(%esi)
3589 movl %ecx, 8(%esi)
3590 movl %edx, 12(%esi)
3591 popl %esi
3592 popl %ebx
3593 ret
3594 SET_SIZE(vmware_port)
3595
3596 #endif /* __i386 */
Unleashed OS