9597 Want hypervisor API for FPU management
[unleashed.git] / usr / src / uts / intel / ia32 / os / archdep.c
blob6f99571a5a187553a0a193f851ab764f038c41f2
1 /*
2 * CDDL HEADER START
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
19 * CDDL HEADER END
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 /* All Rights Reserved */
28 * Copyright (c) 2018, Joyent, Inc.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
32 #include <sys/param.h>
33 #include <sys/types.h>
34 #include <sys/vmparam.h>
35 #include <sys/systm.h>
36 #include <sys/signal.h>
37 #include <sys/stack.h>
38 #include <sys/regset.h>
39 #include <sys/privregs.h>
40 #include <sys/frame.h>
41 #include <sys/proc.h>
42 #include <sys/psw.h>
43 #include <sys/siginfo.h>
44 #include <sys/cpuvar.h>
45 #include <sys/asm_linkage.h>
46 #include <sys/kmem.h>
47 #include <sys/errno.h>
48 #include <sys/bootconf.h>
49 #include <sys/archsystm.h>
50 #include <sys/debug.h>
51 #include <sys/elf.h>
52 #include <sys/spl.h>
53 #include <sys/time.h>
54 #include <sys/atomic.h>
55 #include <sys/sysmacros.h>
56 #include <sys/cmn_err.h>
57 #include <sys/modctl.h>
58 #include <sys/kobj.h>
59 #include <sys/panic.h>
60 #include <sys/reboot.h>
61 #include <sys/time.h>
62 #include <sys/fp.h>
63 #include <sys/x86_archext.h>
64 #include <sys/auxv.h>
65 #include <sys/auxv_386.h>
66 #include <sys/dtrace.h>
67 #include <sys/brand.h>
68 #include <sys/machbrand.h>
69 #include <sys/cmn_err.h>
72 * Map an fnsave-formatted save area into an fxsave-formatted save area.
74 * Most fields are the same width, content and semantics. However
75 * the tag word is compressed.
77 static void
78 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx)
80 uint_t i, tagbits;
82 fx->fx_fcw = fn->f_fcw;
83 fx->fx_fsw = fn->f_fsw;
86 * copy element by element (because of holes)
88 for (i = 0; i < 8; i++)
89 bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0],
90 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
93 * synthesize compressed tag bits
95 fx->fx_fctw = 0;
96 for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2)
97 if ((tagbits & 3) != 3)
98 fx->fx_fctw |= (1 << i);
100 fx->fx_fop = fn->f_fop;
102 #if defined(__amd64)
103 fx->fx_rip = (uint64_t)fn->f_eip;
104 fx->fx_rdp = (uint64_t)fn->f_dp;
105 #else
106 fx->fx_eip = fn->f_eip;
107 fx->fx_cs = fn->f_cs;
108 fx->__fx_ign0 = 0;
109 fx->fx_dp = fn->f_dp;
110 fx->fx_ds = fn->f_ds;
111 fx->__fx_ign1 = 0;
112 #endif
116 * Map from an fxsave-format save area to an fnsave-format save area.
118 static void
119 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn)
121 uint_t i, top, tagbits;
123 fn->f_fcw = fx->fx_fcw;
124 fn->__f_ign0 = 0;
125 fn->f_fsw = fx->fx_fsw;
126 fn->__f_ign1 = 0;
128 top = (fx->fx_fsw & FPS_TOP) >> 11;
131 * copy element by element (because of holes)
133 for (i = 0; i < 8; i++)
134 bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0],
135 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
138 * synthesize uncompressed tag bits
140 fn->f_ftw = 0;
141 for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) {
142 uint_t ibit, expo;
143 const uint16_t *fpp;
144 static const uint16_t zero[5] = { 0, 0, 0, 0, 0 };
146 if ((tagbits & 1) == 0) {
147 fn->f_ftw |= 3 << (i << 1); /* empty */
148 continue;
152 * (tags refer to *physical* registers)
154 fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0];
155 ibit = fpp[3] >> 15;
156 expo = fpp[4] & 0x7fff;
158 if (ibit && expo != 0 && expo != 0x7fff)
159 continue; /* valid fp number */
161 if (bcmp(fpp, &zero, sizeof (zero)))
162 fn->f_ftw |= 2 << (i << 1); /* NaN */
163 else
164 fn->f_ftw |= 1 << (i << 1); /* fp zero */
167 fn->f_fop = fx->fx_fop;
169 fn->__f_ign2 = 0;
170 #if defined(__amd64)
171 fn->f_eip = (uint32_t)fx->fx_rip;
172 fn->f_cs = U32CS_SEL;
173 fn->f_dp = (uint32_t)fx->fx_rdp;
174 fn->f_ds = UDS_SEL;
175 #else
176 fn->f_eip = fx->fx_eip;
177 fn->f_cs = fx->fx_cs;
178 fn->f_dp = fx->fx_dp;
179 fn->f_ds = fx->fx_ds;
180 #endif
181 fn->__f_ign3 = 0;
185 * Map from an fpregset_t into an fxsave-format save area
187 static void
188 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx)
190 #if defined(__amd64)
191 bcopy(fp, fx, sizeof (*fx));
192 #else
193 const struct _fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
195 fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
196 fx->fx_mxcsr = fc->mxcsr;
197 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
198 #endif
200 * avoid useless #gp exceptions - mask reserved bits
202 fx->fx_mxcsr &= sse_mxcsr_mask;
206 * Map from an fxsave-format save area into a fpregset_t
208 static void
209 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp)
211 #if defined(__amd64)
212 bcopy(fx, fp, sizeof (*fx));
213 #else
214 struct _fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
216 fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
217 fc->mxcsr = fx->fx_mxcsr;
218 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
219 #endif
222 #if defined(_SYSCALL32_IMPL)
223 static void
224 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx)
226 const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
228 fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
230 * avoid useless #gp exceptions - mask reserved bits
232 fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr;
233 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
236 static void
237 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp)
239 struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
241 fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
242 fc->mxcsr = fx->fx_mxcsr;
243 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
246 static void
247 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst)
249 fxsave_to_fpregset32((struct fxsave_state *)src, dst);
250 dst->fp_reg_set.fpchip_state.status =
251 src->fp_reg_set.fpchip_state.status;
252 dst->fp_reg_set.fpchip_state.xstatus =
253 src->fp_reg_set.fpchip_state.xstatus;
256 static void
257 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst)
259 fpregset32_to_fxsave(src, (struct fxsave_state *)dst);
260 dst->fp_reg_set.fpchip_state.status =
261 src->fp_reg_set.fpchip_state.status;
262 dst->fp_reg_set.fpchip_state.xstatus =
263 src->fp_reg_set.fpchip_state.xstatus;
265 #endif
268 * Set floating-point registers from a native fpregset_t.
270 void
271 setfpregs(klwp_t *lwp, fpregset_t *fp)
273 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
275 if (fpu->fpu_flags & FPU_EN) {
276 if (!(fpu->fpu_flags & FPU_VALID)) {
278 * FPU context is still active, release the
279 * ownership.
281 fp_free(fpu, 0);
285 * Else: if we are trying to change the FPU state of a thread which
286 * hasn't yet initialized floating point, store the state in
287 * the pcb and indicate that the state is valid. When the
288 * thread enables floating point, it will use this state instead
289 * of the default state.
292 switch (fp_save_mech) {
293 #if defined(__i386)
294 case FP_FNSAVE:
295 bcopy(fp, fpu->fpu_regs.kfpu_u.kfpu_fn,
296 sizeof (*fpu->fpu_regs.kfpu_u.kfpu_fn));
297 break;
298 #endif
299 case FP_FXSAVE:
300 fpregset_to_fxsave(fp, fpu->fpu_regs.kfpu_u.kfpu_fx);
301 fpu->fpu_regs.kfpu_xstatus =
302 fp->fp_reg_set.fpchip_state.xstatus;
303 break;
305 case FP_XSAVE:
306 fpregset_to_fxsave(fp,
307 &fpu->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave);
308 fpu->fpu_regs.kfpu_xstatus =
309 fp->fp_reg_set.fpchip_state.xstatus;
310 fpu->fpu_regs.kfpu_u.kfpu_xs->xs_xstate_bv |=
311 (XFEATURE_LEGACY_FP | XFEATURE_SSE);
312 break;
313 default:
314 panic("Invalid fp_save_mech");
315 /*NOTREACHED*/
318 fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
319 fpu->fpu_flags |= FPU_VALID;
323 * Get floating-point registers into a native fpregset_t.
325 void
326 getfpregs(klwp_t *lwp, fpregset_t *fp)
328 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
330 kpreempt_disable();
331 if (fpu->fpu_flags & FPU_EN) {
333 * If we have FPU hw and the thread's pcb doesn't have
334 * a valid FPU state then get the state from the hw.
336 if (fpu_exists && ttolwp(curthread) == lwp &&
337 !(fpu->fpu_flags & FPU_VALID))
338 fp_save(fpu); /* get the current FPU state */
342 * There are 3 possible cases we have to be aware of here:
344 * 1. FPU is enabled. FPU state is stored in the current LWP.
346 * 2. FPU is not enabled, and there have been no intervening /proc
347 * modifications. Return initial FPU state.
349 * 3. FPU is not enabled, but a /proc consumer has modified FPU state.
350 * FPU state is stored in the current LWP.
352 if ((fpu->fpu_flags & FPU_EN) || (fpu->fpu_flags & FPU_VALID)) {
354 * Cases 1 and 3.
356 switch (fp_save_mech) {
357 #if defined(__i386)
358 case FP_FNSAVE:
359 bcopy(fpu->fpu_regs.kfpu_u.kfpu_fn, fp,
360 sizeof (*fpu->fpu_regs.kfpu_u.kfpu_fn));
361 break;
362 #endif
363 case FP_FXSAVE:
364 fxsave_to_fpregset(fpu->fpu_regs.kfpu_u.kfpu_fx, fp);
365 fp->fp_reg_set.fpchip_state.xstatus =
366 fpu->fpu_regs.kfpu_xstatus;
367 break;
368 case FP_XSAVE:
369 fxsave_to_fpregset(
370 &fpu->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave, fp);
371 fp->fp_reg_set.fpchip_state.xstatus =
372 fpu->fpu_regs.kfpu_xstatus;
373 break;
374 default:
375 panic("Invalid fp_save_mech");
376 /*NOTREACHED*/
378 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
379 } else {
381 * Case 2.
383 switch (fp_save_mech) {
384 #if defined(__i386)
385 case FP_FNSAVE:
386 bcopy(&x87_initial, fp, sizeof (x87_initial));
387 break;
388 #endif
389 case FP_FXSAVE:
390 case FP_XSAVE:
392 * For now, we don't have any AVX specific field in ABI.
393 * If we add any in the future, we need to initial them
394 * as well.
396 fxsave_to_fpregset(&sse_initial, fp);
397 fp->fp_reg_set.fpchip_state.xstatus =
398 fpu->fpu_regs.kfpu_xstatus;
399 break;
400 default:
401 panic("Invalid fp_save_mech");
402 /*NOTREACHED*/
404 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
406 kpreempt_enable();
409 #if defined(_SYSCALL32_IMPL)
412 * Set floating-point registers from an fpregset32_t.
414 void
415 setfpregs32(klwp_t *lwp, fpregset32_t *fp)
417 fpregset_t fpregs;
419 fpregset_32ton(fp, &fpregs);
420 setfpregs(lwp, &fpregs);
424 * Get floating-point registers into an fpregset32_t.
426 void
427 getfpregs32(klwp_t *lwp, fpregset32_t *fp)
429 fpregset_t fpregs;
431 getfpregs(lwp, &fpregs);
432 fpregset_nto32(&fpregs, fp);
435 #endif /* _SYSCALL32_IMPL */
438 * Return the general registers
440 void
441 getgregs(klwp_t *lwp, gregset_t grp)
443 struct regs *rp = lwptoregs(lwp);
444 #if defined(__amd64)
445 struct pcb *pcb = &lwp->lwp_pcb;
446 int thisthread = lwptot(lwp) == curthread;
448 grp[REG_RDI] = rp->r_rdi;
449 grp[REG_RSI] = rp->r_rsi;
450 grp[REG_RDX] = rp->r_rdx;
451 grp[REG_RCX] = rp->r_rcx;
452 grp[REG_R8] = rp->r_r8;
453 grp[REG_R9] = rp->r_r9;
454 grp[REG_RAX] = rp->r_rax;
455 grp[REG_RBX] = rp->r_rbx;
456 grp[REG_RBP] = rp->r_rbp;
457 grp[REG_R10] = rp->r_r10;
458 grp[REG_R11] = rp->r_r11;
459 grp[REG_R12] = rp->r_r12;
460 grp[REG_R13] = rp->r_r13;
461 grp[REG_R14] = rp->r_r14;
462 grp[REG_R15] = rp->r_r15;
463 grp[REG_FSBASE] = pcb->pcb_fsbase;
464 grp[REG_GSBASE] = pcb->pcb_gsbase;
465 if (thisthread)
466 kpreempt_disable();
467 if (pcb->pcb_rupdate == 1) {
468 grp[REG_DS] = pcb->pcb_ds;
469 grp[REG_ES] = pcb->pcb_es;
470 grp[REG_FS] = pcb->pcb_fs;
471 grp[REG_GS] = pcb->pcb_gs;
472 } else {
473 grp[REG_DS] = rp->r_ds;
474 grp[REG_ES] = rp->r_es;
475 grp[REG_FS] = rp->r_fs;
476 grp[REG_GS] = rp->r_gs;
478 if (thisthread)
479 kpreempt_enable();
480 grp[REG_TRAPNO] = rp->r_trapno;
481 grp[REG_ERR] = rp->r_err;
482 grp[REG_RIP] = rp->r_rip;
483 grp[REG_CS] = rp->r_cs;
484 grp[REG_SS] = rp->r_ss;
485 grp[REG_RFL] = rp->r_rfl;
486 grp[REG_RSP] = rp->r_rsp;
487 #else
488 bcopy(&rp->r_gs, grp, sizeof (gregset_t));
489 #endif
492 #if defined(_SYSCALL32_IMPL)
494 void
495 getgregs32(klwp_t *lwp, gregset32_t grp)
497 struct regs *rp = lwptoregs(lwp);
498 struct pcb *pcb = &lwp->lwp_pcb;
499 int thisthread = lwptot(lwp) == curthread;
501 if (thisthread)
502 kpreempt_disable();
503 if (pcb->pcb_rupdate == 1) {
504 grp[GS] = (uint16_t)pcb->pcb_gs;
505 grp[FS] = (uint16_t)pcb->pcb_fs;
506 grp[DS] = (uint16_t)pcb->pcb_ds;
507 grp[ES] = (uint16_t)pcb->pcb_es;
508 } else {
509 grp[GS] = (uint16_t)rp->r_gs;
510 grp[FS] = (uint16_t)rp->r_fs;
511 grp[DS] = (uint16_t)rp->r_ds;
512 grp[ES] = (uint16_t)rp->r_es;
514 if (thisthread)
515 kpreempt_enable();
516 grp[EDI] = (greg32_t)rp->r_rdi;
517 grp[ESI] = (greg32_t)rp->r_rsi;
518 grp[EBP] = (greg32_t)rp->r_rbp;
519 grp[ESP] = 0;
520 grp[EBX] = (greg32_t)rp->r_rbx;
521 grp[EDX] = (greg32_t)rp->r_rdx;
522 grp[ECX] = (greg32_t)rp->r_rcx;
523 grp[EAX] = (greg32_t)rp->r_rax;
524 grp[TRAPNO] = (greg32_t)rp->r_trapno;
525 grp[ERR] = (greg32_t)rp->r_err;
526 grp[EIP] = (greg32_t)rp->r_rip;
527 grp[CS] = (uint16_t)rp->r_cs;
528 grp[EFL] = (greg32_t)rp->r_rfl;
529 grp[UESP] = (greg32_t)rp->r_rsp;
530 grp[SS] = (uint16_t)rp->r_ss;
533 void
534 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst)
536 mcontext_t *dmc = &dst->uc_mcontext;
537 const mcontext32_t *smc = &src->uc_mcontext;
539 bzero(dst, sizeof (*dst));
540 dst->uc_flags = src->uc_flags;
541 dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link;
543 bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask));
545 dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp;
546 dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size;
547 dst->uc_stack.ss_flags = src->uc_stack.ss_flags;
549 dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS];
550 dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS];
551 dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES];
552 dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS];
553 dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI];
554 dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI];
555 dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP];
556 dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX];
557 dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX];
558 dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX];
559 dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX];
560 dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO];
561 dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR];
562 dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP];
563 dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS];
564 dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL];
565 dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP];
566 dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS];
569 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set
570 * otherwise there is no guarantee that anything in fpregs is valid.
572 if (src->uc_flags & UC_FPU)
573 fpregset_32ton(&src->uc_mcontext.fpregs,
574 &dst->uc_mcontext.fpregs);
577 #endif /* _SYSCALL32_IMPL */
580 * Return the user-level PC.
581 * If in a system call, return the address of the syscall trap.
583 greg_t
584 getuserpc()
586 greg_t upc = lwptoregs(ttolwp(curthread))->r_pc;
587 uint32_t insn;
589 if (curthread->t_sysnum == 0)
590 return (upc);
593 * We might've gotten here from sysenter (0xf 0x34),
594 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0).
596 * Go peek at the binary to figure it out..
598 if (fuword32((void *)(upc - 2), &insn) != -1 &&
599 (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f)
600 return (upc - 2);
601 return (upc - 7);
605 * Protect segment registers from non-user privilege levels and GDT selectors
606 * other than USER_CS, USER_DS and lwp FS and GS values. If the segment
607 * selector is non-null and not USER_CS/USER_DS, we make sure that the
608 * TI bit is set to point into the LDT and that the RPL is set to 3.
610 * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we
611 * also explicitly zero the top 16 bits since they may be coming from the
612 * user's address space via setcontext(2) or /proc.
614 * Note about null selector. When running on the hypervisor if we allow a
615 * process to set its %cs to null selector with RPL of 0 the hypervisor will
616 * crash the domain. If running on bare metal we would get a #gp fault and
617 * be able to kill the process and continue on. Therefore we make sure to
618 * force RPL to SEL_UPL even for null selector when setting %cs.
621 #if defined(IS_CS) || defined(IS_NOT_CS)
622 #error "IS_CS and IS_NOT_CS already defined"
623 #endif
625 #define IS_CS 1
626 #define IS_NOT_CS 0
628 /*ARGSUSED*/
629 static greg_t
630 fix_segreg(greg_t sr, int iscs, model_t datamodel)
632 switch (sr &= 0xffff) {
634 case 0:
635 if (iscs == IS_CS)
636 return (0 | SEL_UPL);
637 else
638 return (0);
640 #if defined(__amd64)
642 * If lwp attempts to switch data model then force their
643 * code selector to be null selector.
645 case U32CS_SEL:
646 if (datamodel == DATAMODEL_NATIVE)
647 return (0 | SEL_UPL);
648 else
649 return (sr);
651 case UCS_SEL:
652 if (datamodel == DATAMODEL_ILP32)
653 return (0 | SEL_UPL);
654 #elif defined(__i386)
655 case UCS_SEL:
656 #endif
657 /*FALLTHROUGH*/
658 case UDS_SEL:
659 case LWPFS_SEL:
660 case LWPGS_SEL:
661 case SEL_UPL:
662 return (sr);
663 default:
664 break;
668 * Force it into the LDT in ring 3 for 32-bit processes, which by
669 * default do not have an LDT, so that any attempt to use an invalid
670 * selector will reference the (non-existant) LDT, and cause a #gp
671 * fault for the process.
673 * 64-bit processes get the null gdt selector since they
674 * are not allowed to have a private LDT.
676 #if defined(__amd64)
677 if (datamodel == DATAMODEL_ILP32) {
678 return (sr | SEL_TI_LDT | SEL_UPL);
679 } else {
680 if (iscs == IS_CS)
681 return (0 | SEL_UPL);
682 else
683 return (0);
686 #elif defined(__i386)
687 return (sr | SEL_TI_LDT | SEL_UPL);
688 #endif
692 * Set general registers.
694 void
695 setgregs(klwp_t *lwp, gregset_t grp)
697 struct regs *rp = lwptoregs(lwp);
698 model_t datamodel = lwp_getdatamodel(lwp);
700 #if defined(__amd64)
701 struct pcb *pcb = &lwp->lwp_pcb;
702 int thisthread = lwptot(lwp) == curthread;
704 if (datamodel == DATAMODEL_NATIVE) {
706 if (thisthread)
707 (void) save_syscall_args(); /* copy the args */
709 rp->r_rdi = grp[REG_RDI];
710 rp->r_rsi = grp[REG_RSI];
711 rp->r_rdx = grp[REG_RDX];
712 rp->r_rcx = grp[REG_RCX];
713 rp->r_r8 = grp[REG_R8];
714 rp->r_r9 = grp[REG_R9];
715 rp->r_rax = grp[REG_RAX];
716 rp->r_rbx = grp[REG_RBX];
717 rp->r_rbp = grp[REG_RBP];
718 rp->r_r10 = grp[REG_R10];
719 rp->r_r11 = grp[REG_R11];
720 rp->r_r12 = grp[REG_R12];
721 rp->r_r13 = grp[REG_R13];
722 rp->r_r14 = grp[REG_R14];
723 rp->r_r15 = grp[REG_R15];
724 rp->r_trapno = grp[REG_TRAPNO];
725 rp->r_err = grp[REG_ERR];
726 rp->r_rip = grp[REG_RIP];
728 * Setting %cs or %ss to anything else is quietly but
729 * quite definitely forbidden!
731 rp->r_cs = UCS_SEL;
732 rp->r_ss = UDS_SEL;
733 rp->r_rsp = grp[REG_RSP];
735 if (thisthread)
736 kpreempt_disable();
738 pcb->pcb_ds = UDS_SEL;
739 pcb->pcb_es = UDS_SEL;
742 * 64-bit processes -are- allowed to set their fsbase/gsbase
743 * values directly, but only if they're using the segment
744 * selectors that allow that semantic.
746 * (32-bit processes must use lwp_set_private().)
748 pcb->pcb_fsbase = grp[REG_FSBASE];
749 pcb->pcb_gsbase = grp[REG_GSBASE];
750 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
751 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
754 * Ensure that we go out via update_sregs
756 pcb->pcb_rupdate = 1;
757 lwptot(lwp)->t_post_sys = 1;
758 if (thisthread)
759 kpreempt_enable();
760 #if defined(_SYSCALL32_IMPL)
761 } else {
762 rp->r_rdi = (uint32_t)grp[REG_RDI];
763 rp->r_rsi = (uint32_t)grp[REG_RSI];
764 rp->r_rdx = (uint32_t)grp[REG_RDX];
765 rp->r_rcx = (uint32_t)grp[REG_RCX];
766 rp->r_rax = (uint32_t)grp[REG_RAX];
767 rp->r_rbx = (uint32_t)grp[REG_RBX];
768 rp->r_rbp = (uint32_t)grp[REG_RBP];
769 rp->r_trapno = (uint32_t)grp[REG_TRAPNO];
770 rp->r_err = (uint32_t)grp[REG_ERR];
771 rp->r_rip = (uint32_t)grp[REG_RIP];
773 rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel);
774 rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
776 rp->r_rsp = (uint32_t)grp[REG_RSP];
778 if (thisthread)
779 kpreempt_disable();
781 pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
782 pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel);
785 * (See fsbase/gsbase commentary above)
787 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
788 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
791 * Ensure that we go out via update_sregs
793 pcb->pcb_rupdate = 1;
794 lwptot(lwp)->t_post_sys = 1;
795 if (thisthread)
796 kpreempt_enable();
797 #endif
801 * Only certain bits of the flags register can be modified.
803 rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) |
804 (grp[REG_RFL] & PSL_USERMASK);
806 #elif defined(__i386)
809 * Only certain bits of the flags register can be modified.
811 grp[EFL] = (rp->r_efl & ~PSL_USERMASK) | (grp[EFL] & PSL_USERMASK);
814 * Copy saved registers from user stack.
816 bcopy(grp, &rp->r_gs, sizeof (gregset_t));
818 rp->r_cs = fix_segreg(rp->r_cs, IS_CS, datamodel);
819 rp->r_ss = fix_segreg(rp->r_ss, IS_NOT_CS, datamodel);
820 rp->r_ds = fix_segreg(rp->r_ds, IS_NOT_CS, datamodel);
821 rp->r_es = fix_segreg(rp->r_es, IS_NOT_CS, datamodel);
822 rp->r_fs = fix_segreg(rp->r_fs, IS_NOT_CS, datamodel);
823 rp->r_gs = fix_segreg(rp->r_gs, IS_NOT_CS, datamodel);
825 #endif /* __i386 */
829 * Determine whether eip is likely to have an interrupt frame
830 * on the stack. We do this by comparing the address to the
831 * range of addresses spanned by several well-known routines.
833 extern void _interrupt();
834 extern void _allsyscalls();
835 extern void _cmntrap();
836 extern void fakesoftint();
838 extern size_t _interrupt_size;
839 extern size_t _allsyscalls_size;
840 extern size_t _cmntrap_size;
841 extern size_t _fakesoftint_size;
844 * Get a pc-only stacktrace. Used for kmem_alloc() buffer ownership tracking.
845 * Returns MIN(current stack depth, pcstack_limit).
848 getpcstack(pc_t *pcstack, int pcstack_limit)
850 struct frame *fp = (struct frame *)getfp();
851 struct frame *nextfp, *minfp, *stacktop;
852 int depth = 0;
853 int on_intr;
854 uintptr_t pc;
856 if ((on_intr = CPU_ON_INTR(CPU)) != 0)
857 stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
858 else
859 stacktop = (struct frame *)curthread->t_stk;
860 minfp = fp;
862 pc = ((struct regs *)fp)->r_pc;
864 while (depth < pcstack_limit) {
865 nextfp = (struct frame *)fp->fr_savfp;
866 pc = fp->fr_savpc;
867 if (nextfp <= minfp || nextfp >= stacktop) {
868 if (on_intr) {
870 * Hop from interrupt stack to thread stack.
872 stacktop = (struct frame *)curthread->t_stk;
873 minfp = (struct frame *)curthread->t_stkbase;
874 on_intr = 0;
875 continue;
877 break;
879 pcstack[depth++] = (pc_t)pc;
880 fp = nextfp;
881 minfp = fp;
883 return (depth);
887 * The following ELF header fields are defined as processor-specific
888 * in the V8 ABI:
890 * e_ident[EI_DATA] encoding of the processor-specific
891 * data in the object file
892 * e_machine processor identification
893 * e_flags processor-specific flags associated
894 * with the file
898 * The value of at_flags reflects a platform's cpu module support.
899 * at_flags is used to check for allowing a binary to execute and
900 * is passed as the value of the AT_FLAGS auxiliary vector.
902 int at_flags = 0;
905 * Check the processor-specific fields of an ELF header.
907 * returns 1 if the fields are valid, 0 otherwise
909 /*ARGSUSED2*/
911 elfheadcheck(
912 unsigned char e_data,
913 Elf32_Half e_machine,
914 Elf32_Word e_flags)
916 if (e_data != ELFDATA2LSB)
917 return (0);
918 #if defined(__amd64)
919 if (e_machine == EM_AMD64)
920 return (1);
921 #endif
922 return (e_machine == EM_386);
925 uint_t auxv_hwcap_include = 0; /* patch to enable unrecognized features */
926 uint_t auxv_hwcap_include_2 = 0; /* second word */
927 uint_t auxv_hwcap_exclude = 0; /* patch for broken cpus, debugging */
928 uint_t auxv_hwcap_exclude_2 = 0; /* second word */
929 #if defined(_SYSCALL32_IMPL)
930 uint_t auxv_hwcap32_include = 0; /* ditto for 32-bit apps */
931 uint_t auxv_hwcap32_include_2 = 0; /* ditto for 32-bit apps */
932 uint_t auxv_hwcap32_exclude = 0; /* ditto for 32-bit apps */
933 uint_t auxv_hwcap32_exclude_2 = 0; /* ditto for 32-bit apps */
934 #endif
937 * Gather information about the processor and place it into auxv_hwcap
938 * so that it can be exported to the linker via the aux vector.
940 * We use this seemingly complicated mechanism so that we can ensure
941 * that /etc/system can be used to override what the system can or
942 * cannot discover for itself.
944 void
945 bind_hwcap(void)
947 uint_t cpu_hwcap_flags[2];
948 cpuid_pass4(NULL, cpu_hwcap_flags);
950 auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags[0]) &
951 ~auxv_hwcap_exclude;
952 auxv_hwcap_2 = (auxv_hwcap_include_2 | cpu_hwcap_flags[1]) &
953 ~auxv_hwcap_exclude_2;
955 #if defined(__amd64)
957 * On AMD processors, sysenter just doesn't work at all
958 * when the kernel is in long mode. On IA-32e processors
959 * it does, but there's no real point in all the alternate
960 * mechanism when syscall works on both.
962 * Besides, the kernel's sysenter handler is expecting a
963 * 32-bit lwp ...
965 auxv_hwcap &= ~AV_386_SEP;
966 #else
968 * 32-bit processes can -always- use the lahf/sahf instructions
970 auxv_hwcap |= AV_386_AHF;
971 #endif
973 if (auxv_hwcap_include || auxv_hwcap_exclude || auxv_hwcap_include_2 ||
974 auxv_hwcap_exclude_2) {
976 * The below assignment is regrettably required to get lint
977 * to accept the validity of our format string. The format
978 * string is in fact valid, but whatever intelligence in lint
979 * understands the cmn_err()-specific %b appears to have an
980 * off-by-one error: it (mistakenly) complains about bit
981 * number 32 (even though this is explicitly permitted).
982 * Normally, one would will away such warnings with a "LINTED"
983 * directive, but for reasons unclear and unknown, lint
984 * refuses to be assuaged in this case. Fortunately, lint
985 * doesn't pretend to have solved the Halting Problem --
986 * and as soon as the format string is programmatic, it
987 * knows enough to shut up.
989 char *fmt = "?user ABI extensions: %b\n";
990 cmn_err(CE_CONT, fmt, auxv_hwcap, FMT_AV_386);
991 fmt = "?user ABI extensions (word 2): %b\n";
992 cmn_err(CE_CONT, fmt, auxv_hwcap_2, FMT_AV_386_2);
995 #if defined(_SYSCALL32_IMPL)
996 auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags[0]) &
997 ~auxv_hwcap32_exclude;
998 auxv_hwcap32_2 = (auxv_hwcap32_include_2 | cpu_hwcap_flags[1]) &
999 ~auxv_hwcap32_exclude_2;
1001 #if defined(__amd64)
1003 * If this is an amd64 architecture machine from Intel, then
1004 * syscall -doesn't- work in compatibility mode, only sysenter does.
1006 * Sigh.
1008 if (!cpuid_syscall32_insn(NULL))
1009 auxv_hwcap32 &= ~AV_386_AMD_SYSC;
1012 * 32-bit processes can -always- use the lahf/sahf instructions
1014 auxv_hwcap32 |= AV_386_AHF;
1015 #endif
1017 if (auxv_hwcap32_include || auxv_hwcap32_exclude ||
1018 auxv_hwcap32_include_2 || auxv_hwcap32_exclude_2) {
1020 * See the block comment in the cmn_err() of auxv_hwcap, above.
1022 char *fmt = "?32-bit user ABI extensions: %b\n";
1023 cmn_err(CE_CONT, fmt, auxv_hwcap32, FMT_AV_386);
1024 fmt = "?32-bit user ABI extensions (word 2): %b\n";
1025 cmn_err(CE_CONT, fmt, auxv_hwcap32_2, FMT_AV_386_2);
1027 #endif
1031 * sync_icache() - this is called
1032 * in proc/fs/prusrio.c. x86 has an unified cache and therefore
1033 * this is a nop.
1035 /* ARGSUSED */
1036 void
1037 sync_icache(caddr_t addr, uint_t len)
1039 /* Do nothing for now */
1042 /*ARGSUSED*/
1043 void
1044 sync_data_memory(caddr_t va, size_t len)
1046 /* Not implemented for this platform */
1050 __ipltospl(int ipl)
1052 return (ipltospl(ipl));
1056 * The panic code invokes panic_saveregs() to record the contents of a
1057 * regs structure into the specified panic_data structure for debuggers.
1059 void
1060 panic_saveregs(panic_data_t *pdp, struct regs *rp)
1062 panic_nv_t *pnv = PANICNVGET(pdp);
1064 struct cregs creg;
1066 getcregs(&creg);
1068 #if defined(__amd64)
1069 PANICNVADD(pnv, "rdi", rp->r_rdi);
1070 PANICNVADD(pnv, "rsi", rp->r_rsi);
1071 PANICNVADD(pnv, "rdx", rp->r_rdx);
1072 PANICNVADD(pnv, "rcx", rp->r_rcx);
1073 PANICNVADD(pnv, "r8", rp->r_r8);
1074 PANICNVADD(pnv, "r9", rp->r_r9);
1075 PANICNVADD(pnv, "rax", rp->r_rax);
1076 PANICNVADD(pnv, "rbx", rp->r_rbx);
1077 PANICNVADD(pnv, "rbp", rp->r_rbp);
1078 PANICNVADD(pnv, "r10", rp->r_r10);
1079 PANICNVADD(pnv, "r11", rp->r_r11);
1080 PANICNVADD(pnv, "r12", rp->r_r12);
1081 PANICNVADD(pnv, "r13", rp->r_r13);
1082 PANICNVADD(pnv, "r14", rp->r_r14);
1083 PANICNVADD(pnv, "r15", rp->r_r15);
1084 PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE));
1085 PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE));
1086 PANICNVADD(pnv, "ds", rp->r_ds);
1087 PANICNVADD(pnv, "es", rp->r_es);
1088 PANICNVADD(pnv, "fs", rp->r_fs);
1089 PANICNVADD(pnv, "gs", rp->r_gs);
1090 PANICNVADD(pnv, "trapno", rp->r_trapno);
1091 PANICNVADD(pnv, "err", rp->r_err);
1092 PANICNVADD(pnv, "rip", rp->r_rip);
1093 PANICNVADD(pnv, "cs", rp->r_cs);
1094 PANICNVADD(pnv, "rflags", rp->r_rfl);
1095 PANICNVADD(pnv, "rsp", rp->r_rsp);
1096 PANICNVADD(pnv, "ss", rp->r_ss);
1097 PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3]));
1098 PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0]));
1099 PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3]));
1100 PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0]));
1101 #elif defined(__i386)
1102 PANICNVADD(pnv, "gs", (uint32_t)rp->r_gs);
1103 PANICNVADD(pnv, "fs", (uint32_t)rp->r_fs);
1104 PANICNVADD(pnv, "es", (uint32_t)rp->r_es);
1105 PANICNVADD(pnv, "ds", (uint32_t)rp->r_ds);
1106 PANICNVADD(pnv, "edi", (uint32_t)rp->r_edi);
1107 PANICNVADD(pnv, "esi", (uint32_t)rp->r_esi);
1108 PANICNVADD(pnv, "ebp", (uint32_t)rp->r_ebp);
1109 PANICNVADD(pnv, "esp", (uint32_t)rp->r_esp);
1110 PANICNVADD(pnv, "ebx", (uint32_t)rp->r_ebx);
1111 PANICNVADD(pnv, "edx", (uint32_t)rp->r_edx);
1112 PANICNVADD(pnv, "ecx", (uint32_t)rp->r_ecx);
1113 PANICNVADD(pnv, "eax", (uint32_t)rp->r_eax);
1114 PANICNVADD(pnv, "trapno", (uint32_t)rp->r_trapno);
1115 PANICNVADD(pnv, "err", (uint32_t)rp->r_err);
1116 PANICNVADD(pnv, "eip", (uint32_t)rp->r_eip);
1117 PANICNVADD(pnv, "cs", (uint32_t)rp->r_cs);
1118 PANICNVADD(pnv, "eflags", (uint32_t)rp->r_efl);
1119 PANICNVADD(pnv, "uesp", (uint32_t)rp->r_uesp);
1120 PANICNVADD(pnv, "ss", (uint32_t)rp->r_ss);
1121 PANICNVADD(pnv, "gdt", creg.cr_gdt);
1122 PANICNVADD(pnv, "idt", creg.cr_idt);
1123 #endif /* __i386 */
1125 PANICNVADD(pnv, "ldt", creg.cr_ldt);
1126 PANICNVADD(pnv, "task", creg.cr_task);
1127 PANICNVADD(pnv, "cr0", creg.cr_cr0);
1128 PANICNVADD(pnv, "cr2", creg.cr_cr2);
1129 PANICNVADD(pnv, "cr3", creg.cr_cr3);
1130 if (creg.cr_cr4)
1131 PANICNVADD(pnv, "cr4", creg.cr_cr4);
1133 PANICNVSET(pdp, pnv);
1136 #define TR_ARG_MAX 6 /* Max args to print, same as SPARC */
1138 #if !defined(__amd64)
1141 * Given a return address (%eip), determine the likely number of arguments
1142 * that were pushed on the stack prior to its execution. We do this by
1143 * expecting that a typical call sequence consists of pushing arguments on
1144 * the stack, executing a call instruction, and then performing an add
1145 * on %esp to restore it to the value prior to pushing the arguments for
1146 * the call. We attempt to detect such an add, and divide the addend
1147 * by the size of a word to determine the number of pushed arguments.
1149 * If we do not find such an add, we punt and return TR_ARG_MAX. It is not
1150 * possible to reliably determine if a function took no arguments (i.e. was
1151 * void) because assembler routines do not reliably perform an add on %esp
1152 * immediately upon returning (eg. _sys_call()), so returning TR_ARG_MAX is
1153 * safer than returning 0.
1155 static ulong_t
1156 argcount(uintptr_t eip)
1158 const uint8_t *ins = (const uint8_t *)eip;
1159 ulong_t n;
1161 enum {
1162 M_MODRM_ESP = 0xc4, /* Mod/RM byte indicates %esp */
1163 M_ADD_IMM32 = 0x81, /* ADD imm32 to r/m32 */
1164 M_ADD_IMM8 = 0x83 /* ADD imm8 to r/m32 */
1167 if (eip < KERNELBASE || ins[1] != M_MODRM_ESP)
1168 return (TR_ARG_MAX);
1170 switch (ins[0]) {
1171 case M_ADD_IMM32:
1172 n = ins[2] + (ins[3] << 8) + (ins[4] << 16) + (ins[5] << 24);
1173 break;
1175 case M_ADD_IMM8:
1176 n = ins[2];
1177 break;
1179 default:
1180 return (TR_ARG_MAX);
1183 n /= sizeof (long);
1184 return (MIN(n, TR_ARG_MAX));
1187 #endif /* !__amd64 */
1190 * Print a stack backtrace using the specified frame pointer. We delay two
1191 * seconds before continuing, unless this is the panic traceback.
1192 * If we are in the process of panicking, we also attempt to write the
1193 * stack backtrace to a staticly assigned buffer, to allow the panic
1194 * code to find it and write it in to uncompressed pages within the
1195 * system crash dump.
1196 * Note that the frame for the starting stack pointer value is omitted because
1197 * the corresponding %eip is not known.
1200 extern char *dump_stack_scratch;
1202 #if defined(__amd64)
1204 void
1205 traceback(caddr_t fpreg)
1207 struct frame *fp = (struct frame *)fpreg;
1208 struct frame *nextfp;
1209 uintptr_t pc, nextpc;
1210 ulong_t off;
1211 char args[TR_ARG_MAX * 2 + 16], *sym;
1212 uint_t offset = 0;
1213 uint_t next_offset = 0;
1214 char stack_buffer[1024];
1216 if (!panicstr)
1217 printf("traceback: %%fp = %p\n", (void *)fp);
1219 if (panicstr && !dump_stack_scratch) {
1220 printf("Warning - stack not written to the dump buffer\n");
1223 fp = (struct frame *)plat_traceback(fpreg);
1224 if ((uintptr_t)fp < KERNELBASE)
1225 goto out;
1227 pc = fp->fr_savpc;
1228 fp = (struct frame *)fp->fr_savfp;
1230 while ((uintptr_t)fp >= KERNELBASE) {
1232 * XX64 Until port is complete tolerate 8-byte aligned
1233 * frame pointers but flag with a warning so they can
1234 * be fixed.
1236 if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) {
1237 if (((uintptr_t)fp & (8 - 1)) == 0) {
1238 printf(" >> warning! 8-byte"
1239 " aligned %%fp = %p\n", (void *)fp);
1240 } else {
1241 printf(
1242 " >> mis-aligned %%fp = %p\n", (void *)fp);
1243 break;
1247 args[0] = '\0';
1248 nextpc = (uintptr_t)fp->fr_savpc;
1249 nextfp = (struct frame *)fp->fr_savfp;
1250 if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1251 printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1252 mod_containing_pc((caddr_t)pc), sym, off, args);
1253 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1254 "%s:%s+%lx (%s) | ",
1255 mod_containing_pc((caddr_t)pc), sym, off, args);
1256 } else {
1257 printf("%016lx %lx (%s)\n",
1258 (uintptr_t)fp, pc, args);
1259 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1260 "%lx (%s) | ", pc, args);
1263 if (panicstr && dump_stack_scratch) {
1264 next_offset = offset + strlen(stack_buffer);
1265 if (next_offset < STACK_BUF_SIZE) {
1266 bcopy(stack_buffer, dump_stack_scratch + offset,
1267 strlen(stack_buffer));
1268 offset = next_offset;
1269 } else {
1271 * In attempting to save the panic stack
1272 * to the dumpbuf we have overflowed that area.
1273 * Print a warning and continue to printf the
1274 * stack to the msgbuf
1276 printf("Warning: stack in the dump buffer"
1277 " may be incomplete\n");
1278 offset = next_offset;
1282 pc = nextpc;
1283 fp = nextfp;
1285 out:
1286 if (!panicstr) {
1287 printf("end of traceback\n");
1288 DELAY(2 * MICROSEC);
1289 } else if (dump_stack_scratch) {
1290 dump_stack_scratch[offset] = '\0';
1294 #elif defined(__i386)
1296 void
1297 traceback(caddr_t fpreg)
1299 struct frame *fp = (struct frame *)fpreg;
1300 struct frame *nextfp, *minfp, *stacktop;
1301 uintptr_t pc, nextpc;
1302 uint_t offset = 0;
1303 uint_t next_offset = 0;
1304 char stack_buffer[1024];
1306 cpu_t *cpu;
1309 * args[] holds TR_ARG_MAX hex long args, plus ", " or '\0'.
1311 char args[TR_ARG_MAX * 2 + 8], *p;
1313 int on_intr;
1314 ulong_t off;
1315 char *sym;
1317 if (!panicstr)
1318 printf("traceback: %%fp = %p\n", (void *)fp);
1320 if (panicstr && !dump_stack_scratch) {
1321 printf("Warning - stack not written to the dumpbuf\n");
1325 * If we are panicking, all high-level interrupt information in
1326 * CPU was overwritten. panic_cpu has the correct values.
1328 kpreempt_disable(); /* prevent migration */
1330 cpu = (panicstr && CPU->cpu_id == panic_cpu.cpu_id)? &panic_cpu : CPU;
1332 if ((on_intr = CPU_ON_INTR(cpu)) != 0)
1333 stacktop = (struct frame *)(cpu->cpu_intr_stack + SA(MINFRAME));
1334 else
1335 stacktop = (struct frame *)curthread->t_stk;
1337 kpreempt_enable();
1339 fp = (struct frame *)plat_traceback(fpreg);
1340 if ((uintptr_t)fp < KERNELBASE)
1341 goto out;
1343 minfp = fp; /* Baseline minimum frame pointer */
1344 pc = fp->fr_savpc;
1345 fp = (struct frame *)fp->fr_savfp;
1347 while ((uintptr_t)fp >= KERNELBASE) {
1348 ulong_t argc;
1349 long *argv;
1351 if (fp <= minfp || fp >= stacktop) {
1352 if (on_intr) {
1354 * Hop from interrupt stack to thread stack.
1356 stacktop = (struct frame *)curthread->t_stk;
1357 minfp = (struct frame *)curthread->t_stkbase;
1358 on_intr = 0;
1359 continue;
1361 break; /* we're outside of the expected range */
1364 if ((uintptr_t)fp & (STACK_ALIGN - 1)) {
1365 printf(" >> mis-aligned %%fp = %p\n", (void *)fp);
1366 break;
1369 nextpc = fp->fr_savpc;
1370 nextfp = (struct frame *)fp->fr_savfp;
1371 argc = argcount(nextpc);
1372 argv = (long *)((char *)fp + sizeof (struct frame));
1374 args[0] = '\0';
1375 p = args;
1376 while (argc-- > 0 && argv < (long *)stacktop) {
1377 p += snprintf(p, args + sizeof (args) - p,
1378 "%s%lx", (p == args) ? "" : ", ", *argv++);
1381 if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1382 printf("%08lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1383 mod_containing_pc((caddr_t)pc), sym, off, args);
1384 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1385 "%s:%s+%lx (%s) | ",
1386 mod_containing_pc((caddr_t)pc), sym, off, args);
1388 } else {
1389 printf("%08lx %lx (%s)\n",
1390 (uintptr_t)fp, pc, args);
1391 (void) snprintf(stack_buffer, sizeof (stack_buffer),
1392 "%lx (%s) | ", pc, args);
1396 if (panicstr && dump_stack_scratch) {
1397 next_offset = offset + strlen(stack_buffer);
1398 if (next_offset < STACK_BUF_SIZE) {
1399 bcopy(stack_buffer, dump_stack_scratch + offset,
1400 strlen(stack_buffer));
1401 offset = next_offset;
1402 } else {
1404 * In attempting to save the panic stack
1405 * to the dumpbuf we have overflowed that area.
1406 * Print a warning and continue to printf the
1407 * stack to the msgbuf
1409 printf("Warning: stack in the dumpbuf"
1410 " may be incomplete\n");
1411 offset = next_offset;
1415 minfp = fp;
1416 pc = nextpc;
1417 fp = nextfp;
1419 out:
1420 if (!panicstr) {
1421 printf("end of traceback\n");
1422 DELAY(2 * MICROSEC);
1423 } else if (dump_stack_scratch) {
1424 dump_stack_scratch[offset] = '\0';
1429 #endif /* __i386 */
1432 * Generate a stack backtrace from a saved register set.
1434 void
1435 traceregs(struct regs *rp)
1437 traceback((caddr_t)rp->r_fp);
1440 void
1441 exec_set_sp(size_t stksize)
1443 klwp_t *lwp = ttolwp(curthread);
1445 lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize;
1448 hrtime_t
1449 gethrtime_waitfree(void)
1451 return (dtrace_gethrtime());
1454 hrtime_t
1455 gethrtime(void)
1457 return (gethrtimef());
1460 hrtime_t
1461 gethrtime_unscaled(void)
1463 return (gethrtimeunscaledf());
1466 void
1467 scalehrtime(hrtime_t *hrt)
1469 scalehrtimef(hrt);
1472 uint64_t
1473 unscalehrtime(hrtime_t nsecs)
1475 return (unscalehrtimef(nsecs));
1478 void
1479 gethrestime(timespec_t *tp)
1481 gethrestimef(tp);
1484 #if defined(__amd64)
1486 * Part of the implementation of hres_tick(); this routine is
1487 * easier in C than assembler .. called with the hres_lock held.
1489 * XX64 Many of these timekeeping variables need to be extern'ed in a header
1492 #include <sys/time.h>
1493 #include <sys/machlock.h>
1495 extern int one_sec;
1496 extern int max_hres_adj;
1498 void
1499 __adj_hrestime(void)
1501 long long adj;
1503 if (hrestime_adj == 0)
1504 adj = 0;
1505 else if (hrestime_adj > 0) {
1506 if (hrestime_adj < max_hres_adj)
1507 adj = hrestime_adj;
1508 else
1509 adj = max_hres_adj;
1510 } else {
1511 if (hrestime_adj < -max_hres_adj)
1512 adj = -max_hres_adj;
1513 else
1514 adj = hrestime_adj;
1517 timedelta -= adj;
1518 hrestime_adj = timedelta;
1519 hrestime.tv_nsec += adj;
1521 while (hrestime.tv_nsec >= NANOSEC) {
1522 one_sec++;
1523 hrestime.tv_sec++;
1524 hrestime.tv_nsec -= NANOSEC;
1527 #endif
1530 * Wrapper functions to maintain backwards compability
1533 xcopyin(const void *uaddr, void *kaddr, size_t count)
1535 return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED));
1539 xcopyout(const void *kaddr, void *uaddr, size_t count)
1541 return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED));