4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2017 Joyent, Inc.
26 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
28 /* All Rights Reserved */
30 /* Copyright (c) 1987, 1988 Microsoft Corporation */
31 /* All Rights Reserved */
34 * Copyright (c) 2009, Intel Corporation.
35 * All rights reserved.
38 #include <sys/types.h>
39 #include <sys/param.h>
40 #include <sys/signal.h>
41 #include <sys/regset.h>
42 #include <sys/privregs.h>
45 #include <sys/fault.h>
46 #include <sys/systm.h>
52 #include <sys/cpuvar.h>
53 #include <sys/thread.h>
56 #include <sys/siginfo.h>
57 #include <sys/archsystm.h>
59 #include <sys/debug.h>
60 #include <sys/x86_archext.h>
61 #include <sys/sysmacros.h>
62 #include <sys/cmn_err.h>
64 kmem_cache_t
*fpsave_cachep
;
66 /* Legacy fxsave layout + xsave header + ymm */
67 #define AVX_XSAVE_SIZE (512 + 64 + 256)
70 #pragma align 16 (sse_initial)
73 * Initial kfpu state for SSE/SSE2 used by fpinit()
75 const struct fxsave_state sse_initial
= {
76 FPU_CW_INIT
, /* fx_fcw */
91 SSE_MXCSR_INIT
/* fx_mxcsr */
92 /* rest of structure is zero */
96 #pragma align 64 (avx_initial)
99 * Initial kfpu state for AVX used by fpinit()
101 const struct xsave_state avx_initial
= {
103 * The definition below needs to be identical with sse_initial
107 FPU_CW_INIT
, /* fx_fcw */
122 SSE_MXCSR_INIT
/* fx_mxcsr */
123 /* rest of structure is zero */
126 * bit0 = 1 for XSTATE_BV to indicate that legacy fields are valid,
127 * and CPU should initialize XMM/YMM.
131 /* rest of structure is zero */
135 * mxcsr_mask value (possibly reset in fpu_probe); used to avoid
136 * the #gp exception caused by setting unsupported bits in the
139 uint32_t sse_mxcsr_mask
= SSE_MXCSR_MASK_DEFAULT
;
142 * Initial kfpu state for x87 used by fpinit()
144 const struct fnsave_state x87_initial
= {
145 FPU_CW_INIT
, /* f_fcw */
150 /* rest of structure is zero */
155 * This vector is patched to xsave_ctxt() if we discover we have an
156 * XSAVE-capable chip in fpu_probe.
158 void (*fpsave_ctxt
)(void *) = fpxsave_ctxt
;
159 #elif defined(__i386)
161 * This vector is patched to fpxsave_ctxt() if we discover we have an
162 * SSE-capable chip in fpu_probe(). It is patched to xsave_ctxt
163 * if we discover we have an XSAVE-capable chip in fpu_probe.
165 void (*fpsave_ctxt
)(void *) = fpnsave_ctxt
;
169 * This function pointer is changed to xsaveopt if the CPU is xsaveopt capable.
171 void (*xsavep
)(struct xsave_state
*, uint64_t) = xsave
;
173 static int fpe_sicode(uint_t
);
174 static int fpe_simd_sicode(uint_t
);
177 * Copy the state of parent lwp's floating point context into the new lwp.
178 * Invoked for both fork() and lwp_create().
180 * Note that we inherit -only- the control state (e.g. exception masks,
181 * rounding, precision control, etc.); the FPU registers are otherwise
182 * reset to their initial state.
185 fp_new_lwp(kthread_id_t t
, kthread_id_t ct
)
187 struct fpu_ctx
*fp
; /* parent fpu context */
188 struct fpu_ctx
*cfp
; /* new fpu context */
189 struct fxsave_state
*fx
, *cfx
;
191 struct fnsave_state
*fn
, *cfn
;
193 struct xsave_state
*cxs
;
195 ASSERT(fp_kind
!= FP_NO
);
197 fp
= &t
->t_lwp
->lwp_pcb
.pcb_fpu
;
198 cfp
= &ct
->t_lwp
->lwp_pcb
.pcb_fpu
;
201 * If the parent FPU state is still in the FPU hw then save it;
202 * conveniently, fp_save() already does this for us nicely.
206 cfp
->fpu_flags
= FPU_EN
| FPU_VALID
;
207 cfp
->fpu_regs
.kfpu_status
= 0;
208 cfp
->fpu_regs
.kfpu_xstatus
= 0;
210 switch (fp_save_mech
) {
213 fn
= fp
->fpu_regs
.kfpu_u
.kfpu_fn
;
214 cfn
= cfp
->fpu_regs
.kfpu_u
.kfpu_fn
;
215 bcopy(&x87_initial
, cfn
, sizeof (*cfn
));
216 cfn
->f_fcw
= fn
->f_fcw
;
220 fx
= fp
->fpu_regs
.kfpu_u
.kfpu_fx
;
221 cfx
= cfp
->fpu_regs
.kfpu_u
.kfpu_fx
;
222 bcopy(&sse_initial
, cfx
, sizeof (*cfx
));
223 cfx
->fx_mxcsr
= fx
->fx_mxcsr
& ~SSE_MXCSR_EFLAGS
;
224 cfx
->fx_fcw
= fx
->fx_fcw
;
228 cfp
->fpu_xsave_mask
= fp
->fpu_xsave_mask
;
230 VERIFY(fp
->fpu_regs
.kfpu_u
.kfpu_xs
!= NULL
);
232 fx
= &fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_fxsave
;
233 cxs
= cfp
->fpu_regs
.kfpu_u
.kfpu_xs
;
234 cfx
= &cxs
->xs_fxsave
;
236 bcopy(&avx_initial
, cxs
, sizeof (*cxs
));
237 cfx
->fx_mxcsr
= fx
->fx_mxcsr
& ~SSE_MXCSR_EFLAGS
;
238 cfx
->fx_fcw
= fx
->fx_fcw
;
239 cxs
->xs_xstate_bv
|= (get_xcr(XFEATURE_ENABLED_MASK
) &
243 panic("Invalid fp_save_mech");
248 fpsave_ctxt
, NULL
, fp_new_lwp
, fp_new_lwp
, NULL
, fp_free
);
250 * Now, when the new lwp starts running, it will take a trap
251 * that will be handled inline in the trap table to cause
252 * the appropriate f*rstor instruction to load the save area we
253 * constructed above directly into the hardware.
258 * Free any state associated with floating point context.
259 * Fp_free can be called in three cases:
260 * 1) from reaper -> thread_free -> freectx-> fp_free
261 * fp context belongs to a thread on deathrow
262 * nothing to do, thread will never be resumed
263 * thread calling ctxfree is reaper
265 * 2) from exec -> freectx -> fp_free
266 * fp context belongs to the current thread
267 * must disable fpu, thread calling ctxfree is curthread
269 * 3) from restorecontext -> setfpregs -> fp_free
270 * we have a modified context in the memory (lwp->pcb_fpu)
271 * disable fpu and release the fp context for the CPU
276 fp_free(struct fpu_ctx
*fp
, int isexec
)
278 ASSERT(fp_kind
!= FP_NO
);
280 if (fp
->fpu_flags
& FPU_VALID
)
285 * We want to do fpsave rather than fpdisable so that we can
286 * keep the fpu_flags as FPU_VALID tracking the CR0_TS bit
288 fp
->fpu_flags
|= FPU_VALID
;
289 /* If for current thread disable FP to track FPU_VALID */
290 if (curthread
->t_lwp
&& fp
== &curthread
->t_lwp
->lwp_pcb
.pcb_fpu
) {
291 /* Clear errors if any to prevent frstor from complaining */
292 (void) fperr_reset();
293 if (fp_kind
& __FP_SSE
)
294 (void) fpxerr_reset();
301 * Store the floating point state and disable the floating point unit.
304 fp_save(struct fpu_ctx
*fp
)
306 ASSERT(fp_kind
!= FP_NO
);
309 if (!fp
|| fp
->fpu_flags
& FPU_VALID
) {
313 ASSERT(curthread
->t_lwp
&& fp
== &curthread
->t_lwp
->lwp_pcb
.pcb_fpu
);
315 switch (fp_save_mech
) {
318 fpsave(fp
->fpu_regs
.kfpu_u
.kfpu_fn
);
322 fpxsave(fp
->fpu_regs
.kfpu_u
.kfpu_fx
);
326 xsavep(fp
->fpu_regs
.kfpu_u
.kfpu_xs
, fp
->fpu_xsave_mask
);
329 panic("Invalid fp_save_mech");
333 fp
->fpu_flags
|= FPU_VALID
;
338 * Restore the FPU context for the thread:
339 * The possibilities are:
340 * 1. No active FPU context: Load the new context into the FPU hw
341 * and enable the FPU.
344 fp_restore(struct fpu_ctx
*fp
)
346 switch (fp_save_mech
) {
349 fprestore(fp
->fpu_regs
.kfpu_u
.kfpu_fn
);
353 fpxrestore(fp
->fpu_regs
.kfpu_u
.kfpu_fx
);
357 xrestore(fp
->fpu_regs
.kfpu_u
.kfpu_xs
, fp
->fpu_xsave_mask
);
360 panic("Invalid fp_save_mech");
364 fp
->fpu_flags
&= ~FPU_VALID
;
369 * Seeds the initial state for the current thread. The possibilities are:
370 * 1. Another process has modified the FPU state before we have done any
371 * initialization: Load the FPU state from the LWP state.
372 * 2. The FPU state has not been externally modified: Load a clean state.
377 struct fpu_ctx
*fp
= &ttolwp(curthread
)->lwp_pcb
.pcb_fpu
;
379 ASSERT(curthread
->t_preempt
>= 1);
380 ASSERT((fp
->fpu_flags
& FPU_EN
) == 0);
383 * Always initialize a new context and initialize the hardware.
385 if (fp_save_mech
== FP_XSAVE
) {
386 fp
->fpu_xsave_mask
= get_xcr(XFEATURE_ENABLED_MASK
) &
390 installctx(curthread
, fp
,
391 fpsave_ctxt
, NULL
, fp_new_lwp
, fp_new_lwp
, NULL
, fp_free
);
395 * If FPU_VALID is set, it means someone has modified registers via
396 * /proc. In this case, restore the current lwp's state.
398 if (fp
->fpu_flags
& FPU_VALID
)
401 ASSERT((fp
->fpu_flags
& FPU_VALID
) == 0);
402 fp
->fpu_flags
= FPU_EN
;
406 * When using xsave/xrstor, these three functions are used by the lwp code to
407 * manage the memory for the xsave area.
410 fp_lwp_init(struct _klwp
*lwp
)
412 struct fpu_ctx
*fp
= &lwp
->lwp_pcb
.pcb_fpu
;
415 * We keep a copy of the pointer in lwp_fpu so that we can restore the
416 * value in forklwp() after we duplicate the parent's LWP state.
418 lwp
->lwp_fpu
= fp
->fpu_regs
.kfpu_u
.kfpu_generic
=
419 kmem_cache_alloc(fpsave_cachep
, KM_SLEEP
);
421 if (fp_save_mech
== FP_XSAVE
) {
424 * We bzero since the fpinit() code path will only
425 * partially initialize the xsave area using avx_inital.
427 ASSERT(cpuid_get_xsave_size() >= sizeof (struct xsave_state
));
428 bzero(fp
->fpu_regs
.kfpu_u
.kfpu_xs
, cpuid_get_xsave_size());
433 fp_lwp_cleanup(struct _klwp
*lwp
)
435 struct fpu_ctx
*fp
= &lwp
->lwp_pcb
.pcb_fpu
;
437 if (fp
->fpu_regs
.kfpu_u
.kfpu_generic
!= NULL
) {
438 kmem_cache_free(fpsave_cachep
,
439 fp
->fpu_regs
.kfpu_u
.kfpu_generic
);
440 lwp
->lwp_fpu
= fp
->fpu_regs
.kfpu_u
.kfpu_generic
= NULL
;
445 * Called during the process of forklwp(). The kfpu_u pointer will have been
446 * overwritten while copying the parent's LWP structure. We have a valid copy
447 * stashed in the child's lwp_fpu which we use to restore the correct value.
450 fp_lwp_dup(struct _klwp
*lwp
)
452 void *xp
= lwp
->lwp_fpu
;
455 switch (fp_save_mech
) {
458 sz
= sizeof (struct fnsave_state
);
462 sz
= sizeof (struct fxsave_state
);
465 sz
= cpuid_get_xsave_size();
468 panic("Invalid fp_save_mech");
472 /* copy the parent's values into the new lwp's struct */
473 bcopy(lwp
->lwp_pcb
.pcb_fpu
.fpu_regs
.kfpu_u
.kfpu_generic
, xp
, sz
);
474 /* now restore the pointer */
475 lwp
->lwp_pcb
.pcb_fpu
.fpu_regs
.kfpu_u
.kfpu_generic
= xp
;
480 * This routine is called from trap() when User thread takes No Extension
481 * Fault. The possiblities are:
482 * 1. User thread has executed a FP instruction for the first time.
483 * Save current FPU context if any. Initialize FPU, setup FPU
484 * context for the thread and enable FP hw.
485 * 2. Thread's pcb has a valid FPU state: Restore the FPU state and
488 * Note that case #2 is inlined in the trap table.
491 fpnoextflt(struct regs
*rp
)
493 struct fpu_ctx
*fp
= &ttolwp(curthread
)->lwp_pcb
.pcb_fpu
;
495 ASSERT(sizeof (struct fxsave_state
) == 512 &&
496 sizeof (struct fnsave_state
) == 108);
497 ASSERT((offsetof(struct fxsave_state
, fx_xmm
[0]) & 0xf) == 0);
499 ASSERT(sizeof (struct xsave_state
) >= AVX_XSAVE_SIZE
);
502 ASSERT(sizeof (struct _fpu
) == sizeof (struct __old_fpu
));
507 * Now we can enable the interrupts.
508 * (NOTE: fp-no-coprocessor comes thru interrupt gate)
512 if (!fpu_exists
) { /* check for FPU hw exists */
513 if (fp_kind
== FP_NO
) {
517 * When the system has no floating point support,
518 * i.e. no FP hardware and no emulator, skip the
519 * two kinds of FP instruction that occur in
520 * fpstart. Allows processes that do no real FP
523 if (fuword32((void *)rp
->r_pc
, &inst
) != -1 &&
524 ((inst
& 0xFFFF) == 0x7dd9 ||
525 (inst
& 0xFFFF) == 0x6dd9)) {
533 * If we have neither a processor extension nor
534 * an emulator, kill the process OR panic the kernel.
537 return (1); /* error */
540 #if !defined(__xpv) /* XXPV Is this ifdef needed now? */
542 * A paranoid cross-check: for the SSE case, ensure that %cr4 is
543 * configured to enable fully fledged (%xmm) fxsave/fxrestor on
544 * this CPU. For the non-SSE case, ensure that it isn't.
546 ASSERT(((fp_kind
& __FP_SSE
) &&
547 (getcr4() & CR4_OSFXSR
) == CR4_OSFXSR
) ||
548 (!(fp_kind
& __FP_SSE
) &&
549 (getcr4() & (CR4_OSXMMEXCPT
|CR4_OSFXSR
)) == 0));
552 if (fp
->fpu_flags
& FPU_EN
) {
565 * Handle a processor extension overrun fault
566 * Returns non zero for error.
568 * XXX Shouldn't this just be abolished given that we're not supporting
569 * anything prior to Pentium?
574 fpextovrflt(struct regs
*rp
)
576 #if !defined(__xpv) /* XXPV Do we need this ifdef either */
579 ASSERT(fp_kind
!= FP_NO
);
582 fpinit(); /* initialize the FPU hardware */
586 return (1); /* error, send SIGSEGV signal to the thread */
590 * Handle a processor extension error fault
591 * Returns non zero for error.
596 fpexterrflt(struct regs
*rp
)
599 fpu_ctx_t
*fp
= &ttolwp(curthread
)->lwp_pcb
.pcb_fpu
;
601 ASSERT(fp_kind
!= FP_NO
);
604 * Now we can enable the interrupts.
605 * (NOTE: x87 fp exceptions come thru interrupt gate)
613 * Do an unconditional save of the FP state. If it's dirty (TS=0),
614 * it'll be saved into the fpu context area passed in (that of the
615 * current thread). If it's not dirty (it may not be, due to
616 * an intervening save due to a context switch between the sti(),
617 * above and here, then it's safe to just use the stored values in
618 * the context save area to determine the cause of the fault.
622 /* clear exception flags in saved state, as if by fnclex */
623 switch (fp_save_mech
) {
626 fpsw
= fp
->fpu_regs
.kfpu_u
.kfpu_fn
->f_fsw
;
627 fpcw
= fp
->fpu_regs
.kfpu_u
.kfpu_fn
->f_fcw
;
628 fp
->fpu_regs
.kfpu_u
.kfpu_fn
->f_fsw
&= ~FPS_SW_EFLAGS
;
633 fpsw
= fp
->fpu_regs
.kfpu_u
.kfpu_fx
->fx_fsw
;
634 fpcw
= fp
->fpu_regs
.kfpu_u
.kfpu_fx
->fx_fcw
;
635 fp
->fpu_regs
.kfpu_u
.kfpu_fx
->fx_fsw
&= ~FPS_SW_EFLAGS
;
639 fpsw
= fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_fxsave
.fx_fsw
;
640 fpcw
= fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_fxsave
.fx_fcw
;
641 fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_fxsave
.fx_fsw
&= ~FPS_SW_EFLAGS
;
643 * Always set LEGACY_FP as it may have been cleared by XSAVE
646 fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_xstate_bv
|= XFEATURE_LEGACY_FP
;
649 panic("Invalid fp_save_mech");
653 fp
->fpu_regs
.kfpu_status
= fpsw
;
655 if ((fpsw
& FPS_ES
) == 0)
656 return (0); /* No exception */
659 * "and" the exception flags with the complement of the mask
660 * bits to determine which exception occurred
662 return (fpe_sicode(fpsw
& ~fpcw
& 0x3f));
666 * Handle an SSE/SSE2 precise exception.
667 * Returns a non-zero sicode for error.
671 fpsimderrflt(struct regs
*rp
)
673 uint32_t mxcsr
, xmask
;
674 fpu_ctx_t
*fp
= &ttolwp(curthread
)->lwp_pcb
.pcb_fpu
;
676 ASSERT(fp_kind
& __FP_SSE
);
679 * NOTE: Interrupts are disabled during execution of this
680 * function. They are enabled by the caller in trap.c.
684 * The only way we could have gotten here if there is no FP unit
685 * is via a user executing an INT $19 instruction, so there is
686 * no fault in that case.
692 * Do an unconditional save of the FP state. If it's dirty (TS=0),
693 * it'll be saved into the fpu context area passed in (that of the
694 * current thread). If it's not dirty, then it's safe to just use
695 * the stored values in the context save area to determine the
696 * cause of the fault.
698 fp_save(fp
); /* save the FPU state */
700 if (fp_save_mech
== FP_XSAVE
) {
701 mxcsr
= fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_fxsave
.fx_mxcsr
;
702 fp
->fpu_regs
.kfpu_status
=
703 fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_fxsave
.fx_fsw
;
705 mxcsr
= fp
->fpu_regs
.kfpu_u
.kfpu_fx
->fx_mxcsr
;
706 fp
->fpu_regs
.kfpu_status
= fp
->fpu_regs
.kfpu_u
.kfpu_fx
->fx_fsw
;
708 fp
->fpu_regs
.kfpu_xstatus
= mxcsr
;
711 * compute the mask that determines which conditions can cause
712 * a #xm exception, and use this to clean the status bits so that
713 * we can identify the true cause of this one.
715 xmask
= (mxcsr
>> 7) & SSE_MXCSR_EFLAGS
;
716 return (fpe_simd_sicode((mxcsr
& SSE_MXCSR_EFLAGS
) & ~xmask
));
720 * In the unlikely event that someone is relying on this subcode being
721 * FPE_FLTILL for denormalize exceptions, it can always be patched back
722 * again to restore old behaviour.
724 int fpe_fltden
= FPE_FLTDEN
;
727 * Map from the FPU status word to the FP exception si_code.
730 fpe_sicode(uint_t sw
)
744 return (FPE_FLTINV
); /* default si_code for other exceptions */
748 * Map from the SSE status word to the FP exception si_code.
751 fpe_simd_sicode(uint_t sw
)
765 return (FPE_FLTINV
); /* default si_code for other exceptions */
769 * This routine is invoked as part of libc's __fpstart implementation
772 * It may be called -before- any context has been assigned in which case
773 * we try and avoid touching the hardware. Or it may be invoked well
774 * after the context has been assigned and fiddled with, in which case
775 * just tweak it directly.
778 fpsetcw(uint16_t fcw
, uint32_t mxcsr
)
780 struct fpu_ctx
*fp
= &curthread
->t_lwp
->lwp_pcb
.pcb_fpu
;
781 struct fxsave_state
*fx
;
783 if (!fpu_exists
|| fp_kind
== FP_NO
)
786 if ((fp
->fpu_flags
& FPU_EN
) == 0) {
787 if (fcw
== FPU_CW_INIT
&& mxcsr
== SSE_MXCSR_INIT
) {
789 * Common case. Floating point unit not yet
790 * enabled, and kernel already intends to initialize
791 * the hardware the way the caller wants.
796 * Hmm. Userland wants a different default.
797 * Do a fake "first trap" to establish the context, then
798 * handle as if we already had a context before we came in.
806 * Ensure that the current hardware state is flushed back to the
807 * pcb, then modify that copy. Next use of the fp will
808 * restore the context.
812 switch (fp_save_mech
) {
815 fp
->fpu_regs
.kfpu_u
.kfpu_fn
->f_fcw
= fcw
;
819 fx
= fp
->fpu_regs
.kfpu_u
.kfpu_fx
;
821 fx
->fx_mxcsr
= sse_mxcsr_mask
& mxcsr
;
825 fx
= &fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_fxsave
;
827 fx
->fx_mxcsr
= sse_mxcsr_mask
& mxcsr
;
829 * Always set LEGACY_FP as it may have been cleared by XSAVE
832 fp
->fpu_regs
.kfpu_u
.kfpu_xs
->xs_xstate_bv
|= XFEATURE_LEGACY_FP
;
835 panic("Invalid fp_save_mech");