2 * Copyright (C) 1994 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * General FPU state handling cleanups
6 * Gareth Hughes <gareth@valinux.com>, May 2000
8 #include <linux/module.h>
9 #include <linux/regset.h>
10 #include <linux/sched.h>
11 #include <linux/slab.h>
13 #include <asm/sigcontext.h>
14 #include <asm/processor.h>
15 #include <asm/math_emu.h>
16 #include <asm/uaccess.h>
17 #include <asm/ptrace.h>
19 #include <asm/fpu-internal.h>
23 # include <asm/sigcontext32.h>
24 # include <asm/user32.h>
26 # define save_i387_xstate_ia32 save_i387_xstate
27 # define restore_i387_xstate_ia32 restore_i387_xstate
28 # define _fpstate_ia32 _fpstate
29 # define _xstate_ia32 _xstate
30 # define sig_xstate_ia32_size sig_xstate_size
31 # define fx_sw_reserved_ia32 fx_sw_reserved
32 # define user_i387_ia32_struct user_i387_struct
33 # define user32_fxsr_struct user_fxsr_struct
37 * Were we in an interrupt that interrupted kernel mode?
39 * We can do a kernel_fpu_begin/end() pair *ONLY* if that
40 * pair does nothing at all: the thread must not have fpu (so
41 * that we don't try to save the FPU state), and TS must
42 * be set (so that the clts/stts pair does nothing that is
43 * visible in the interrupted kernel thread).
45 static inline bool interrupted_kernel_fpu_idle(void)
47 return !__thread_has_fpu(current
) &&
48 (read_cr0() & X86_CR0_TS
);
52 * Were we in user mode (or vm86 mode) when we were
55 * Doing kernel_fpu_begin/end() is ok if we are running
56 * in an interrupt context from user mode - we'll just
57 * save the FPU state as required.
59 static inline bool interrupted_user_mode(void)
61 struct pt_regs
*regs
= get_irq_regs();
62 return regs
&& user_mode_vm(regs
);
66 * Can we use the FPU in kernel mode with the
67 * whole "kernel_fpu_begin/end()" sequence?
69 * It's always ok in process context (ie "not interrupt")
70 * but it is sometimes ok even from an irq.
72 bool irq_fpu_usable(void)
74 return !in_interrupt() ||
75 interrupted_user_mode() ||
76 interrupted_kernel_fpu_idle();
78 EXPORT_SYMBOL(irq_fpu_usable
);
80 void kernel_fpu_begin(void)
82 struct task_struct
*me
= current
;
84 WARN_ON_ONCE(!irq_fpu_usable());
86 if (__thread_has_fpu(me
)) {
88 __thread_clear_has_fpu(me
);
89 /* We do 'stts()' in kernel_fpu_end() */
91 percpu_write(fpu_owner_task
, NULL
);
95 EXPORT_SYMBOL(kernel_fpu_begin
);
97 void kernel_fpu_end(void)
102 EXPORT_SYMBOL(kernel_fpu_end
);
104 void unlazy_fpu(struct task_struct
*tsk
)
107 if (__thread_has_fpu(tsk
)) {
108 __save_init_fpu(tsk
);
109 __thread_fpu_end(tsk
);
111 tsk
->fpu_counter
= 0;
114 EXPORT_SYMBOL(unlazy_fpu
);
116 #ifdef CONFIG_MATH_EMULATION
117 # define HAVE_HWFP (boot_cpu_data.hard_math)
122 static unsigned int mxcsr_feature_mask __read_mostly
= 0xffffffffu
;
123 unsigned int xstate_size
;
124 EXPORT_SYMBOL_GPL(xstate_size
);
125 unsigned int sig_xstate_ia32_size
= sizeof(struct _fpstate_ia32
);
126 static struct i387_fxsave_struct fx_scratch __cpuinitdata
;
128 static void __cpuinit
mxcsr_feature_mask_init(void)
130 unsigned long mask
= 0;
134 memset(&fx_scratch
, 0, sizeof(struct i387_fxsave_struct
));
135 asm volatile("fxsave %0" : : "m" (fx_scratch
));
136 mask
= fx_scratch
.mxcsr_mask
;
140 mxcsr_feature_mask
&= mask
;
144 static void __cpuinit
init_thread_xstate(void)
147 * Note that xstate_size might be overwriten later during
153 * Disable xsave as we do not support it if i387
154 * emulation is enabled.
156 setup_clear_cpu_cap(X86_FEATURE_XSAVE
);
157 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT
);
158 xstate_size
= sizeof(struct i387_soft_struct
);
163 xstate_size
= sizeof(struct i387_fxsave_struct
);
165 xstate_size
= sizeof(struct i387_fsave_struct
);
169 * Called at bootup to set up the initial FPU state that is later cloned
170 * into all processes.
173 void __cpuinit
fpu_init(void)
176 unsigned long cr4_mask
= 0;
179 cr4_mask
|= X86_CR4_OSFXSR
;
181 cr4_mask
|= X86_CR4_OSXMMEXCPT
;
183 set_in_cr4(cr4_mask
);
186 cr0
&= ~(X86_CR0_TS
|X86_CR0_EM
); /* clear TS and EM */
191 if (!smp_processor_id())
192 init_thread_xstate();
194 mxcsr_feature_mask_init();
195 /* clean state in init */
196 current_thread_info()->status
= 0;
200 void fpu_finit(struct fpu
*fpu
)
203 finit_soft_fpu(&fpu
->state
->soft
);
208 struct i387_fxsave_struct
*fx
= &fpu
->state
->fxsave
;
210 memset(fx
, 0, xstate_size
);
213 fx
->mxcsr
= MXCSR_DEFAULT
;
215 struct i387_fsave_struct
*fp
= &fpu
->state
->fsave
;
216 memset(fp
, 0, xstate_size
);
217 fp
->cwd
= 0xffff037fu
;
218 fp
->swd
= 0xffff0000u
;
219 fp
->twd
= 0xffffffffu
;
220 fp
->fos
= 0xffff0000u
;
223 EXPORT_SYMBOL_GPL(fpu_finit
);
226 * The _current_ task is using the FPU for the first time
227 * so initialize it and set the mxcsr to its default
228 * value at reset if we support XMM instructions and then
229 * remember the current task has used the FPU.
231 int init_fpu(struct task_struct
*tsk
)
235 if (tsk_used_math(tsk
)) {
236 if (HAVE_HWFP
&& tsk
== current
)
242 * Memory allocation at the first usage of the FPU and other state.
244 ret
= fpu_alloc(&tsk
->thread
.fpu
);
248 fpu_finit(&tsk
->thread
.fpu
);
250 set_stopped_child_used_math(tsk
);
253 EXPORT_SYMBOL_GPL(init_fpu
);
256 * The xstateregs_active() routine is the same as the fpregs_active() routine,
257 * as the "regset->n" for the xstate regset will be updated based on the feature
258 * capabilites supported by the xsave.
260 int fpregs_active(struct task_struct
*target
, const struct user_regset
*regset
)
262 return tsk_used_math(target
) ? regset
->n
: 0;
265 int xfpregs_active(struct task_struct
*target
, const struct user_regset
*regset
)
267 return (cpu_has_fxsr
&& tsk_used_math(target
)) ? regset
->n
: 0;
270 int xfpregs_get(struct task_struct
*target
, const struct user_regset
*regset
,
271 unsigned int pos
, unsigned int count
,
272 void *kbuf
, void __user
*ubuf
)
279 ret
= init_fpu(target
);
283 sanitize_i387_state(target
);
285 return user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
,
286 &target
->thread
.fpu
.state
->fxsave
, 0, -1);
289 int xfpregs_set(struct task_struct
*target
, const struct user_regset
*regset
,
290 unsigned int pos
, unsigned int count
,
291 const void *kbuf
, const void __user
*ubuf
)
298 ret
= init_fpu(target
);
302 sanitize_i387_state(target
);
304 ret
= user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
,
305 &target
->thread
.fpu
.state
->fxsave
, 0, -1);
308 * mxcsr reserved bits must be masked to zero for security reasons.
310 target
->thread
.fpu
.state
->fxsave
.mxcsr
&= mxcsr_feature_mask
;
313 * update the header bits in the xsave header, indicating the
314 * presence of FP and SSE state.
317 target
->thread
.fpu
.state
->xsave
.xsave_hdr
.xstate_bv
|= XSTATE_FPSSE
;
322 int xstateregs_get(struct task_struct
*target
, const struct user_regset
*regset
,
323 unsigned int pos
, unsigned int count
,
324 void *kbuf
, void __user
*ubuf
)
331 ret
= init_fpu(target
);
336 * Copy the 48bytes defined by the software first into the xstate
337 * memory layout in the thread struct, so that we can copy the entire
338 * xstateregs to the user using one user_regset_copyout().
340 memcpy(&target
->thread
.fpu
.state
->fxsave
.sw_reserved
,
341 xstate_fx_sw_bytes
, sizeof(xstate_fx_sw_bytes
));
344 * Copy the xstate memory layout.
346 ret
= user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
,
347 &target
->thread
.fpu
.state
->xsave
, 0, -1);
351 int xstateregs_set(struct task_struct
*target
, const struct user_regset
*regset
,
352 unsigned int pos
, unsigned int count
,
353 const void *kbuf
, const void __user
*ubuf
)
356 struct xsave_hdr_struct
*xsave_hdr
;
361 ret
= init_fpu(target
);
365 ret
= user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
,
366 &target
->thread
.fpu
.state
->xsave
, 0, -1);
369 * mxcsr reserved bits must be masked to zero for security reasons.
371 target
->thread
.fpu
.state
->fxsave
.mxcsr
&= mxcsr_feature_mask
;
373 xsave_hdr
= &target
->thread
.fpu
.state
->xsave
.xsave_hdr
;
375 xsave_hdr
->xstate_bv
&= pcntxt_mask
;
377 * These bits must be zero.
379 xsave_hdr
->reserved1
[0] = xsave_hdr
->reserved1
[1] = 0;
384 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
387 * FPU tag word conversions.
390 static inline unsigned short twd_i387_to_fxsr(unsigned short twd
)
392 unsigned int tmp
; /* to avoid 16 bit prefixes in the code */
394 /* Transform each pair of bits into 01 (valid) or 00 (empty) */
396 tmp
= (tmp
| (tmp
>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
397 /* and move the valid bits to the lower byte. */
398 tmp
= (tmp
| (tmp
>> 1)) & 0x3333; /* 00VV00VV00VV00VV */
399 tmp
= (tmp
| (tmp
>> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
400 tmp
= (tmp
| (tmp
>> 4)) & 0x00ff; /* 00000000VVVVVVVV */
405 #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16)
406 #define FP_EXP_TAG_VALID 0
407 #define FP_EXP_TAG_ZERO 1
408 #define FP_EXP_TAG_SPECIAL 2
409 #define FP_EXP_TAG_EMPTY 3
411 static inline u32
twd_fxsr_to_i387(struct i387_fxsave_struct
*fxsave
)
414 u32 tos
= (fxsave
->swd
>> 11) & 7;
415 u32 twd
= (unsigned long) fxsave
->twd
;
417 u32 ret
= 0xffff0000u
;
420 for (i
= 0; i
< 8; i
++, twd
>>= 1) {
422 st
= FPREG_ADDR(fxsave
, (i
- tos
) & 7);
424 switch (st
->exponent
& 0x7fff) {
426 tag
= FP_EXP_TAG_SPECIAL
;
429 if (!st
->significand
[0] &&
430 !st
->significand
[1] &&
431 !st
->significand
[2] &&
433 tag
= FP_EXP_TAG_ZERO
;
435 tag
= FP_EXP_TAG_SPECIAL
;
438 if (st
->significand
[3] & 0x8000)
439 tag
= FP_EXP_TAG_VALID
;
441 tag
= FP_EXP_TAG_SPECIAL
;
445 tag
= FP_EXP_TAG_EMPTY
;
447 ret
|= tag
<< (2 * i
);
453 * FXSR floating point environment conversions.
457 convert_from_fxsr(struct user_i387_ia32_struct
*env
, struct task_struct
*tsk
)
459 struct i387_fxsave_struct
*fxsave
= &tsk
->thread
.fpu
.state
->fxsave
;
460 struct _fpreg
*to
= (struct _fpreg
*) &env
->st_space
[0];
461 struct _fpxreg
*from
= (struct _fpxreg
*) &fxsave
->st_space
[0];
464 env
->cwd
= fxsave
->cwd
| 0xffff0000u
;
465 env
->swd
= fxsave
->swd
| 0xffff0000u
;
466 env
->twd
= twd_fxsr_to_i387(fxsave
);
469 env
->fip
= fxsave
->rip
;
470 env
->foo
= fxsave
->rdp
;
472 * should be actually ds/cs at fpu exception time, but
473 * that information is not available in 64bit mode.
475 env
->fcs
= task_pt_regs(tsk
)->cs
;
476 if (tsk
== current
) {
477 savesegment(ds
, env
->fos
);
479 env
->fos
= tsk
->thread
.ds
;
481 env
->fos
|= 0xffff0000;
483 env
->fip
= fxsave
->fip
;
484 env
->fcs
= (u16
) fxsave
->fcs
| ((u32
) fxsave
->fop
<< 16);
485 env
->foo
= fxsave
->foo
;
486 env
->fos
= fxsave
->fos
;
489 for (i
= 0; i
< 8; ++i
)
490 memcpy(&to
[i
], &from
[i
], sizeof(to
[0]));
493 static void convert_to_fxsr(struct task_struct
*tsk
,
494 const struct user_i387_ia32_struct
*env
)
497 struct i387_fxsave_struct
*fxsave
= &tsk
->thread
.fpu
.state
->fxsave
;
498 struct _fpreg
*from
= (struct _fpreg
*) &env
->st_space
[0];
499 struct _fpxreg
*to
= (struct _fpxreg
*) &fxsave
->st_space
[0];
502 fxsave
->cwd
= env
->cwd
;
503 fxsave
->swd
= env
->swd
;
504 fxsave
->twd
= twd_i387_to_fxsr(env
->twd
);
505 fxsave
->fop
= (u16
) ((u32
) env
->fcs
>> 16);
507 fxsave
->rip
= env
->fip
;
508 fxsave
->rdp
= env
->foo
;
509 /* cs and ds ignored */
511 fxsave
->fip
= env
->fip
;
512 fxsave
->fcs
= (env
->fcs
& 0xffff);
513 fxsave
->foo
= env
->foo
;
514 fxsave
->fos
= env
->fos
;
517 for (i
= 0; i
< 8; ++i
)
518 memcpy(&to
[i
], &from
[i
], sizeof(from
[0]));
521 int fpregs_get(struct task_struct
*target
, const struct user_regset
*regset
,
522 unsigned int pos
, unsigned int count
,
523 void *kbuf
, void __user
*ubuf
)
525 struct user_i387_ia32_struct env
;
528 ret
= init_fpu(target
);
533 return fpregs_soft_get(target
, regset
, pos
, count
, kbuf
, ubuf
);
536 return user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
,
537 &target
->thread
.fpu
.state
->fsave
, 0,
541 sanitize_i387_state(target
);
543 if (kbuf
&& pos
== 0 && count
== sizeof(env
)) {
544 convert_from_fxsr(kbuf
, target
);
548 convert_from_fxsr(&env
, target
);
550 return user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
, &env
, 0, -1);
553 int fpregs_set(struct task_struct
*target
, const struct user_regset
*regset
,
554 unsigned int pos
, unsigned int count
,
555 const void *kbuf
, const void __user
*ubuf
)
557 struct user_i387_ia32_struct env
;
560 ret
= init_fpu(target
);
564 sanitize_i387_state(target
);
567 return fpregs_soft_set(target
, regset
, pos
, count
, kbuf
, ubuf
);
570 return user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
,
571 &target
->thread
.fpu
.state
->fsave
, 0, -1);
574 if (pos
> 0 || count
< sizeof(env
))
575 convert_from_fxsr(&env
, target
);
577 ret
= user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
, &env
, 0, -1);
579 convert_to_fxsr(target
, &env
);
582 * update the header bit in the xsave header, indicating the
586 target
->thread
.fpu
.state
->xsave
.xsave_hdr
.xstate_bv
|= XSTATE_FP
;
591 * Signal frame handlers.
594 static inline int save_i387_fsave(struct _fpstate_ia32 __user
*buf
)
596 struct task_struct
*tsk
= current
;
597 struct i387_fsave_struct
*fp
= &tsk
->thread
.fpu
.state
->fsave
;
599 fp
->status
= fp
->swd
;
600 if (__copy_to_user(buf
, fp
, sizeof(struct i387_fsave_struct
)))
605 static int save_i387_fxsave(struct _fpstate_ia32 __user
*buf
)
607 struct task_struct
*tsk
= current
;
608 struct i387_fxsave_struct
*fx
= &tsk
->thread
.fpu
.state
->fxsave
;
609 struct user_i387_ia32_struct env
;
612 convert_from_fxsr(&env
, tsk
);
613 if (__copy_to_user(buf
, &env
, sizeof(env
)))
616 err
|= __put_user(fx
->swd
, &buf
->status
);
617 err
|= __put_user(X86_FXSR_MAGIC
, &buf
->magic
);
621 if (__copy_to_user(&buf
->_fxsr_env
[0], fx
, xstate_size
))
626 static int save_i387_xsave(void __user
*buf
)
628 struct task_struct
*tsk
= current
;
629 struct _fpstate_ia32 __user
*fx
= buf
;
633 sanitize_i387_state(tsk
);
636 * For legacy compatible, we always set FP/SSE bits in the bit
637 * vector while saving the state to the user context.
638 * This will enable us capturing any changes(during sigreturn) to
639 * the FP/SSE bits by the legacy applications which don't touch
640 * xstate_bv in the xsave header.
642 * xsave aware applications can change the xstate_bv in the xsave
643 * header as well as change any contents in the memory layout.
644 * xrestore as part of sigreturn will capture all the changes.
646 tsk
->thread
.fpu
.state
->xsave
.xsave_hdr
.xstate_bv
|= XSTATE_FPSSE
;
648 if (save_i387_fxsave(fx
) < 0)
651 err
= __copy_to_user(&fx
->sw_reserved
, &fx_sw_reserved_ia32
,
652 sizeof(struct _fpx_sw_bytes
));
653 err
|= __put_user(FP_XSTATE_MAGIC2
,
654 (__u32 __user
*) (buf
+ sig_xstate_ia32_size
655 - FP_XSTATE_MAGIC2_SIZE
));
662 int save_i387_xstate_ia32(void __user
*buf
)
664 struct _fpstate_ia32 __user
*fp
= (struct _fpstate_ia32 __user
*) buf
;
665 struct task_struct
*tsk
= current
;
670 if (!access_ok(VERIFY_WRITE
, buf
, sig_xstate_ia32_size
))
673 * This will cause a "finit" to be triggered by the next
674 * attempted FPU operation by the 'current' process.
679 return fpregs_soft_get(current
, NULL
,
680 0, sizeof(struct user_i387_ia32_struct
),
687 return save_i387_xsave(fp
);
689 return save_i387_fxsave(fp
);
691 return save_i387_fsave(fp
);
694 static inline int restore_i387_fsave(struct _fpstate_ia32 __user
*buf
)
696 struct task_struct
*tsk
= current
;
698 return __copy_from_user(&tsk
->thread
.fpu
.state
->fsave
, buf
,
699 sizeof(struct i387_fsave_struct
));
702 static int restore_i387_fxsave(struct _fpstate_ia32 __user
*buf
,
705 struct task_struct
*tsk
= current
;
706 struct user_i387_ia32_struct env
;
709 err
= __copy_from_user(&tsk
->thread
.fpu
.state
->fxsave
, &buf
->_fxsr_env
[0],
711 /* mxcsr reserved bits must be masked to zero for security reasons */
712 tsk
->thread
.fpu
.state
->fxsave
.mxcsr
&= mxcsr_feature_mask
;
713 if (err
|| __copy_from_user(&env
, buf
, sizeof(env
)))
715 convert_to_fxsr(tsk
, &env
);
720 static int restore_i387_xsave(void __user
*buf
)
722 struct _fpx_sw_bytes fx_sw_user
;
723 struct _fpstate_ia32 __user
*fx_user
=
724 ((struct _fpstate_ia32 __user
*) buf
);
725 struct i387_fxsave_struct __user
*fx
=
726 (struct i387_fxsave_struct __user
*) &fx_user
->_fxsr_env
[0];
727 struct xsave_hdr_struct
*xsave_hdr
=
728 ¤t
->thread
.fpu
.state
->xsave
.xsave_hdr
;
732 if (check_for_xstate(fx
, buf
, &fx_sw_user
))
735 mask
= fx_sw_user
.xstate_bv
;
737 err
= restore_i387_fxsave(buf
, fx_sw_user
.xstate_size
);
739 xsave_hdr
->xstate_bv
&= pcntxt_mask
;
741 * These bits must be zero.
743 xsave_hdr
->reserved1
[0] = xsave_hdr
->reserved1
[1] = 0;
746 * Init the state that is not present in the memory layout
747 * and enabled by the OS.
749 mask
= ~(pcntxt_mask
& ~mask
);
750 xsave_hdr
->xstate_bv
&= mask
;
755 * Couldn't find the extended state information in the memory
756 * layout. Restore the FP/SSE and init the other extended state
759 xsave_hdr
->xstate_bv
= XSTATE_FPSSE
;
760 return restore_i387_fxsave(buf
, sizeof(struct i387_fxsave_struct
));
763 int restore_i387_xstate_ia32(void __user
*buf
)
766 struct task_struct
*tsk
= current
;
767 struct _fpstate_ia32 __user
*fp
= (struct _fpstate_ia32 __user
*) buf
;
780 if (!access_ok(VERIFY_READ
, buf
, sig_xstate_ia32_size
))
791 err
= restore_i387_xsave(buf
);
792 else if (cpu_has_fxsr
)
793 err
= restore_i387_fxsave(fp
, sizeof(struct
794 i387_fxsave_struct
));
796 err
= restore_i387_fsave(fp
);
798 err
= fpregs_soft_set(current
, NULL
,
799 0, sizeof(struct user_i387_ia32_struct
),
808 * FPU state for core dumps.
809 * This is only used for a.out dumps now.
810 * It is declared generically using elf_fpregset_t (which is
811 * struct user_i387_struct) but is in fact only used for 32-bit
812 * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
814 int dump_fpu(struct pt_regs
*regs
, struct user_i387_struct
*fpu
)
816 struct task_struct
*tsk
= current
;
819 fpvalid
= !!used_math();
821 fpvalid
= !fpregs_get(tsk
, NULL
,
822 0, sizeof(struct user_i387_ia32_struct
),
827 EXPORT_SYMBOL(dump_fpu
);
829 #endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */