arch/i386/kernel/i387.c

   1 /*
   2  *  linux/arch/i386/kernel/i387.c
   3  *
   4  *  Copyright (C) 1994 Linus Torvalds
   5  *
   6  *  Pentium III FXSR, SSE support
   7  *  General FPU state handling cleanups
   8  *      Gareth Hughes <gareth@valinux.com>, May 2000
   9  */
  10
  11 #include <linux/config.h>
  12 #include <linux/sched.h>
  13 #include <asm/processor.h>
  14 #include <asm/i387.h>
  15 #include <asm/math_emu.h>
  16 #include <asm/sigcontext.h>
  17 #include <asm/user.h>
  18 #include <asm/ptrace.h>
  19 #include <asm/uaccess.h>
  20
  21 #if defined(CONFIG_X86_FXSR)
  22 #define HAVE_FXSR 1
  23 #elif defined(CONFIG_X86_RUNTIME_FXSR)
  24 #define HAVE_FXSR (cpu_has_fxsr)
  25 #else
  26 #define HAVE_FXSR 0
  27 #endif
  28
  29 #ifdef CONFIG_MATH_EMULATION
  30 #define HAVE_HWFP (boot_cpu_data.hard_math)
  31 #else
  32 #define HAVE_HWFP 1
  33 #endif
  34
  35 /*
  36  * The _current_ task is using the FPU for the first time
  37  * so initialize it and set the mxcsr to its default
  38  * value at reset if we support FXSR and then
  39  * remeber the current task has used the FPU.
  40  */
  41 void init_fpu(void)
  42 {
  43         __asm__("fninit");
  44         if ( HAVE_FXSR )
  45                 load_mxcsr(0x1f80);
  46
  47         current->used_math = 1;
  48 }
  49
  50 /*
  51  * FPU lazy state save handling.
  52  */
  53
  54 void save_init_fpu( struct task_struct *tsk )
  55 {
  56         if ( HAVE_FXSR ) {
  57                 asm volatile( "fxsave %0 ; fnclex"
  58                               : "=m" (tsk->thread.i387.fxsave) );
  59         } else {
  60                 asm volatile( "fnsave %0 ; fwait"
  61                               : "=m" (tsk->thread.i387.fsave) );
  62         }
  63         tsk->flags &= ~PF_USEDFPU;
  64         stts();
  65 }
  66
  67 void restore_fpu( struct task_struct *tsk )
  68 {
  69         if ( HAVE_FXSR ) {
  70                 asm volatile( "fxrstor %0"
  71                               : : "m" (tsk->thread.i387.fxsave) );
  72         } else {
  73                 asm volatile( "frstor %0"
  74                               : : "m" (tsk->thread.i387.fsave) );
  75         }
  76 }
  77
  78 /*
  79  * FPU tag word conversions.
  80  */
  81
  82 static inline unsigned short twd_i387_to_fxsr( unsigned short twd )
  83 {
  84         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
  85
  86         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
  87         tmp = ~twd;
  88         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
  89         /* and move the valid bits to the lower byte. */
  90         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
  91         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
  92         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
  93         return tmp;
  94 }
  95
  96 static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave )
  97 {
  98         struct _fpxreg *st = NULL;
  99         unsigned long twd = (unsigned long) fxsave->twd;
 100         unsigned long tag;
 101         unsigned long ret = 0xffff0000;
 102         int i;
 103
 104 #define FPREG_ADDR(f, n)        ((char *)&(f)->st_space + (n) * 16);
 105
 106         for ( i = 0 ; i < 8 ; i++ ) {
 107                 if ( twd & 0x1 ) {
 108                         st = (struct _fpxreg *) FPREG_ADDR( fxsave, i );
 109
 110                         switch ( st->exponent & 0x7fff ) {
 111                         case 0x7fff:
 112                                 tag = 2;                /* Special */
 113                                 break;
 114                         case 0x0000:
 115                                 if ( !st->significand[0] &&
 116                                      !st->significand[1] &&
 117                                      !st->significand[2] &&
 118                                      !st->significand[3] ) {
 119                                         tag = 1;        /* Zero */
 120                                 } else {
 121                                         tag = 2;        /* Special */
 122                                 }
 123                                 break;
 124                         default:
 125                                 if ( st->significand[3] & 0x8000 ) {
 126                                         tag = 0;        /* Valid */
 127                                 } else {
 128                                         tag = 2;        /* Special */
 129                                 }
 130                                 break;
 131                         }
 132                 } else {
 133                         tag = 3;                        /* Empty */
 134                 }
 135                 ret |= (tag << (2 * i));
 136                 twd = twd >> 1;
 137         }
 138         return ret;
 139 }
 140
 141 /*
 142  * FPU state interaction.
 143  */
 144
 145 unsigned short get_fpu_cwd( struct task_struct *tsk )
 146 {
 147         if ( HAVE_FXSR ) {
 148                 return tsk->thread.i387.fxsave.cwd;
 149         } else {
 150                 return (unsigned short)tsk->thread.i387.fsave.cwd;
 151         }
 152 }
 153
 154 unsigned short get_fpu_swd( struct task_struct *tsk )
 155 {
 156         if ( HAVE_FXSR ) {
 157                 return tsk->thread.i387.fxsave.swd;
 158         } else {
 159                 return (unsigned short)tsk->thread.i387.fsave.swd;
 160         }
 161 }
 162
 163 unsigned short get_fpu_twd( struct task_struct *tsk )
 164 {
 165         if ( HAVE_FXSR ) {
 166                 return tsk->thread.i387.fxsave.twd;
 167         } else {
 168                 return (unsigned short)tsk->thread.i387.fsave.twd;
 169         }
 170 }
 171
 172 unsigned short get_fpu_mxcsr( struct task_struct *tsk )
 173 {
 174         if ( HAVE_FXSR ) {
 175                 return tsk->thread.i387.fxsave.mxcsr;
 176         } else {
 177                 return 0x1f80;
 178         }
 179 }
 180
 181 void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd )
 182 {
 183         if ( HAVE_FXSR ) {
 184                 tsk->thread.i387.fxsave.cwd = cwd;
 185         } else {
 186                 tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000);
 187         }
 188 }
 189
 190 void set_fpu_swd( struct task_struct *tsk, unsigned short swd )
 191 {
 192         if ( HAVE_FXSR ) {
 193                 tsk->thread.i387.fxsave.swd = swd;
 194         } else {
 195                 tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000);
 196         }
 197 }
 198
 199 void set_fpu_twd( struct task_struct *tsk, unsigned short twd )
 200 {
 201         if ( HAVE_FXSR ) {
 202                 tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd);
 203         } else {
 204                 tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000);
 205         }
 206 }
 207
 208 void set_fpu_mxcsr( struct task_struct *tsk, unsigned short mxcsr )
 209 {
 210         if ( HAVE_FXSR ) {
 211                 tsk->thread.i387.fxsave.mxcsr = mxcsr;
 212         }
 213 }
 214
 215 /*
 216  * FXSR floating point environment conversions.
 217  */
 218
 219 static inline int convert_fxsr_to_user( struct _fpstate *buf,
 220                                         struct i387_fxsave_struct *fxsave )
 221 {
 222         unsigned long env[7];
 223         struct _fpreg *to;
 224         struct _fpxreg *from;
 225         int i;
 226
 227         env[0] = (unsigned long)fxsave->cwd | 0xffff0000;
 228         env[1] = (unsigned long)fxsave->swd | 0xffff0000;
 229         env[2] = twd_fxsr_to_i387(fxsave);
 230         env[3] = fxsave->fip;
 231         env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16);
 232         env[5] = fxsave->foo;
 233         env[6] = fxsave->fos;
 234
 235         if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) )
 236                 return 1;
 237
 238         to = &buf->_st[0];
 239         from = (struct _fpxreg *) &fxsave->st_space[0];
 240         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
 241                 if ( __copy_to_user( to, from, sizeof(*to) ) )
 242                         return 1;
 243         }
 244         return 0;
 245 }
 246
 247 static inline int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave,
 248                                           struct _fpstate *buf )
 249 {
 250         unsigned long env[7];
 251         struct _fpxreg *to;
 252         struct _fpreg *from;
 253         int i;
 254
 255         if ( __copy_from_user( env, buf, 7 * sizeof(long) ) )
 256                 return 1;
 257
 258         fxsave->cwd = (unsigned short)(env[0] & 0xffff);
 259         fxsave->swd = (unsigned short)(env[1] & 0xffff);
 260         fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff));
 261         fxsave->fip = env[3];
 262         fxsave->fop = (unsigned short)((env[4] & 0xffff0000) >> 16);
 263         fxsave->fcs = (env[4] & 0xffff);
 264         fxsave->foo = env[5];
 265         fxsave->fos = env[6];
 266
 267         to = (struct _fpxreg *) &fxsave->st_space[0];
 268         from = &buf->_st[0];
 269         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
 270                 if ( __copy_from_user( to, from, sizeof(*from) ) )
 271                         return 1;
 272         }
 273         return 0;
 274 }
 275
 276 /*
 277  * Signal frame handlers.
 278  */
 279
 280 static inline int save_i387_fsave( struct _fpstate *buf )
 281 {
 282         struct task_struct *tsk = current;
 283
 284         unlazy_fpu( tsk );
 285         tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd;
 286         if ( __copy_to_user( buf, &tsk->thread.i387.fsave,
 287                              sizeof(struct i387_fsave_struct) ) )
 288                 return -1;
 289         return 1;
 290 }
 291
 292 static inline int save_i387_fxsave( struct _fpstate *buf )
 293 {
 294         struct task_struct *tsk = current;
 295         int err = 0;
 296
 297         unlazy_fpu( tsk );
 298
 299         if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) )
 300                 return -1;
 301
 302         err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status );
 303         err |= __put_user( X86_FXSR_MAGIC, &buf->magic );
 304         if ( err )
 305                 return -1;
 306
 307         if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave,
 308                              sizeof(struct i387_fxsave_struct) ) )
 309                 return -1;
 310         return 1;
 311 }
 312
 313 int save_i387( struct _fpstate *buf )
 314 {
 315         if ( !current->used_math )
 316                 return 0;
 317
 318         /* This will cause a "finit" to be triggered by the next
 319          * attempted FPU operation by the 'current' process.
 320          */
 321         current->used_math = 0;
 322
 323         if ( HAVE_HWFP ) {
 324                 if ( HAVE_FXSR ) {
 325                         return save_i387_fxsave( buf );
 326                 } else {
 327                         return save_i387_fsave( buf );
 328                 }
 329         } else {
 330                 return save_i387_soft( &current->thread.i387.soft, buf );
 331         }
 332 }
 333
 334 static inline int restore_i387_fsave( struct _fpstate *buf )
 335 {
 336         struct task_struct *tsk = current;
 337         clear_fpu( tsk );
 338         return __copy_from_user( &tsk->thread.i387.fsave, buf,
 339                                  sizeof(struct i387_fsave_struct) );
 340 }
 341
 342 static inline int restore_i387_fxsave( struct _fpstate *buf )
 343 {
 344         struct task_struct *tsk = current;
 345         clear_fpu( tsk );
 346         if ( __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0],
 347                                sizeof(struct i387_fxsave_struct) ) )
 348                 return 1;
 349         return convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf );
 350 }
 351
 352 int restore_i387( struct _fpstate *buf )
 353 {
 354         int err;
 355
 356         if ( HAVE_HWFP ) {
 357                 if ( HAVE_FXSR ) {
 358                         err =  restore_i387_fxsave( buf );
 359                 } else {
 360                         err = restore_i387_fsave( buf );
 361                 }
 362         } else {
 363                 err = restore_i387_soft( &current->thread.i387.soft, buf );
 364         }
 365         current->used_math = 1;
 366         return err;
 367 }
 368
 369 /*
 370  * ptrace request handlers.
 371  */
 372
 373 static inline int get_fpregs_fsave( struct user_i387_struct *buf,
 374                                     struct task_struct *tsk )
 375 {
 376         return __copy_to_user( buf, &tsk->thread.i387.fsave,
 377                                sizeof(struct user_i387_struct) );
 378 }
 379
 380 static inline int get_fpregs_fxsave( struct user_i387_struct *buf,
 381                                      struct task_struct *tsk )
 382 {
 383         return convert_fxsr_to_user( (struct _fpstate *)buf,
 384                                      &tsk->thread.i387.fxsave );
 385 }
 386
 387 int get_fpregs( struct user_i387_struct *buf, struct task_struct *tsk )
 388 {
 389         if ( HAVE_HWFP ) {
 390                 if ( HAVE_FXSR ) {
 391                         return get_fpregs_fxsave( buf, tsk );
 392                 } else {
 393                         return get_fpregs_fsave( buf, tsk );
 394                 }
 395         } else {
 396                 return save_i387_soft( &tsk->thread.i387.soft,
 397                                        (struct _fpstate *)buf );
 398         }
 399 }
 400
 401 static inline int set_fpregs_fsave( struct task_struct *tsk,
 402                                     struct user_i387_struct *buf )
 403 {
 404         return __copy_from_user( &tsk->thread.i387.fsave, buf,
 405                                  sizeof(struct user_i387_struct) );
 406 }
 407
 408 static inline int set_fpregs_fxsave( struct task_struct *tsk,
 409                                      struct user_i387_struct *buf )
 410 {
 411         return convert_fxsr_from_user( &tsk->thread.i387.fxsave,
 412                                        (struct _fpstate *)buf );
 413 }
 414
 415 int set_fpregs( struct task_struct *tsk, struct user_i387_struct *buf )
 416 {
 417         if ( HAVE_HWFP ) {
 418                 if ( HAVE_FXSR ) {
 419                         return set_fpregs_fxsave( tsk, buf );
 420                 } else {
 421                         return set_fpregs_fsave( tsk, buf );
 422                 }
 423         } else {
 424                 return restore_i387_soft( &tsk->thread.i387.soft,
 425                                           (struct _fpstate *)buf );
 426         }
 427 }
 428
 429 int get_fpxregs( struct user_fxsr_struct *buf, struct task_struct *tsk )
 430 {
 431         if ( HAVE_FXSR ) {
 432                 __copy_to_user( (void *)buf, &tsk->thread.i387.fxsave,
 433                                 sizeof(struct user_fxsr_struct) );
 434                 return 0;
 435         } else {
 436                 return -EIO;
 437         }
 438 }
 439
 440 int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct *buf )
 441 {
 442         if ( HAVE_FXSR ) {
 443                 __copy_from_user( &tsk->thread.i387.fxsave, (void *)buf,
 444                                   sizeof(struct user_fxsr_struct) );
 445                 /* mxcsr bit 6 and 31-16 must be zero for security reasons */
 446                 tsk->thread.i387.fxsave.mxcsr &= 0xffbf;
 447                 return 0;
 448         } else {
 449                 return -EIO;
 450         }
 451 }
 452
 453 /*
 454  * FPU state for core dumps.
 455  */
 456
 457 static inline void copy_fpu_fsave( struct task_struct *tsk,
 458                                    struct user_i387_struct *fpu )
 459 {
 460         memcpy( fpu, &tsk->thread.i387.fsave,
 461                 sizeof(struct user_i387_struct) );
 462 }
 463
 464 static inline void copy_fpu_fxsave( struct task_struct *tsk,
 465                                    struct user_i387_struct *fpu )
 466 {
 467         unsigned short *to;
 468         unsigned short *from;
 469         int i;
 470
 471         memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) );
 472
 473         to = (unsigned short *)&fpu->st_space[0];
 474         from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0];
 475         for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) {
 476                 memcpy( to, from, 5 * sizeof(unsigned short) );
 477         }
 478 }
 479
 480 int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu )
 481 {
 482         int fpvalid;
 483         struct task_struct *tsk = current;
 484
 485         fpvalid = tsk->used_math;
 486         if ( fpvalid ) {
 487                 unlazy_fpu( tsk );
 488                 if ( HAVE_FXSR ) {
 489                         copy_fpu_fxsave( tsk, fpu );
 490                 } else {
 491                         copy_fpu_fsave( tsk, fpu );
 492                 }
 493         }
 494
 495         return fpvalid;
 496 }
 497
 498 int dump_extended_fpu( struct pt_regs *regs, struct user_fxsr_struct *fpu )
 499 {
 500         int fpvalid;
 501         struct task_struct *tsk = current;
 502
 503         fpvalid = tsk->used_math && HAVE_FXSR;
 504         if ( fpvalid ) {
 505                 unlazy_fpu( tsk );
 506                 memcpy( fpu, &tsk->thread.i387.fxsave,
 507                         sizeof(struct user_fxsr_struct) );
 508         }
 509
 510         return fpvalid;
 511 }