arch/i386/kernel/i387.c

   1 /*
   2  *  linux/arch/i386/kernel/i387.c
   3  *
   4  *  Copyright (C) 1994 Linus Torvalds
   5  *
   6  *  Pentium III FXSR, SSE support
   7  *  General FPU state handling cleanups
   8  *      Gareth Hughes <gareth@valinux.com>, May 2000
   9  */
  10
  11 #include <linux/config.h>
  12 #include <linux/sched.h>
  13 #include <asm/processor.h>
  14 #include <asm/i387.h>
  15 #include <asm/math_emu.h>
  16 #include <asm/sigcontext.h>
  17 #include <asm/user.h>
  18 #include <asm/ptrace.h>
  19 #include <asm/uaccess.h>
  20
  21 #if defined(CONFIG_X86_FXSR)
  22 #define HAVE_FXSR 1
  23 #elif defined(CONFIG_X86_RUNTIME_FXSR)
  24 #define HAVE_FXSR (cpu_has_fxsr)
  25 #else
  26 #define HAVE_FXSR 0
  27 #endif
  28
  29 #ifdef CONFIG_MATH_EMULATION
  30 #define HAVE_HWFP (boot_cpu_data.hard_math)
  31 #else
  32 #define HAVE_HWFP 1
  33 #endif
  34
  35 /*
  36  * FPU lazy state save handling.
  37  */
  38
  39 void save_fpu( struct task_struct *tsk )
  40 {
  41         if ( HAVE_FXSR ) {
  42                 asm volatile( "fxsave %0 ; fwait"
  43                               : "=m" (tsk->thread.i387.fxsave) );
  44         } else {
  45                 asm volatile( "fnsave %0 ; fwait"
  46                               : "=m" (tsk->thread.i387.fsave) );
  47         }
  48         tsk->flags &= ~PF_USEDFPU;
  49         stts();
  50 }
  51
  52 void save_init_fpu( struct task_struct *tsk )
  53 {
  54         if ( HAVE_FXSR ) {
  55                 asm volatile( "fxsave %0 ; fnclex"
  56                               : "=m" (tsk->thread.i387.fxsave) );
  57         } else {
  58                 asm volatile( "fnsave %0 ; fwait"
  59                               : "=m" (tsk->thread.i387.fsave) );
  60         }
  61         tsk->flags &= ~PF_USEDFPU;
  62         stts();
  63 }
  64
  65 void restore_fpu( struct task_struct *tsk )
  66 {
  67         if ( HAVE_FXSR ) {
  68                 asm volatile( "fxrstor %0"
  69                               : : "m" (tsk->thread.i387.fxsave) );
  70         } else {
  71                 asm volatile( "frstor %0"
  72                               : : "m" (tsk->thread.i387.fsave) );
  73         }
  74 }
  75
  76 /*
  77  * FPU tag word conversions.
  78  */
  79
  80 static inline unsigned short twd_i387_to_fxsr( unsigned short twd )
  81 {
  82         unsigned short ret = 0;
  83         int i;
  84
  85         for ( i = 0 ; i < 8 ; i++ ) {
  86                 if ( (twd & 0x3) != 0x3 ) {
  87                         ret |= (1 << i);
  88                 }
  89                 twd = twd >> 2;
  90         }
  91         return ret;
  92 }
  93
  94 static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave )
  95 {
  96         struct _fpxreg *st = NULL;
  97         unsigned long twd = (unsigned long) fxsave->twd;
  98         unsigned long tag;
  99         unsigned long ret = 0xffff0000;
 100         int i;
 101
 102 #define FPREG_ADDR(f, n)        ((char *)&(f)->st_space + (n) * 16);
 103
 104         for ( i = 0 ; i < 8 ; i++ ) {
 105                 if ( twd & 0x1 ) {
 106                         st = (struct _fpxreg *) FPREG_ADDR( fxsave, i );
 107
 108                         switch ( st->exponent ) {
 109                         case 0xffff:
 110                                 tag = 2;                /* Special */
 111                                 break;
 112                         case 0x0000:
 113                                 if ( !st->significand[0] &&
 114                                      !st->significand[1] &&
 115                                      !st->significand[2] &&
 116                                      !st->significand[3] ) {
 117                                         tag = 1;        /* Zero */
 118                                 } else {
 119                                         tag = 2;        /* Special */
 120                                 }
 121                                 break;
 122                         default:
 123                                 if ( st->significand[3] & 0x8000 ) {
 124                                         tag = 0;        /* Valid */
 125                                 } else {
 126                                         tag = 2;        /* Special */
 127                                 }
 128                                 break;
 129                         }
 130                 } else {
 131                         tag = 3;                        /* Empty */
 132                 }
 133                 ret |= (tag << (2 * i));
 134                 twd = twd >> 1;
 135         }
 136         return ret;
 137 }
 138
 139 /*
 140  * FPU state interaction.
 141  */
 142
 143 unsigned short get_fpu_cwd( struct task_struct *tsk )
 144 {
 145         if ( HAVE_FXSR ) {
 146                 return tsk->thread.i387.fxsave.cwd;
 147         } else {
 148                 return (unsigned short)tsk->thread.i387.fsave.cwd;
 149         }
 150 }
 151
 152 unsigned short get_fpu_swd( struct task_struct *tsk )
 153 {
 154         if ( HAVE_FXSR ) {
 155                 return tsk->thread.i387.fxsave.swd;
 156         } else {
 157                 return (unsigned short)tsk->thread.i387.fsave.swd;
 158         }
 159 }
 160
 161 unsigned short get_fpu_twd( struct task_struct *tsk )
 162 {
 163         if ( HAVE_FXSR ) {
 164                 return tsk->thread.i387.fxsave.twd;
 165         } else {
 166                 return (unsigned short)tsk->thread.i387.fsave.twd;
 167         }
 168 }
 169
 170 unsigned short get_fpu_mxcsr( struct task_struct *tsk )
 171 {
 172         if ( HAVE_FXSR ) {
 173                 return tsk->thread.i387.fxsave.mxcsr;
 174         } else {
 175                 return 0x1f80;
 176         }
 177 }
 178
 179 void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd )
 180 {
 181         if ( HAVE_FXSR ) {
 182                 tsk->thread.i387.fxsave.cwd = cwd;
 183         } else {
 184                 tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000);
 185         }
 186 }
 187
 188 void set_fpu_swd( struct task_struct *tsk, unsigned short swd )
 189 {
 190         if ( HAVE_FXSR ) {
 191                 tsk->thread.i387.fxsave.swd = swd;
 192         } else {
 193                 tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000);
 194         }
 195 }
 196
 197 void set_fpu_twd( struct task_struct *tsk, unsigned short twd )
 198 {
 199         if ( HAVE_FXSR ) {
 200                 tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd);
 201         } else {
 202                 tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000);
 203         }
 204 }
 205
 206 void set_fpu_mxcsr( struct task_struct *tsk, unsigned short mxcsr )
 207 {
 208         if ( HAVE_FXSR ) {
 209                 tsk->thread.i387.fxsave.mxcsr = mxcsr;
 210         }
 211 }
 212
 213 /*
 214  * FXSR floating point environment conversions.
 215  */
 216
 217 static inline int convert_fxsr_to_user( struct _fpstate *buf,
 218                                         struct i387_fxsave_struct *fxsave )
 219 {
 220         unsigned long env[7];
 221         struct _fpreg *to;
 222         struct _fpxreg *from;
 223         int i;
 224
 225         env[0] = (unsigned long)fxsave->cwd | 0xffff0000;
 226         env[1] = (unsigned long)fxsave->swd | 0xffff0000;
 227         env[2] = twd_fxsr_to_i387(fxsave);
 228         env[3] = fxsave->fip;
 229         env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16);
 230         env[5] = fxsave->foo;
 231         env[6] = fxsave->fos;
 232
 233         if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) )
 234                 return 1;
 235
 236         to = &buf->_st[0];
 237         from = (struct _fpxreg *) &fxsave->st_space[0];
 238         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
 239                 if ( __copy_to_user( to, from, sizeof(*to) ) )
 240                         return 1;
 241         }
 242         return 0;
 243 }
 244
 245 static inline int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave,
 246                                           struct _fpstate *buf )
 247 {
 248         unsigned long env[7];
 249         struct _fpxreg *to;
 250         struct _fpreg *from;
 251         int i;
 252
 253         if ( __copy_from_user( env, buf, 7 * sizeof(long) ) )
 254                 return 1;
 255
 256         fxsave->cwd = (unsigned short)(env[0] & 0xffff);
 257         fxsave->swd = (unsigned short)(env[1] & 0xffff);
 258         fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff));
 259         fxsave->fip = env[3];
 260         fxsave->fop = (unsigned short)((env[4] & 0xffff0000) >> 16);
 261         fxsave->fcs = (env[4] & 0xffff);
 262         fxsave->foo = env[5];
 263         fxsave->fos = env[6];
 264
 265         to = (struct _fpxreg *) &fxsave->st_space[0];
 266         from = &buf->_st[0];
 267         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
 268                 if ( __copy_from_user( to, from, sizeof(*from) ) )
 269                         return 1;
 270         }
 271         return 0;
 272 }
 273
 274 /*
 275  * Signal frame handlers.
 276  */
 277
 278 static inline int save_i387_fsave( struct _fpstate *buf )
 279 {
 280         struct task_struct *tsk = current;
 281
 282         unlazy_fpu( tsk );
 283         tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd;
 284         if ( __copy_to_user( buf, &tsk->thread.i387.fsave,
 285                              sizeof(struct i387_fsave_struct) ) )
 286                 return -1;
 287         return 1;
 288 }
 289
 290 static inline int save_i387_fxsave( struct _fpstate *buf )
 291 {
 292         struct task_struct *tsk = current;
 293         int err = 0;
 294
 295         unlazy_fpu( tsk );
 296
 297         if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) )
 298                 return -1;
 299
 300         err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status );
 301         err |= __put_user( X86_FXSR_MAGIC, &buf->magic );
 302         if ( err )
 303                 return -1;
 304
 305         if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave,
 306                              sizeof(struct i387_fxsave_struct) ) )
 307                 return -1;
 308         return 1;
 309 }
 310
 311 int save_i387( struct _fpstate *buf )
 312 {
 313         if ( !current->used_math )
 314                 return 0;
 315
 316         /* This will cause a "finit" to be triggered by the next
 317          * attempted FPU operation by the 'current' process.
 318          */
 319         current->used_math = 0;
 320
 321         if ( HAVE_HWFP ) {
 322                 if ( HAVE_FXSR ) {
 323                         return save_i387_fxsave( buf );
 324                 } else {
 325                         return save_i387_fsave( buf );
 326                 }
 327         } else {
 328                 return save_i387_soft( &current->thread.i387.soft, buf );
 329         }
 330 }
 331
 332 static inline int restore_i387_fsave( struct _fpstate *buf )
 333 {
 334         struct task_struct *tsk = current;
 335         clear_fpu( tsk );
 336         return __copy_from_user( &tsk->thread.i387.fsave, buf,
 337                                  sizeof(struct i387_fsave_struct) );
 338 }
 339
 340 static inline int restore_i387_fxsave( struct _fpstate *buf )
 341 {
 342         struct task_struct *tsk = current;
 343         clear_fpu( tsk );
 344         if ( __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0],
 345                                sizeof(struct i387_fxsave_struct) ) )
 346                 return 1;
 347         return convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf );
 348 }
 349
 350 int restore_i387( struct _fpstate *buf )
 351 {
 352         int err;
 353
 354         if ( HAVE_HWFP ) {
 355                 if ( HAVE_FXSR ) {
 356                         err =  restore_i387_fxsave( buf );
 357                 } else {
 358                         err = restore_i387_fsave( buf );
 359                 }
 360         } else {
 361                 err = restore_i387_soft( &current->thread.i387.soft, buf );
 362         }
 363         current->used_math = 1;
 364         return err;
 365 }
 366
 367 /*
 368  * ptrace request handlers.
 369  */
 370
 371 static inline int get_fpregs_fsave( struct user_i387_struct *buf,
 372                                     struct task_struct *tsk )
 373 {
 374         return __copy_to_user( buf, &tsk->thread.i387.fsave,
 375                                sizeof(struct user_i387_struct) );
 376 }
 377
 378 static inline int get_fpregs_fxsave( struct user_i387_struct *buf,
 379                                      struct task_struct *tsk )
 380 {
 381         return convert_fxsr_to_user( (struct _fpstate *)buf,
 382                                      &tsk->thread.i387.fxsave );
 383 }
 384
 385 int get_fpregs( struct user_i387_struct *buf, struct task_struct *tsk )
 386 {
 387         if ( HAVE_HWFP ) {
 388                 if ( HAVE_FXSR ) {
 389                         return get_fpregs_fxsave( buf, tsk );
 390                 } else {
 391                         return get_fpregs_fsave( buf, tsk );
 392                 }
 393         } else {
 394                 return save_i387_soft( &tsk->thread.i387.soft,
 395                                        (struct _fpstate *)buf );
 396         }
 397 }
 398
 399 static inline int set_fpregs_fsave( struct task_struct *tsk,
 400                                     struct user_i387_struct *buf )
 401 {
 402         return __copy_from_user( &tsk->thread.i387.fsave, buf,
 403                                  sizeof(struct user_i387_struct) );
 404 }
 405
 406 static inline int set_fpregs_fxsave( struct task_struct *tsk,
 407                                      struct user_i387_struct *buf )
 408 {
 409         return convert_fxsr_from_user( &tsk->thread.i387.fxsave,
 410                                        (struct _fpstate *)buf );
 411 }
 412
 413 int set_fpregs( struct task_struct *tsk, struct user_i387_struct *buf )
 414 {
 415         if ( HAVE_HWFP ) {
 416                 if ( HAVE_FXSR ) {
 417                         return set_fpregs_fxsave( tsk, buf );
 418                 } else {
 419                         return set_fpregs_fsave( tsk, buf );
 420                 }
 421         } else {
 422                 return restore_i387_soft( &tsk->thread.i387.soft,
 423                                           (struct _fpstate *)buf );
 424         }
 425 }
 426
 427 int get_fpxregs( struct user_fxsr_struct *buf, struct task_struct *tsk )
 428 {
 429         if ( HAVE_FXSR ) {
 430                 __copy_to_user( (void *)buf, &tsk->thread.i387.fxsave,
 431                                 sizeof(struct user_fxsr_struct) );
 432                 return 0;
 433         } else {
 434                 return -EIO;
 435         }
 436 }
 437
 438 int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct *buf )
 439 {
 440         if ( HAVE_FXSR ) {
 441                 __copy_from_user( &tsk->thread.i387.fxsave, (void *)buf,
 442                                   sizeof(struct user_fxsr_struct) );
 443                 return 0;
 444         } else {
 445                 return -EIO;
 446         }
 447 }
 448
 449 /*
 450  * FPU state for core dumps.
 451  */
 452
 453 static inline void copy_fpu_fsave( struct task_struct *tsk,
 454                                    struct user_i387_struct *fpu )
 455 {
 456         memcpy( fpu, &tsk->thread.i387.fsave,
 457                 sizeof(struct user_i387_struct) );
 458 }
 459
 460 static inline void copy_fpu_fxsave( struct task_struct *tsk,
 461                                    struct user_i387_struct *fpu )
 462 {
 463         unsigned short *to;
 464         unsigned short *from;
 465         int i;
 466
 467         memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) );
 468
 469         to = (unsigned short *)&fpu->st_space[0];
 470         from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0];
 471         for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) {
 472                 memcpy( to, from, 5 * sizeof(unsigned short) );
 473         }
 474 }
 475
 476 int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu )
 477 {
 478         int fpvalid;
 479         struct task_struct *tsk = current;
 480
 481         fpvalid = tsk->used_math;
 482         if ( fpvalid ) {
 483                 unlazy_fpu( tsk );
 484                 if ( HAVE_FXSR ) {
 485                         copy_fpu_fxsave( tsk, fpu );
 486                 } else {
 487                         copy_fpu_fsave( tsk, fpu );
 488                 }
 489         }
 490
 491         return fpvalid;
 492 }
 493
 494 int dump_extended_fpu( struct pt_regs *regs, struct user_fxsr_struct *fpu )
 495 {
 496         int fpvalid;
 497         struct task_struct *tsk = current;
 498
 499         fpvalid = tsk->used_math && HAVE_FXSR;
 500         if ( fpvalid ) {
 501                 unlazy_fpu( tsk );
 502                 memcpy( fpu, &tsk->thread.i387.fxsave,
 503                         sizeof(struct user_fxsr_struct) );
 504         }
 505
 506         return fpvalid;
 507 }