arch/x86_64/kernel/process.c

   1 /*
   2  *  linux/arch/x86-64/kernel/process.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *
   6  *  Pentium III FXSR, SSE support
   7  *      Gareth Hughes <gareth@valinux.com>, May 2000
   8  *
   9  *  X86-64 port
  10  *      Andi Kleen.
  11  *
  12  *      CPU hotplug support - ashok.raj@intel.com
  13  */
  14
  15 /*
  16  * This file handles the architecture-dependent parts of process handling..
  17  */
  18
  19 #include <stdarg.h>
  20
  21 #include <linux/cpu.h>
  22 #include <linux/errno.h>
  23 #include <linux/sched.h>
  24 #include <linux/kernel.h>
  25 #include <linux/mm.h>
  26 #include <linux/elfcore.h>
  27 #include <linux/smp.h>
  28 #include <linux/slab.h>
  29 #include <linux/user.h>
  30 #include <linux/module.h>
  31 #include <linux/a.out.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/delay.h>
  34 #include <linux/ptrace.h>
  35 #include <linux/utsname.h>
  36 #include <linux/random.h>
  37 #include <linux/notifier.h>
  38 #include <linux/kprobes.h>
  39
  40 #include <asm/uaccess.h>
  41 #include <asm/pgtable.h>
  42 #include <asm/system.h>
  43 #include <asm/io.h>
  44 #include <asm/processor.h>
  45 #include <asm/i387.h>
  46 #include <asm/mmu_context.h>
  47 #include <asm/pda.h>
  48 #include <asm/prctl.h>
  49 #include <asm/kdebug.h>
  50 #include <asm/desc.h>
  51 #include <asm/proto.h>
  52 #include <asm/ia32.h>
  53 #include <asm/idle.h>
  54
  55 asmlinkage extern void ret_from_fork(void);
  56
  57 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
  58
  59 unsigned long boot_option_idle_override = 0;
  60 EXPORT_SYMBOL(boot_option_idle_override);
  61
  62 /*
  63  * Powermanagement idle function, if any..
  64  */
  65 void (*pm_idle)(void);
  66 EXPORT_SYMBOL(pm_idle);
  67 static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
  68
  69 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
  70
  71 void idle_notifier_register(struct notifier_block *n)
  72 {
  73         atomic_notifier_chain_register(&idle_notifier, n);
  74 }
  75 EXPORT_SYMBOL_GPL(idle_notifier_register);
  76
  77 void idle_notifier_unregister(struct notifier_block *n)
  78 {
  79         atomic_notifier_chain_unregister(&idle_notifier, n);
  80 }
  81 EXPORT_SYMBOL(idle_notifier_unregister);
  82
  83 void enter_idle(void)
  84 {
  85         write_pda(isidle, 1);
  86         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
  87 }
  88
  89 static void __exit_idle(void)
  90 {
  91         if (test_and_clear_bit_pda(0, isidle) == 0)
  92                 return;
  93         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
  94 }
  95
  96 /* Called from interrupts to signify idle end */
  97 void exit_idle(void)
  98 {
  99         /* idle loop has pid 0 */
 100         if (current->pid)
 101                 return;
 102         __exit_idle();
 103 }
 104
 105 /*
 106  * We use this if we don't have any better
 107  * idle routine..
 108  */
 109 static void default_idle(void)
 110 {
 111         current_thread_info()->status &= ~TS_POLLING;
 112         /*
 113          * TS_POLLING-cleared state must be visible before we
 114          * test NEED_RESCHED:
 115          */
 116         smp_mb();
 117         local_irq_disable();
 118         if (!need_resched()) {
 119                 /* Enables interrupts one instruction before HLT.
 120                    x86 special cases this so there is no race. */
 121                 safe_halt();
 122         } else
 123                 local_irq_enable();
 124         current_thread_info()->status |= TS_POLLING;
 125 }
 126
 127 /*
 128  * On SMP it's slightly faster (but much more power-consuming!)
 129  * to poll the ->need_resched flag instead of waiting for the
 130  * cross-CPU IPI to arrive. Use this option with caution.
 131  */
 132 static void poll_idle (void)
 133 {
 134         local_irq_enable();
 135         cpu_relax();
 136 }
 137
 138 void cpu_idle_wait(void)
 139 {
 140         unsigned int cpu, this_cpu = get_cpu();
 141         cpumask_t map, tmp = current->cpus_allowed;
 142
 143         set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
 144         put_cpu();
 145
 146         cpus_clear(map);
 147         for_each_online_cpu(cpu) {
 148                 per_cpu(cpu_idle_state, cpu) = 1;
 149                 cpu_set(cpu, map);
 150         }
 151
 152         __get_cpu_var(cpu_idle_state) = 0;
 153
 154         wmb();
 155         do {
 156                 ssleep(1);
 157                 for_each_online_cpu(cpu) {
 158                         if (cpu_isset(cpu, map) &&
 159                                         !per_cpu(cpu_idle_state, cpu))
 160                                 cpu_clear(cpu, map);
 161                 }
 162                 cpus_and(map, map, cpu_online_map);
 163         } while (!cpus_empty(map));
 164
 165         set_cpus_allowed(current, tmp);
 166 }
 167 EXPORT_SYMBOL_GPL(cpu_idle_wait);
 168
 169 #ifdef CONFIG_HOTPLUG_CPU
 170 DECLARE_PER_CPU(int, cpu_state);
 171
 172 #include <asm/nmi.h>
 173 /* We halt the CPU with physical CPU hotplug */
 174 static inline void play_dead(void)
 175 {
 176         idle_task_exit();
 177         wbinvd();
 178         mb();
 179         /* Ack it */
 180         __get_cpu_var(cpu_state) = CPU_DEAD;
 181
 182         local_irq_disable();
 183         while (1)
 184                 halt();
 185 }
 186 #else
 187 static inline void play_dead(void)
 188 {
 189         BUG();
 190 }
 191 #endif /* CONFIG_HOTPLUG_CPU */
 192
 193 /*
 194  * The idle thread. There's no useful work to be
 195  * done, so just try to conserve power and have a
 196  * low exit latency (ie sit in a loop waiting for
 197  * somebody to say that they'd like to reschedule)
 198  */
 199 void cpu_idle (void)
 200 {
 201         current_thread_info()->status |= TS_POLLING;
 202         /* endless idle loop with no priority at all */
 203         while (1) {
 204                 while (!need_resched()) {
 205                         void (*idle)(void);
 206
 207                         if (__get_cpu_var(cpu_idle_state))
 208                                 __get_cpu_var(cpu_idle_state) = 0;
 209
 210                         rmb();
 211                         idle = pm_idle;
 212                         if (!idle)
 213                                 idle = default_idle;
 214                         if (cpu_is_offline(smp_processor_id()))
 215                                 play_dead();
 216                         /*
 217                          * Idle routines should keep interrupts disabled
 218                          * from here on, until they go to idle.
 219                          * Otherwise, idle callbacks can misfire.
 220                          */
 221                         local_irq_disable();
 222                         enter_idle();
 223                         idle();
 224                         /* In many cases the interrupt that ended idle
 225                            has already called exit_idle. But some idle
 226                            loops can be woken up without interrupt. */
 227                         __exit_idle();
 228                 }
 229
 230                 preempt_enable_no_resched();
 231                 schedule();
 232                 preempt_disable();
 233         }
 234 }
 235
 236 /*
 237  * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
 238  * which can obviate IPI to trigger checking of need_resched.
 239  * We execute MONITOR against need_resched and enter optimized wait state
 240  * through MWAIT. Whenever someone changes need_resched, we would be woken
 241  * up from MWAIT (without an IPI).
 242  *
 243  * New with Core Duo processors, MWAIT can take some hints based on CPU
 244  * capability.
 245  */
 246 void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
 247 {
 248         if (!need_resched()) {
 249                 __monitor((void *)&current_thread_info()->flags, 0, 0);
 250                 smp_mb();
 251                 if (!need_resched())
 252                         __mwait(eax, ecx);
 253         }
 254 }
 255
 256 /* Default MONITOR/MWAIT with no hints, used for default C1 state */
 257 static void mwait_idle(void)
 258 {
 259         if (!need_resched()) {
 260                 __monitor((void *)&current_thread_info()->flags, 0, 0);
 261                 smp_mb();
 262                 if (!need_resched())
 263                         __sti_mwait(0, 0);
 264                 else
 265                         local_irq_enable();
 266         } else {
 267                 local_irq_enable();
 268         }
 269 }
 270
 271 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
 272 {
 273         static int printed;
 274         if (cpu_has(c, X86_FEATURE_MWAIT)) {
 275                 /*
 276                  * Skip, if setup has overridden idle.
 277                  * One CPU supports mwait => All CPUs supports mwait
 278                  */
 279                 if (!pm_idle) {
 280                         if (!printed) {
 281                                 printk("using mwait in idle threads.\n");
 282                                 printed = 1;
 283                         }
 284                         pm_idle = mwait_idle;
 285                 }
 286         }
 287 }
 288
 289 static int __init idle_setup (char *str)
 290 {
 291         if (!strncmp(str, "poll", 4)) {
 292                 printk("using polling idle threads.\n");
 293                 pm_idle = poll_idle;
 294         }
 295
 296         boot_option_idle_override = 1;
 297         return 1;
 298 }
 299
 300 __setup("idle=", idle_setup);
 301
 302 /* Prints also some state that isn't saved in the pt_regs */
 303 void __show_regs(struct pt_regs * regs)
 304 {
 305         unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
 306         unsigned int fsindex,gsindex;
 307         unsigned int ds,cs,es;
 308
 309         printk("\n");
 310         print_modules();
 311         printk("Pid: %d, comm: %.20s %s %s %.*s\n",
 312                 current->pid, current->comm, print_tainted(),
 313                 init_utsname()->release,
 314                 (int)strcspn(init_utsname()->version, " "),
 315                 init_utsname()->version);
 316         printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
 317         printk_address(regs->rip);
 318         printk("RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss, regs->rsp,
 319                 regs->eflags);
 320         printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
 321                regs->rax, regs->rbx, regs->rcx);
 322         printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
 323                regs->rdx, regs->rsi, regs->rdi);
 324         printk("RBP: %016lx R08: %016lx R09: %016lx\n",
 325                regs->rbp, regs->r8, regs->r9);
 326         printk("R10: %016lx R11: %016lx R12: %016lx\n",
 327                regs->r10, regs->r11, regs->r12);
 328         printk("R13: %016lx R14: %016lx R15: %016lx\n",
 329                regs->r13, regs->r14, regs->r15);
 330
 331         asm("movl %%ds,%0" : "=r" (ds));
 332         asm("movl %%cs,%0" : "=r" (cs));
 333         asm("movl %%es,%0" : "=r" (es));
 334         asm("movl %%fs,%0" : "=r" (fsindex));
 335         asm("movl %%gs,%0" : "=r" (gsindex));
 336
 337         rdmsrl(MSR_FS_BASE, fs);
 338         rdmsrl(MSR_GS_BASE, gs);
 339         rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
 340
 341         asm("movq %%cr0, %0": "=r" (cr0));
 342         asm("movq %%cr2, %0": "=r" (cr2));
 343         asm("movq %%cr3, %0": "=r" (cr3));
 344         asm("movq %%cr4, %0": "=r" (cr4));
 345
 346         printk("FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
 347                fs,fsindex,gs,gsindex,shadowgs);
 348         printk("CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
 349         printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
 350 }
 351
 352 void show_regs(struct pt_regs *regs)
 353 {
 354         printk("CPU %d:", smp_processor_id());
 355         __show_regs(regs);
 356         show_trace(NULL, regs, (void *)(regs + 1));
 357 }
 358
 359 /*
 360  * Free current thread data structures etc..
 361  */
 362 void exit_thread(void)
 363 {
 364         struct task_struct *me = current;
 365         struct thread_struct *t = &me->thread;
 366
 367         if (me->thread.io_bitmap_ptr) {
 368                 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
 369
 370                 kfree(t->io_bitmap_ptr);
 371                 t->io_bitmap_ptr = NULL;
 372                 clear_thread_flag(TIF_IO_BITMAP);
 373                 /*
 374                  * Careful, clear this in the TSS too:
 375                  */
 376                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
 377                 t->io_bitmap_max = 0;
 378                 put_cpu();
 379         }
 380 }
 381
 382 void flush_thread(void)
 383 {
 384         struct task_struct *tsk = current;
 385
 386         if (test_tsk_thread_flag(tsk, TIF_ABI_PENDING)) {
 387                 clear_tsk_thread_flag(tsk, TIF_ABI_PENDING);
 388                 if (test_tsk_thread_flag(tsk, TIF_IA32)) {
 389                         clear_tsk_thread_flag(tsk, TIF_IA32);
 390                 } else {
 391                         set_tsk_thread_flag(tsk, TIF_IA32);
 392                         current_thread_info()->status |= TS_COMPAT;
 393                 }
 394         }
 395         clear_tsk_thread_flag(tsk, TIF_DEBUG);
 396
 397         tsk->thread.debugreg0 = 0;
 398         tsk->thread.debugreg1 = 0;
 399         tsk->thread.debugreg2 = 0;
 400         tsk->thread.debugreg3 = 0;
 401         tsk->thread.debugreg6 = 0;
 402         tsk->thread.debugreg7 = 0;
 403         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
 404         /*
 405          * Forget coprocessor state..
 406          */
 407         clear_fpu(tsk);
 408         clear_used_math();
 409 }
 410
 411 void release_thread(struct task_struct *dead_task)
 412 {
 413         if (dead_task->mm) {
 414                 if (dead_task->mm->context.size) {
 415                         printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
 416                                         dead_task->comm,
 417                                         dead_task->mm->context.ldt,
 418                                         dead_task->mm->context.size);
 419                         BUG();
 420                 }
 421         }
 422 }
 423
 424 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
 425 {
 426         struct user_desc ud = {
 427                 .base_addr = addr,
 428                 .limit = 0xfffff,
 429                 .seg_32bit = 1,
 430                 .limit_in_pages = 1,
 431                 .useable = 1,
 432         };
 433         struct n_desc_struct *desc = (void *)t->thread.tls_array;
 434         desc += tls;
 435         desc->a = LDT_entry_a(&ud);
 436         desc->b = LDT_entry_b(&ud);
 437 }
 438
 439 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
 440 {
 441         struct desc_struct *desc = (void *)t->thread.tls_array;
 442         desc += tls;
 443         return desc->base0 |
 444                 (((u32)desc->base1) << 16) |
 445                 (((u32)desc->base2) << 24);
 446 }
 447
 448 /*
 449  * This gets called before we allocate a new thread and copy
 450  * the current task into it.
 451  */
 452 void prepare_to_copy(struct task_struct *tsk)
 453 {
 454         unlazy_fpu(tsk);
 455 }
 456
 457 int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
 458                 unsigned long unused,
 459         struct task_struct * p, struct pt_regs * regs)
 460 {
 461         int err;
 462         struct pt_regs * childregs;
 463         struct task_struct *me = current;
 464
 465         childregs = ((struct pt_regs *)
 466                         (THREAD_SIZE + task_stack_page(p))) - 1;
 467         *childregs = *regs;
 468
 469         childregs->rax = 0;
 470         childregs->rsp = rsp;
 471         if (rsp == ~0UL)
 472                 childregs->rsp = (unsigned long)childregs;
 473
 474         p->thread.rsp = (unsigned long) childregs;
 475         p->thread.rsp0 = (unsigned long) (childregs+1);
 476         p->thread.userrsp = me->thread.userrsp;
 477
 478         set_tsk_thread_flag(p, TIF_FORK);
 479
 480         p->thread.fs = me->thread.fs;
 481         p->thread.gs = me->thread.gs;
 482
 483         asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
 484         asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
 485         asm("mov %%es,%0" : "=m" (p->thread.es));
 486         asm("mov %%ds,%0" : "=m" (p->thread.ds));
 487
 488         if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
 489                 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
 490                 if (!p->thread.io_bitmap_ptr) {
 491                         p->thread.io_bitmap_max = 0;
 492                         return -ENOMEM;
 493                 }
 494                 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
 495                                 IO_BITMAP_BYTES);
 496                 set_tsk_thread_flag(p, TIF_IO_BITMAP);
 497         }
 498
 499         /*
 500          * Set a new TLS for the child thread?
 501          */
 502         if (clone_flags & CLONE_SETTLS) {
 503 #ifdef CONFIG_IA32_EMULATION
 504                 if (test_thread_flag(TIF_IA32))
 505                         err = ia32_child_tls(p, childregs);
 506                 else
 507 #endif
 508                         err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
 509                 if (err)
 510                         goto out;
 511         }
 512         err = 0;
 513 out:
 514         if (err && p->thread.io_bitmap_ptr) {
 515                 kfree(p->thread.io_bitmap_ptr);
 516                 p->thread.io_bitmap_max = 0;
 517         }
 518         return err;
 519 }
 520
 521 /*
 522  * This special macro can be used to load a debugging register
 523  */
 524 #define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
 525
 526 static inline void __switch_to_xtra(struct task_struct *prev_p,
 527                                     struct task_struct *next_p,
 528                                     struct tss_struct *tss)
 529 {
 530         struct thread_struct *prev, *next;
 531
 532         prev = &prev_p->thread,
 533         next = &next_p->thread;
 534
 535         if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
 536                 loaddebug(next, 0);
 537                 loaddebug(next, 1);
 538                 loaddebug(next, 2);
 539                 loaddebug(next, 3);
 540                 /* no 4 and 5 */
 541                 loaddebug(next, 6);
 542                 loaddebug(next, 7);
 543         }
 544
 545         if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
 546                 /*
 547                  * Copy the relevant range of the IO bitmap.
 548                  * Normally this is 128 bytes or less:
 549                  */
 550                 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
 551                        max(prev->io_bitmap_max, next->io_bitmap_max));
 552         } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
 553                 /*
 554                  * Clear any possible leftover bits:
 555                  */
 556                 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
 557         }
 558 }
 559
 560 /*
 561  *      switch_to(x,y) should switch tasks from x to y.
 562  *
 563  * This could still be optimized:
 564  * - fold all the options into a flag word and test it with a single test.
 565  * - could test fs/gs bitsliced
 566  *
 567  * Kprobes not supported here. Set the probe on schedule instead.
 568  */
 569 __kprobes struct task_struct *
 570 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 571 {
 572         struct thread_struct *prev = &prev_p->thread,
 573                                  *next = &next_p->thread;
 574         int cpu = smp_processor_id();
 575         struct tss_struct *tss = &per_cpu(init_tss, cpu);
 576
 577         /* we're going to use this soon, after a few expensive things */
 578         if (next_p->fpu_counter>5)
 579                 prefetch(&next->i387.fxsave);
 580
 581         /*
 582          * Reload esp0, LDT and the page table pointer:
 583          */
 584         tss->rsp0 = next->rsp0;
 585
 586         /*
 587          * Switch DS and ES.
 588          * This won't pick up thread selector changes, but I guess that is ok.
 589          */
 590         asm volatile("mov %%es,%0" : "=m" (prev->es));
 591         if (unlikely(next->es | prev->es))
 592                 loadsegment(es, next->es);
 593
 594         asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
 595         if (unlikely(next->ds | prev->ds))
 596                 loadsegment(ds, next->ds);
 597
 598         load_TLS(next, cpu);
 599
 600         /*
 601          * Switch FS and GS.
 602          */
 603         {
 604                 unsigned fsindex;
 605                 asm volatile("movl %%fs,%0" : "=r" (fsindex));
 606                 /* segment register != 0 always requires a reload.
 607                    also reload when it has changed.
 608                    when prev process used 64bit base always reload
 609                    to avoid an information leak. */
 610                 if (unlikely(fsindex | next->fsindex | prev->fs)) {
 611                         loadsegment(fs, next->fsindex);
 612                         /* check if the user used a selector != 0
 613                          * if yes clear 64bit base, since overloaded base
 614                          * is always mapped to the Null selector
 615                          */
 616                         if (fsindex)
 617                         prev->fs = 0;
 618                 }
 619                 /* when next process has a 64bit base use it */
 620                 if (next->fs)
 621                         wrmsrl(MSR_FS_BASE, next->fs);
 622                 prev->fsindex = fsindex;
 623         }
 624         {
 625                 unsigned gsindex;
 626                 asm volatile("movl %%gs,%0" : "=r" (gsindex));
 627                 if (unlikely(gsindex | next->gsindex | prev->gs)) {
 628                         load_gs_index(next->gsindex);
 629                         if (gsindex)
 630                         prev->gs = 0;
 631                 }
 632                 if (next->gs)
 633                         wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
 634                 prev->gsindex = gsindex;
 635         }
 636
 637         /* Must be after DS reload */
 638         unlazy_fpu(prev_p);
 639
 640         /*
 641          * Switch the PDA and FPU contexts.
 642          */
 643         prev->userrsp = read_pda(oldrsp);
 644         write_pda(oldrsp, next->userrsp);
 645         write_pda(pcurrent, next_p);
 646
 647         write_pda(kernelstack,
 648         (unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
 649 #ifdef CONFIG_CC_STACKPROTECTOR
 650         write_pda(stack_canary, next_p->stack_canary);
 651         /*
 652          * Build time only check to make sure the stack_canary is at
 653          * offset 40 in the pda; this is a gcc ABI requirement
 654          */
 655         BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
 656 #endif
 657
 658         /*
 659          * Now maybe reload the debug registers and handle I/O bitmaps
 660          */
 661         if (unlikely((task_thread_info(next_p)->flags & _TIF_WORK_CTXSW))
 662             || test_tsk_thread_flag(prev_p, TIF_IO_BITMAP))
 663                 __switch_to_xtra(prev_p, next_p, tss);
 664
 665         /* If the task has used fpu the last 5 timeslices, just do a full
 666          * restore of the math state immediately to avoid the trap; the
 667          * chances of needing FPU soon are obviously high now
 668          */
 669         if (next_p->fpu_counter>5)
 670                 math_state_restore();
 671         return prev_p;
 672 }
 673
 674 /*
 675  * sys_execve() executes a new program.
 676  */
 677 asmlinkage
 678 long sys_execve(char __user *name, char __user * __user *argv,
 679                 char __user * __user *envp, struct pt_regs regs)
 680 {
 681         long error;
 682         char * filename;
 683
 684         filename = getname(name);
 685         error = PTR_ERR(filename);
 686         if (IS_ERR(filename))
 687                 return error;
 688         error = do_execve(filename, argv, envp, &regs);
 689         if (error == 0) {
 690                 task_lock(current);
 691                 current->ptrace &= ~PT_DTRACE;
 692                 task_unlock(current);
 693         }
 694         putname(filename);
 695         return error;
 696 }
 697
 698 void set_personality_64bit(void)
 699 {
 700         /* inherit personality from parent */
 701
 702         /* Make sure to be in 64bit mode */
 703         clear_thread_flag(TIF_IA32);
 704
 705         /* TBD: overwrites user setup. Should have two bits.
 706            But 64bit processes have always behaved this way,
 707            so it's not too bad. The main problem is just that
 708            32bit childs are affected again. */
 709         current->personality &= ~READ_IMPLIES_EXEC;
 710 }
 711
 712 asmlinkage long sys_fork(struct pt_regs *regs)
 713 {
 714         return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
 715 }
 716
 717 asmlinkage long
 718 sys_clone(unsigned long clone_flags, unsigned long newsp,
 719           void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
 720 {
 721         if (!newsp)
 722                 newsp = regs->rsp;
 723         return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
 724 }
 725
 726 /*
 727  * This is trivial, and on the face of it looks like it
 728  * could equally well be done in user mode.
 729  *
 730  * Not so, for quite unobvious reasons - register pressure.
 731  * In user mode vfork() cannot have a stack frame, and if
 732  * done by calling the "clone()" system call directly, you
 733  * do not have enough call-clobbered registers to hold all
 734  * the information you need.
 735  */
 736 asmlinkage long sys_vfork(struct pt_regs *regs)
 737 {
 738         return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
 739                     NULL, NULL);
 740 }
 741
 742 unsigned long get_wchan(struct task_struct *p)
 743 {
 744         unsigned long stack;
 745         u64 fp,rip;
 746         int count = 0;
 747
 748         if (!p || p == current || p->state==TASK_RUNNING)
 749                 return 0;
 750         stack = (unsigned long)task_stack_page(p);
 751         if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
 752                 return 0;
 753         fp = *(u64 *)(p->thread.rsp);
 754         do {
 755                 if (fp < (unsigned long)stack ||
 756                     fp > (unsigned long)stack+THREAD_SIZE)
 757                         return 0;
 758                 rip = *(u64 *)(fp+8);
 759                 if (!in_sched_functions(rip))
 760                         return rip;
 761                 fp = *(u64 *)fp;
 762         } while (count++ < 16);
 763         return 0;
 764 }
 765
 766 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
 767 {
 768         int ret = 0;
 769         int doit = task == current;
 770         int cpu;
 771
 772         switch (code) {
 773         case ARCH_SET_GS:
 774                 if (addr >= TASK_SIZE_OF(task))
 775                         return -EPERM;
 776                 cpu = get_cpu();
 777                 /* handle small bases via the GDT because that's faster to
 778                    switch. */
 779                 if (addr <= 0xffffffff) {
 780                         set_32bit_tls(task, GS_TLS, addr);
 781                         if (doit) {
 782                                 load_TLS(&task->thread, cpu);
 783                                 load_gs_index(GS_TLS_SEL);
 784                         }
 785                         task->thread.gsindex = GS_TLS_SEL;
 786                         task->thread.gs = 0;
 787                 } else {
 788                         task->thread.gsindex = 0;
 789                         task->thread.gs = addr;
 790                         if (doit) {
 791                                 load_gs_index(0);
 792                                 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
 793                         }
 794                 }
 795                 put_cpu();
 796                 break;
 797         case ARCH_SET_FS:
 798                 /* Not strictly needed for fs, but do it for symmetry
 799                    with gs */
 800                 if (addr >= TASK_SIZE_OF(task))
 801                         return -EPERM;
 802                 cpu = get_cpu();
 803                 /* handle small bases via the GDT because that's faster to
 804                    switch. */
 805                 if (addr <= 0xffffffff) {
 806                         set_32bit_tls(task, FS_TLS, addr);
 807                         if (doit) {
 808                                 load_TLS(&task->thread, cpu);
 809                                 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
 810                         }
 811                         task->thread.fsindex = FS_TLS_SEL;
 812                         task->thread.fs = 0;
 813                 } else {
 814                         task->thread.fsindex = 0;
 815                         task->thread.fs = addr;
 816                         if (doit) {
 817                                 /* set the selector to 0 to not confuse
 818                                    __switch_to */
 819                                 asm volatile("movl %0,%%fs" :: "r" (0));
 820                                 ret = checking_wrmsrl(MSR_FS_BASE, addr);
 821                         }
 822                 }
 823                 put_cpu();
 824                 break;
 825         case ARCH_GET_FS: {
 826                 unsigned long base;
 827                 if (task->thread.fsindex == FS_TLS_SEL)
 828                         base = read_32bit_tls(task, FS_TLS);
 829                 else if (doit)
 830                         rdmsrl(MSR_FS_BASE, base);
 831                 else
 832                         base = task->thread.fs;
 833                 ret = put_user(base, (unsigned long __user *)addr);
 834                 break;
 835         }
 836         case ARCH_GET_GS: {
 837                 unsigned long base;
 838                 unsigned gsindex;
 839                 if (task->thread.gsindex == GS_TLS_SEL)
 840                         base = read_32bit_tls(task, GS_TLS);
 841                 else if (doit) {
 842                         asm("movl %%gs,%0" : "=r" (gsindex));
 843                         if (gsindex)
 844                                 rdmsrl(MSR_KERNEL_GS_BASE, base);
 845                         else
 846                                 base = task->thread.gs;
 847                 }
 848                 else
 849                         base = task->thread.gs;
 850                 ret = put_user(base, (unsigned long __user *)addr);
 851                 break;
 852         }
 853
 854         default:
 855                 ret = -EINVAL;
 856                 break;
 857         }
 858
 859         return ret;
 860 }
 861
 862 long sys_arch_prctl(int code, unsigned long addr)
 863 {
 864         return do_arch_prctl(current, code, addr);
 865 }
 866
 867 /*
 868  * Capture the user space registers if the task is not running (in user space)
 869  */
 870 int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
 871 {
 872         struct pt_regs *pp, ptregs;
 873
 874         pp = task_pt_regs(tsk);
 875
 876         ptregs = *pp;
 877         ptregs.cs &= 0xffff;
 878         ptregs.ss &= 0xffff;
 879
 880         elf_core_copy_regs(regs, &ptregs);
 881
 882         return 1;
 883 }
 884
 885 unsigned long arch_align_stack(unsigned long sp)
 886 {
 887         if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
 888                 sp -= get_random_int() % 8192;
 889         return sp & ~0xf;
 890 }