2 * linux/arch/x86-64/kernel/process.c
4 * Copyright (C) 1995 Linus Torvalds
6 * Pentium III FXSR, SSE support
7 * Gareth Hughes <gareth@valinux.com>, May 2000
12 * CPU hotplug support - ashok.raj@intel.com
13 * $Id: process.c,v 1.38 2002/01/15 10:08:03 ak Exp $
17 * This file handles the architecture-dependent parts of process handling..
22 #include <linux/cpu.h>
23 #include <linux/errno.h>
24 #include <linux/sched.h>
25 #include <linux/kernel.h>
27 #include <linux/elfcore.h>
28 #include <linux/smp.h>
29 #include <linux/slab.h>
30 #include <linux/user.h>
31 #include <linux/module.h>
32 #include <linux/a.out.h>
33 #include <linux/interrupt.h>
34 #include <linux/delay.h>
35 #include <linux/ptrace.h>
36 #include <linux/utsname.h>
37 #include <linux/random.h>
38 #include <linux/notifier.h>
39 #include <linux/kprobes.h>
41 #include <asm/uaccess.h>
42 #include <asm/pgtable.h>
43 #include <asm/system.h>
45 #include <asm/processor.h>
47 #include <asm/mmu_context.h>
49 #include <asm/prctl.h>
50 #include <asm/kdebug.h>
52 #include <asm/proto.h>
56 asmlinkage
extern void ret_from_fork(void);
58 unsigned long kernel_thread_flags
= CLONE_VM
| CLONE_UNTRACED
;
60 unsigned long boot_option_idle_override
= 0;
61 EXPORT_SYMBOL(boot_option_idle_override
);
64 * Powermanagement idle function, if any..
66 void (*pm_idle
)(void);
67 static DEFINE_PER_CPU(unsigned int, cpu_idle_state
);
69 static ATOMIC_NOTIFIER_HEAD(idle_notifier
);
71 void idle_notifier_register(struct notifier_block
*n
)
73 atomic_notifier_chain_register(&idle_notifier
, n
);
75 EXPORT_SYMBOL_GPL(idle_notifier_register
);
77 void idle_notifier_unregister(struct notifier_block
*n
)
79 atomic_notifier_chain_unregister(&idle_notifier
, n
);
81 EXPORT_SYMBOL(idle_notifier_unregister
);
83 enum idle_state
{ CPU_IDLE
, CPU_NOT_IDLE
};
84 static DEFINE_PER_CPU(enum idle_state
, idle_state
) = CPU_NOT_IDLE
;
88 __get_cpu_var(idle_state
) = CPU_IDLE
;
89 atomic_notifier_call_chain(&idle_notifier
, IDLE_START
, NULL
);
92 static void __exit_idle(void)
94 __get_cpu_var(idle_state
) = CPU_NOT_IDLE
;
95 atomic_notifier_call_chain(&idle_notifier
, IDLE_END
, NULL
);
98 /* Called from interrupts to signify idle end */
101 if (current
->pid
| read_pda(irqcount
))
107 * We use this if we don't have any better
110 static void default_idle(void)
114 clear_thread_flag(TIF_POLLING_NRFLAG
);
115 smp_mb__after_clear_bit();
116 while (!need_resched()) {
123 set_thread_flag(TIF_POLLING_NRFLAG
);
127 * On SMP it's slightly faster (but much more power-consuming!)
128 * to poll the ->need_resched flag instead of waiting for the
129 * cross-CPU IPI to arrive. Use this option with caution.
131 static void poll_idle (void)
141 "i" (_TIF_NEED_RESCHED
),
142 "m" (current_thread_info()->flags
));
145 void cpu_idle_wait(void)
147 unsigned int cpu
, this_cpu
= get_cpu();
150 set_cpus_allowed(current
, cpumask_of_cpu(this_cpu
));
154 for_each_online_cpu(cpu
) {
155 per_cpu(cpu_idle_state
, cpu
) = 1;
159 __get_cpu_var(cpu_idle_state
) = 0;
164 for_each_online_cpu(cpu
) {
165 if (cpu_isset(cpu
, map
) &&
166 !per_cpu(cpu_idle_state
, cpu
))
169 cpus_and(map
, map
, cpu_online_map
);
170 } while (!cpus_empty(map
));
172 EXPORT_SYMBOL_GPL(cpu_idle_wait
);
174 #ifdef CONFIG_HOTPLUG_CPU
175 DECLARE_PER_CPU(int, cpu_state
);
178 /* We halt the CPU with physical CPU hotplug */
179 static inline void play_dead(void)
185 __get_cpu_var(cpu_state
) = CPU_DEAD
;
192 static inline void play_dead(void)
196 #endif /* CONFIG_HOTPLUG_CPU */
199 * The idle thread. There's no useful work to be
200 * done, so just try to conserve power and have a
201 * low exit latency (ie sit in a loop waiting for
202 * somebody to say that they'd like to reschedule)
206 set_thread_flag(TIF_POLLING_NRFLAG
);
208 /* endless idle loop with no priority at all */
210 while (!need_resched()) {
213 if (__get_cpu_var(cpu_idle_state
))
214 __get_cpu_var(cpu_idle_state
) = 0;
220 if (cpu_is_offline(smp_processor_id()))
227 preempt_enable_no_resched();
234 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
235 * which can obviate IPI to trigger checking of need_resched.
236 * We execute MONITOR against need_resched and enter optimized wait state
237 * through MWAIT. Whenever someone changes need_resched, we would be woken
238 * up from MWAIT (without an IPI).
240 static void mwait_idle(void)
244 while (!need_resched()) {
245 __monitor((void *)¤t_thread_info()->flags
, 0, 0);
253 void __cpuinit
select_idle_routine(const struct cpuinfo_x86
*c
)
256 if (cpu_has(c
, X86_FEATURE_MWAIT
)) {
258 * Skip, if setup has overridden idle.
259 * One CPU supports mwait => All CPUs supports mwait
263 printk("using mwait in idle threads.\n");
266 pm_idle
= mwait_idle
;
271 static int __init
idle_setup (char *str
)
273 if (!strncmp(str
, "poll", 4)) {
274 printk("using polling idle threads.\n");
278 boot_option_idle_override
= 1;
282 __setup("idle=", idle_setup
);
284 /* Prints also some state that isn't saved in the pt_regs */
285 void __show_regs(struct pt_regs
* regs
)
287 unsigned long cr0
= 0L, cr2
= 0L, cr3
= 0L, cr4
= 0L, fs
, gs
, shadowgs
;
288 unsigned int fsindex
,gsindex
;
289 unsigned int ds
,cs
,es
;
293 printk("Pid: %d, comm: %.20s %s %s %.*s\n",
294 current
->pid
, current
->comm
, print_tainted(),
295 system_utsname
.release
,
296 (int)strcspn(system_utsname
.version
, " "),
297 system_utsname
.version
);
298 printk("RIP: %04lx:[<%016lx>] ", regs
->cs
& 0xffff, regs
->rip
);
299 printk_address(regs
->rip
);
300 printk("\nRSP: %04lx:%016lx EFLAGS: %08lx\n", regs
->ss
, regs
->rsp
,
302 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
303 regs
->rax
, regs
->rbx
, regs
->rcx
);
304 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
305 regs
->rdx
, regs
->rsi
, regs
->rdi
);
306 printk("RBP: %016lx R08: %016lx R09: %016lx\n",
307 regs
->rbp
, regs
->r8
, regs
->r9
);
308 printk("R10: %016lx R11: %016lx R12: %016lx\n",
309 regs
->r10
, regs
->r11
, regs
->r12
);
310 printk("R13: %016lx R14: %016lx R15: %016lx\n",
311 regs
->r13
, regs
->r14
, regs
->r15
);
313 asm("movl %%ds,%0" : "=r" (ds
));
314 asm("movl %%cs,%0" : "=r" (cs
));
315 asm("movl %%es,%0" : "=r" (es
));
316 asm("movl %%fs,%0" : "=r" (fsindex
));
317 asm("movl %%gs,%0" : "=r" (gsindex
));
319 rdmsrl(MSR_FS_BASE
, fs
);
320 rdmsrl(MSR_GS_BASE
, gs
);
321 rdmsrl(MSR_KERNEL_GS_BASE
, shadowgs
);
323 asm("movq %%cr0, %0": "=r" (cr0
));
324 asm("movq %%cr2, %0": "=r" (cr2
));
325 asm("movq %%cr3, %0": "=r" (cr3
));
326 asm("movq %%cr4, %0": "=r" (cr4
));
328 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
329 fs
,fsindex
,gs
,gsindex
,shadowgs
);
330 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs
, ds
, es
, cr0
);
331 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2
, cr3
, cr4
);
334 void show_regs(struct pt_regs
*regs
)
336 printk("CPU %d:", smp_processor_id());
338 show_trace(®s
->rsp
);
342 * Free current thread data structures etc..
344 void exit_thread(void)
346 struct task_struct
*me
= current
;
347 struct thread_struct
*t
= &me
->thread
;
349 if (me
->thread
.io_bitmap_ptr
) {
350 struct tss_struct
*tss
= &per_cpu(init_tss
, get_cpu());
352 kfree(t
->io_bitmap_ptr
);
353 t
->io_bitmap_ptr
= NULL
;
355 * Careful, clear this in the TSS too:
357 memset(tss
->io_bitmap
, 0xff, t
->io_bitmap_max
);
358 t
->io_bitmap_max
= 0;
363 void flush_thread(void)
365 struct task_struct
*tsk
= current
;
366 struct thread_info
*t
= current_thread_info();
368 if (t
->flags
& _TIF_ABI_PENDING
)
369 t
->flags
^= (_TIF_ABI_PENDING
| _TIF_IA32
);
371 tsk
->thread
.debugreg0
= 0;
372 tsk
->thread
.debugreg1
= 0;
373 tsk
->thread
.debugreg2
= 0;
374 tsk
->thread
.debugreg3
= 0;
375 tsk
->thread
.debugreg6
= 0;
376 tsk
->thread
.debugreg7
= 0;
377 memset(tsk
->thread
.tls_array
, 0, sizeof(tsk
->thread
.tls_array
));
379 * Forget coprocessor state..
385 void release_thread(struct task_struct
*dead_task
)
388 if (dead_task
->mm
->context
.size
) {
389 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
391 dead_task
->mm
->context
.ldt
,
392 dead_task
->mm
->context
.size
);
398 static inline void set_32bit_tls(struct task_struct
*t
, int tls
, u32 addr
)
400 struct user_desc ud
= {
407 struct n_desc_struct
*desc
= (void *)t
->thread
.tls_array
;
409 desc
->a
= LDT_entry_a(&ud
);
410 desc
->b
= LDT_entry_b(&ud
);
413 static inline u32
read_32bit_tls(struct task_struct
*t
, int tls
)
415 struct desc_struct
*desc
= (void *)t
->thread
.tls_array
;
418 (((u32
)desc
->base1
) << 16) |
419 (((u32
)desc
->base2
) << 24);
423 * This gets called before we allocate a new thread and copy
424 * the current task into it.
426 void prepare_to_copy(struct task_struct
*tsk
)
431 int copy_thread(int nr
, unsigned long clone_flags
, unsigned long rsp
,
432 unsigned long unused
,
433 struct task_struct
* p
, struct pt_regs
* regs
)
436 struct pt_regs
* childregs
;
437 struct task_struct
*me
= current
;
439 childregs
= ((struct pt_regs
*)
440 (THREAD_SIZE
+ task_stack_page(p
))) - 1;
444 childregs
->rsp
= rsp
;
446 childregs
->rsp
= (unsigned long)childregs
;
448 p
->thread
.rsp
= (unsigned long) childregs
;
449 p
->thread
.rsp0
= (unsigned long) (childregs
+1);
450 p
->thread
.userrsp
= me
->thread
.userrsp
;
452 set_tsk_thread_flag(p
, TIF_FORK
);
454 p
->thread
.fs
= me
->thread
.fs
;
455 p
->thread
.gs
= me
->thread
.gs
;
457 asm("mov %%gs,%0" : "=m" (p
->thread
.gsindex
));
458 asm("mov %%fs,%0" : "=m" (p
->thread
.fsindex
));
459 asm("mov %%es,%0" : "=m" (p
->thread
.es
));
460 asm("mov %%ds,%0" : "=m" (p
->thread
.ds
));
462 if (unlikely(me
->thread
.io_bitmap_ptr
!= NULL
)) {
463 p
->thread
.io_bitmap_ptr
= kmalloc(IO_BITMAP_BYTES
, GFP_KERNEL
);
464 if (!p
->thread
.io_bitmap_ptr
) {
465 p
->thread
.io_bitmap_max
= 0;
468 memcpy(p
->thread
.io_bitmap_ptr
, me
->thread
.io_bitmap_ptr
,
473 * Set a new TLS for the child thread?
475 if (clone_flags
& CLONE_SETTLS
) {
476 #ifdef CONFIG_IA32_EMULATION
477 if (test_thread_flag(TIF_IA32
))
478 err
= ia32_child_tls(p
, childregs
);
481 err
= do_arch_prctl(p
, ARCH_SET_FS
, childregs
->r8
);
487 if (err
&& p
->thread
.io_bitmap_ptr
) {
488 kfree(p
->thread
.io_bitmap_ptr
);
489 p
->thread
.io_bitmap_max
= 0;
495 * This special macro can be used to load a debugging register
497 #define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
500 * switch_to(x,y) should switch tasks from x to y.
502 * This could still be optimized:
503 * - fold all the options into a flag word and test it with a single test.
504 * - could test fs/gs bitsliced
506 * Kprobes not supported here. Set the probe on schedule instead.
508 __kprobes
struct task_struct
*
509 __switch_to(struct task_struct
*prev_p
, struct task_struct
*next_p
)
511 struct thread_struct
*prev
= &prev_p
->thread
,
512 *next
= &next_p
->thread
;
513 int cpu
= smp_processor_id();
514 struct tss_struct
*tss
= &per_cpu(init_tss
, cpu
);
517 * Reload esp0, LDT and the page table pointer:
519 tss
->rsp0
= next
->rsp0
;
523 * This won't pick up thread selector changes, but I guess that is ok.
525 asm volatile("mov %%es,%0" : "=m" (prev
->es
));
526 if (unlikely(next
->es
| prev
->es
))
527 loadsegment(es
, next
->es
);
529 asm volatile ("mov %%ds,%0" : "=m" (prev
->ds
));
530 if (unlikely(next
->ds
| prev
->ds
))
531 loadsegment(ds
, next
->ds
);
540 asm volatile("movl %%fs,%0" : "=r" (fsindex
));
541 /* segment register != 0 always requires a reload.
542 also reload when it has changed.
543 when prev process used 64bit base always reload
544 to avoid an information leak. */
545 if (unlikely(fsindex
| next
->fsindex
| prev
->fs
)) {
546 loadsegment(fs
, next
->fsindex
);
547 /* check if the user used a selector != 0
548 * if yes clear 64bit base, since overloaded base
549 * is always mapped to the Null selector
554 /* when next process has a 64bit base use it */
556 wrmsrl(MSR_FS_BASE
, next
->fs
);
557 prev
->fsindex
= fsindex
;
561 asm volatile("movl %%gs,%0" : "=r" (gsindex
));
562 if (unlikely(gsindex
| next
->gsindex
| prev
->gs
)) {
563 load_gs_index(next
->gsindex
);
568 wrmsrl(MSR_KERNEL_GS_BASE
, next
->gs
);
569 prev
->gsindex
= gsindex
;
573 * Switch the PDA and FPU contexts.
575 prev
->userrsp
= read_pda(oldrsp
);
576 write_pda(oldrsp
, next
->userrsp
);
577 write_pda(pcurrent
, next_p
);
579 /* This must be here to ensure both math_state_restore() and
580 kernel_fpu_begin() work consistently.
581 And the AMD workaround requires it to be after DS reload. */
583 write_pda(kernelstack
,
584 task_stack_page(next_p
) + THREAD_SIZE
- PDA_STACKOFFSET
);
587 * Now maybe reload the debug registers
589 if (unlikely(next
->debugreg7
)) {
601 * Handle the IO bitmap
603 if (unlikely(prev
->io_bitmap_ptr
|| next
->io_bitmap_ptr
)) {
604 if (next
->io_bitmap_ptr
)
606 * Copy the relevant range of the IO bitmap.
607 * Normally this is 128 bytes or less:
609 memcpy(tss
->io_bitmap
, next
->io_bitmap_ptr
,
610 max(prev
->io_bitmap_max
, next
->io_bitmap_max
));
613 * Clear any possible leftover bits:
615 memset(tss
->io_bitmap
, 0xff, prev
->io_bitmap_max
);
623 * sys_execve() executes a new program.
626 long sys_execve(char __user
*name
, char __user
* __user
*argv
,
627 char __user
* __user
*envp
, struct pt_regs regs
)
632 filename
= getname(name
);
633 error
= PTR_ERR(filename
);
634 if (IS_ERR(filename
))
636 error
= do_execve(filename
, argv
, envp
, ®s
);
639 current
->ptrace
&= ~PT_DTRACE
;
640 task_unlock(current
);
646 void set_personality_64bit(void)
648 /* inherit personality from parent */
650 /* Make sure to be in 64bit mode */
651 clear_thread_flag(TIF_IA32
);
653 /* TBD: overwrites user setup. Should have two bits.
654 But 64bit processes have always behaved this way,
655 so it's not too bad. The main problem is just that
656 32bit childs are affected again. */
657 current
->personality
&= ~READ_IMPLIES_EXEC
;
660 asmlinkage
long sys_fork(struct pt_regs
*regs
)
662 return do_fork(SIGCHLD
, regs
->rsp
, regs
, 0, NULL
, NULL
);
666 sys_clone(unsigned long clone_flags
, unsigned long newsp
,
667 void __user
*parent_tid
, void __user
*child_tid
, struct pt_regs
*regs
)
671 return do_fork(clone_flags
, newsp
, regs
, 0, parent_tid
, child_tid
);
675 * This is trivial, and on the face of it looks like it
676 * could equally well be done in user mode.
678 * Not so, for quite unobvious reasons - register pressure.
679 * In user mode vfork() cannot have a stack frame, and if
680 * done by calling the "clone()" system call directly, you
681 * do not have enough call-clobbered registers to hold all
682 * the information you need.
684 asmlinkage
long sys_vfork(struct pt_regs
*regs
)
686 return do_fork(CLONE_VFORK
| CLONE_VM
| SIGCHLD
, regs
->rsp
, regs
, 0,
690 unsigned long get_wchan(struct task_struct
*p
)
696 if (!p
|| p
== current
|| p
->state
==TASK_RUNNING
)
698 stack
= (unsigned long)task_stack_page(p
);
699 if (p
->thread
.rsp
< stack
|| p
->thread
.rsp
> stack
+THREAD_SIZE
)
701 fp
= *(u64
*)(p
->thread
.rsp
);
703 if (fp
< (unsigned long)stack
||
704 fp
> (unsigned long)stack
+THREAD_SIZE
)
706 rip
= *(u64
*)(fp
+8);
707 if (!in_sched_functions(rip
))
710 } while (count
++ < 16);
714 long do_arch_prctl(struct task_struct
*task
, int code
, unsigned long addr
)
717 int doit
= task
== current
;
722 if (addr
>= TASK_SIZE_OF(task
))
725 /* handle small bases via the GDT because that's faster to
727 if (addr
<= 0xffffffff) {
728 set_32bit_tls(task
, GS_TLS
, addr
);
730 load_TLS(&task
->thread
, cpu
);
731 load_gs_index(GS_TLS_SEL
);
733 task
->thread
.gsindex
= GS_TLS_SEL
;
736 task
->thread
.gsindex
= 0;
737 task
->thread
.gs
= addr
;
740 ret
= checking_wrmsrl(MSR_KERNEL_GS_BASE
, addr
);
746 /* Not strictly needed for fs, but do it for symmetry
748 if (addr
>= TASK_SIZE_OF(task
))
751 /* handle small bases via the GDT because that's faster to
753 if (addr
<= 0xffffffff) {
754 set_32bit_tls(task
, FS_TLS
, addr
);
756 load_TLS(&task
->thread
, cpu
);
757 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL
));
759 task
->thread
.fsindex
= FS_TLS_SEL
;
762 task
->thread
.fsindex
= 0;
763 task
->thread
.fs
= addr
;
765 /* set the selector to 0 to not confuse
767 asm volatile("movl %0,%%fs" :: "r" (0));
768 ret
= checking_wrmsrl(MSR_FS_BASE
, addr
);
775 if (task
->thread
.fsindex
== FS_TLS_SEL
)
776 base
= read_32bit_tls(task
, FS_TLS
);
778 rdmsrl(MSR_FS_BASE
, base
);
780 base
= task
->thread
.fs
;
781 ret
= put_user(base
, (unsigned long __user
*)addr
);
787 if (task
->thread
.gsindex
== GS_TLS_SEL
)
788 base
= read_32bit_tls(task
, GS_TLS
);
790 asm("movl %%gs,%0" : "=r" (gsindex
));
792 rdmsrl(MSR_KERNEL_GS_BASE
, base
);
794 base
= task
->thread
.gs
;
797 base
= task
->thread
.gs
;
798 ret
= put_user(base
, (unsigned long __user
*)addr
);
810 long sys_arch_prctl(int code
, unsigned long addr
)
812 return do_arch_prctl(current
, code
, addr
);
816 * Capture the user space registers if the task is not running (in user space)
818 int dump_task_regs(struct task_struct
*tsk
, elf_gregset_t
*regs
)
820 struct pt_regs
*pp
, ptregs
;
822 pp
= task_pt_regs(tsk
);
828 elf_core_copy_regs(regs
, &ptregs
);
833 unsigned long arch_align_stack(unsigned long sp
)
835 if (randomize_va_space
)
836 sp
-= get_random_int() % 8192;