2 * Copyright (C) 1995 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * Gareth Hughes <gareth@valinux.com>, May 2000
10 * CPU hotplug support - ashok.raj@intel.com
14 * This file handles the architecture-dependent parts of process handling..
17 #include <linux/cpu.h>
18 #include <linux/errno.h>
19 #include <linux/sched.h>
21 #include <linux/kernel.h>
23 #include <linux/elfcore.h>
24 #include <linux/smp.h>
25 #include <linux/slab.h>
26 #include <linux/user.h>
27 #include <linux/interrupt.h>
28 #include <linux/delay.h>
29 #include <linux/module.h>
30 #include <linux/ptrace.h>
31 #include <linux/notifier.h>
32 #include <linux/kprobes.h>
33 #include <linux/kdebug.h>
34 #include <linux/prctl.h>
35 #include <linux/uaccess.h>
37 #include <linux/ftrace.h>
39 #include <asm/pgtable.h>
40 #include <asm/processor.h>
42 #include <asm/fpu-internal.h>
43 #include <asm/mmu_context.h>
44 #include <asm/prctl.h>
46 #include <asm/proto.h>
49 #include <asm/syscalls.h>
50 #include <asm/debugreg.h>
51 #include <asm/switch_to.h>
53 asmlinkage
extern void ret_from_fork(void);
55 DEFINE_PER_CPU(unsigned long, old_rsp
);
57 /* Prints also some state that isn't saved in the pt_regs */
58 void __show_regs(struct pt_regs
*regs
, int all
)
60 unsigned long cr0
= 0L, cr2
= 0L, cr3
= 0L, cr4
= 0L, fs
, gs
, shadowgs
;
61 unsigned long d0
, d1
, d2
, d3
, d6
, d7
;
62 unsigned int fsindex
, gsindex
;
63 unsigned int ds
, cs
, es
;
66 printk(KERN_DEFAULT
"RIP: %04lx:[<%016lx>] ", regs
->cs
& 0xffff, regs
->ip
);
67 printk_address(regs
->ip
, 1);
68 printk(KERN_DEFAULT
"RSP: %04lx:%016lx EFLAGS: %08lx\n", regs
->ss
,
69 regs
->sp
, regs
->flags
);
70 printk(KERN_DEFAULT
"RAX: %016lx RBX: %016lx RCX: %016lx\n",
71 regs
->ax
, regs
->bx
, regs
->cx
);
72 printk(KERN_DEFAULT
"RDX: %016lx RSI: %016lx RDI: %016lx\n",
73 regs
->dx
, regs
->si
, regs
->di
);
74 printk(KERN_DEFAULT
"RBP: %016lx R08: %016lx R09: %016lx\n",
75 regs
->bp
, regs
->r8
, regs
->r9
);
76 printk(KERN_DEFAULT
"R10: %016lx R11: %016lx R12: %016lx\n",
77 regs
->r10
, regs
->r11
, regs
->r12
);
78 printk(KERN_DEFAULT
"R13: %016lx R14: %016lx R15: %016lx\n",
79 regs
->r13
, regs
->r14
, regs
->r15
);
81 asm("movl %%ds,%0" : "=r" (ds
));
82 asm("movl %%cs,%0" : "=r" (cs
));
83 asm("movl %%es,%0" : "=r" (es
));
84 asm("movl %%fs,%0" : "=r" (fsindex
));
85 asm("movl %%gs,%0" : "=r" (gsindex
));
87 rdmsrl(MSR_FS_BASE
, fs
);
88 rdmsrl(MSR_GS_BASE
, gs
);
89 rdmsrl(MSR_KERNEL_GS_BASE
, shadowgs
);
99 printk(KERN_DEFAULT
"FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
100 fs
, fsindex
, gs
, gsindex
, shadowgs
);
101 printk(KERN_DEFAULT
"CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs
, ds
,
103 printk(KERN_DEFAULT
"CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2
, cr3
,
109 printk(KERN_DEFAULT
"DR0: %016lx DR1: %016lx DR2: %016lx\n", d0
, d1
, d2
);
113 printk(KERN_DEFAULT
"DR3: %016lx DR6: %016lx DR7: %016lx\n", d3
, d6
, d7
);
116 void release_thread(struct task_struct
*dead_task
)
119 if (dead_task
->mm
->context
.size
) {
120 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
122 dead_task
->mm
->context
.ldt
,
123 dead_task
->mm
->context
.size
);
129 static inline void set_32bit_tls(struct task_struct
*t
, int tls
, u32 addr
)
131 struct user_desc ud
= {
138 struct desc_struct
*desc
= t
->thread
.tls_array
;
143 static inline u32
read_32bit_tls(struct task_struct
*t
, int tls
)
145 return get_desc_base(&t
->thread
.tls_array
[tls
]);
148 int copy_thread(unsigned long clone_flags
, unsigned long sp
,
149 unsigned long unused
,
150 struct task_struct
*p
, struct pt_regs
*regs
)
153 struct pt_regs
*childregs
;
154 struct task_struct
*me
= current
;
156 childregs
= ((struct pt_regs
*)
157 (THREAD_SIZE
+ task_stack_page(p
))) - 1;
164 childregs
->sp
= (unsigned long)childregs
;
166 p
->thread
.sp
= (unsigned long) childregs
;
167 p
->thread
.sp0
= (unsigned long) (childregs
+1);
168 p
->thread
.usersp
= me
->thread
.usersp
;
170 set_tsk_thread_flag(p
, TIF_FORK
);
173 p
->thread
.io_bitmap_ptr
= NULL
;
175 savesegment(gs
, p
->thread
.gsindex
);
176 p
->thread
.gs
= p
->thread
.gsindex
? 0 : me
->thread
.gs
;
177 savesegment(fs
, p
->thread
.fsindex
);
178 p
->thread
.fs
= p
->thread
.fsindex
? 0 : me
->thread
.fs
;
179 savesegment(es
, p
->thread
.es
);
180 savesegment(ds
, p
->thread
.ds
);
183 memset(p
->thread
.ptrace_bps
, 0, sizeof(p
->thread
.ptrace_bps
));
185 if (unlikely(test_tsk_thread_flag(me
, TIF_IO_BITMAP
))) {
186 p
->thread
.io_bitmap_ptr
= kmemdup(me
->thread
.io_bitmap_ptr
,
187 IO_BITMAP_BYTES
, GFP_KERNEL
);
188 if (!p
->thread
.io_bitmap_ptr
) {
189 p
->thread
.io_bitmap_max
= 0;
192 set_tsk_thread_flag(p
, TIF_IO_BITMAP
);
196 * Set a new TLS for the child thread?
198 if (clone_flags
& CLONE_SETTLS
) {
199 #ifdef CONFIG_IA32_EMULATION
200 if (test_thread_flag(TIF_IA32
))
201 err
= do_set_thread_area(p
, -1,
202 (struct user_desc __user
*)childregs
->si
, 0);
205 err
= do_arch_prctl(p
, ARCH_SET_FS
, childregs
->r8
);
211 if (err
&& p
->thread
.io_bitmap_ptr
) {
212 kfree(p
->thread
.io_bitmap_ptr
);
213 p
->thread
.io_bitmap_max
= 0;
220 start_thread_common(struct pt_regs
*regs
, unsigned long new_ip
,
221 unsigned long new_sp
,
222 unsigned int _cs
, unsigned int _ss
, unsigned int _ds
)
225 loadsegment(es
, _ds
);
226 loadsegment(ds
, _ds
);
228 current
->thread
.usersp
= new_sp
;
231 this_cpu_write(old_rsp
, new_sp
);
234 regs
->flags
= X86_EFLAGS_IF
;
236 * Free the old FP and other extended state
238 free_thread_xstate(current
);
242 start_thread(struct pt_regs
*regs
, unsigned long new_ip
, unsigned long new_sp
)
244 start_thread_common(regs
, new_ip
, new_sp
,
245 __USER_CS
, __USER_DS
, 0);
248 #ifdef CONFIG_IA32_EMULATION
249 void start_thread_ia32(struct pt_regs
*regs
, u32 new_ip
, u32 new_sp
)
251 start_thread_common(regs
, new_ip
, new_sp
,
252 test_thread_flag(TIF_X32
)
253 ? __USER_CS
: __USER32_CS
,
254 __USER_DS
, __USER_DS
);
259 * switch_to(x,y) should switch tasks from x to y.
261 * This could still be optimized:
262 * - fold all the options into a flag word and test it with a single test.
263 * - could test fs/gs bitsliced
265 * Kprobes not supported here. Set the probe on schedule instead.
266 * Function graph tracer not supported too.
268 __notrace_funcgraph
struct task_struct
*
269 __switch_to(struct task_struct
*prev_p
, struct task_struct
*next_p
)
271 struct thread_struct
*prev
= &prev_p
->thread
;
272 struct thread_struct
*next
= &next_p
->thread
;
273 int cpu
= smp_processor_id();
274 struct tss_struct
*tss
= &per_cpu(init_tss
, cpu
);
275 unsigned fsindex
, gsindex
;
278 fpu
= switch_fpu_prepare(prev_p
, next_p
, cpu
);
281 * Reload esp0, LDT and the page table pointer:
287 * This won't pick up thread selector changes, but I guess that is ok.
289 savesegment(es
, prev
->es
);
290 if (unlikely(next
->es
| prev
->es
))
291 loadsegment(es
, next
->es
);
293 savesegment(ds
, prev
->ds
);
294 if (unlikely(next
->ds
| prev
->ds
))
295 loadsegment(ds
, next
->ds
);
298 /* We must save %fs and %gs before load_TLS() because
299 * %fs and %gs may be cleared by load_TLS().
301 * (e.g. xen_load_tls())
303 savesegment(fs
, fsindex
);
304 savesegment(gs
, gsindex
);
309 * Leave lazy mode, flushing any hypercalls made here.
310 * This must be done before restoring TLS segments so
311 * the GDT and LDT are properly updated, and must be
312 * done before math_state_restore, so the TS bit is up
315 arch_end_context_switch(next_p
);
320 * Segment register != 0 always requires a reload. Also
321 * reload when it has changed. When prev process used 64bit
322 * base always reload to avoid an information leak.
324 if (unlikely(fsindex
| next
->fsindex
| prev
->fs
)) {
325 loadsegment(fs
, next
->fsindex
);
327 * Check if the user used a selector != 0; if yes
328 * clear 64bit base, since overloaded base is always
329 * mapped to the Null selector
334 /* when next process has a 64bit base use it */
336 wrmsrl(MSR_FS_BASE
, next
->fs
);
337 prev
->fsindex
= fsindex
;
339 if (unlikely(gsindex
| next
->gsindex
| prev
->gs
)) {
340 load_gs_index(next
->gsindex
);
345 wrmsrl(MSR_KERNEL_GS_BASE
, next
->gs
);
346 prev
->gsindex
= gsindex
;
348 switch_fpu_finish(next_p
, fpu
);
351 * Switch the PDA and FPU contexts.
353 prev
->usersp
= this_cpu_read(old_rsp
);
354 this_cpu_write(old_rsp
, next
->usersp
);
355 this_cpu_write(current_task
, next_p
);
357 this_cpu_write(kernel_stack
,
358 (unsigned long)task_stack_page(next_p
) +
359 THREAD_SIZE
- KERNEL_STACK_OFFSET
);
362 * Now maybe reload the debug registers and handle I/O bitmaps
364 if (unlikely(task_thread_info(next_p
)->flags
& _TIF_WORK_CTXSW_NEXT
||
365 task_thread_info(prev_p
)->flags
& _TIF_WORK_CTXSW_PREV
))
366 __switch_to_xtra(prev_p
, next_p
, tss
);
371 void set_personality_64bit(void)
373 /* inherit personality from parent */
375 /* Make sure to be in 64bit mode */
376 clear_thread_flag(TIF_IA32
);
377 clear_thread_flag(TIF_ADDR32
);
378 clear_thread_flag(TIF_X32
);
380 /* Ensure the corresponding mm is not marked. */
382 current
->mm
->context
.ia32_compat
= 0;
384 /* TBD: overwrites user setup. Should have two bits.
385 But 64bit processes have always behaved this way,
386 so it's not too bad. The main problem is just that
387 32bit childs are affected again. */
388 current
->personality
&= ~READ_IMPLIES_EXEC
;
391 void set_personality_ia32(bool x32
)
393 /* inherit personality from parent */
395 /* Make sure to be in 32bit mode */
396 set_thread_flag(TIF_ADDR32
);
398 /* Mark the associated mm as containing 32-bit tasks. */
400 current
->mm
->context
.ia32_compat
= 1;
403 clear_thread_flag(TIF_IA32
);
404 set_thread_flag(TIF_X32
);
405 current
->personality
&= ~READ_IMPLIES_EXEC
;
406 /* is_compat_task() uses the presence of the x32
407 syscall bit flag to determine compat status */
408 current_thread_info()->status
&= ~TS_COMPAT
;
410 set_thread_flag(TIF_IA32
);
411 clear_thread_flag(TIF_X32
);
412 current
->personality
|= force_personality32
;
413 /* Prepare the first "return" to user space */
414 current_thread_info()->status
|= TS_COMPAT
;
417 EXPORT_SYMBOL_GPL(set_personality_ia32
);
419 unsigned long get_wchan(struct task_struct
*p
)
425 if (!p
|| p
== current
|| p
->state
== TASK_RUNNING
)
427 stack
= (unsigned long)task_stack_page(p
);
428 if (p
->thread
.sp
< stack
|| p
->thread
.sp
>= stack
+THREAD_SIZE
)
430 fp
= *(u64
*)(p
->thread
.sp
);
432 if (fp
< (unsigned long)stack
||
433 fp
>= (unsigned long)stack
+THREAD_SIZE
)
436 if (!in_sched_functions(ip
))
439 } while (count
++ < 16);
443 long do_arch_prctl(struct task_struct
*task
, int code
, unsigned long addr
)
446 int doit
= task
== current
;
451 if (addr
>= TASK_SIZE_OF(task
))
454 /* handle small bases via the GDT because that's faster to
456 if (addr
<= 0xffffffff) {
457 set_32bit_tls(task
, GS_TLS
, addr
);
459 load_TLS(&task
->thread
, cpu
);
460 load_gs_index(GS_TLS_SEL
);
462 task
->thread
.gsindex
= GS_TLS_SEL
;
465 task
->thread
.gsindex
= 0;
466 task
->thread
.gs
= addr
;
469 ret
= checking_wrmsrl(MSR_KERNEL_GS_BASE
, addr
);
475 /* Not strictly needed for fs, but do it for symmetry
477 if (addr
>= TASK_SIZE_OF(task
))
480 /* handle small bases via the GDT because that's faster to
482 if (addr
<= 0xffffffff) {
483 set_32bit_tls(task
, FS_TLS
, addr
);
485 load_TLS(&task
->thread
, cpu
);
486 loadsegment(fs
, FS_TLS_SEL
);
488 task
->thread
.fsindex
= FS_TLS_SEL
;
491 task
->thread
.fsindex
= 0;
492 task
->thread
.fs
= addr
;
494 /* set the selector to 0 to not confuse
497 ret
= checking_wrmsrl(MSR_FS_BASE
, addr
);
504 if (task
->thread
.fsindex
== FS_TLS_SEL
)
505 base
= read_32bit_tls(task
, FS_TLS
);
507 rdmsrl(MSR_FS_BASE
, base
);
509 base
= task
->thread
.fs
;
510 ret
= put_user(base
, (unsigned long __user
*)addr
);
516 if (task
->thread
.gsindex
== GS_TLS_SEL
)
517 base
= read_32bit_tls(task
, GS_TLS
);
519 savesegment(gs
, gsindex
);
521 rdmsrl(MSR_KERNEL_GS_BASE
, base
);
523 base
= task
->thread
.gs
;
525 base
= task
->thread
.gs
;
526 ret
= put_user(base
, (unsigned long __user
*)addr
);
538 long sys_arch_prctl(int code
, unsigned long addr
)
540 return do_arch_prctl(current
, code
, addr
);
543 unsigned long KSTK_ESP(struct task_struct
*task
)
545 return (test_tsk_thread_flag(task
, TIF_IA32
)) ?
546 (task_pt_regs(task
)->sp
) : ((task
)->thread
.usersp
);