2 * Copyright (C) 1995 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * Gareth Hughes <gareth@valinux.com>, May 2000
9 * This file handles the architecture-dependent parts of process handling..
12 #include <linux/stackprotector.h>
13 #include <linux/cpu.h>
14 #include <linux/errno.h>
15 #include <linux/sched.h>
17 #include <linux/kernel.h>
19 #include <linux/elfcore.h>
20 #include <linux/smp.h>
21 #include <linux/stddef.h>
22 #include <linux/slab.h>
23 #include <linux/vmalloc.h>
24 #include <linux/user.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <linux/reboot.h>
28 #include <linux/init.h>
29 #include <linux/mc146818rtc.h>
30 #include <linux/module.h>
31 #include <linux/kallsyms.h>
32 #include <linux/ptrace.h>
33 #include <linux/personality.h>
34 #include <linux/tick.h>
35 #include <linux/percpu.h>
36 #include <linux/prctl.h>
37 #include <linux/ftrace.h>
38 #include <linux/uaccess.h>
40 #include <linux/kdebug.h>
42 #include <asm/pgtable.h>
43 #include <asm/system.h>
45 #include <asm/processor.h>
48 #ifdef CONFIG_MATH_EMULATION
49 #include <asm/math_emu.h>
52 #include <linux/err.h>
54 #include <asm/tlbflush.h>
57 #include <asm/syscalls.h>
59 #include <asm/debugreg.h>
61 asmlinkage
void ret_from_fork(void) __asm__("ret_from_fork");
64 * Return saved PC of a blocked thread.
66 unsigned long thread_saved_pc(struct task_struct
*tsk
)
68 return ((unsigned long *)tsk
->thread
.sp
)[3];
72 static inline void play_dead(void)
79 * The idle thread. There's no useful work to be
80 * done, so just try to conserve power and have a
81 * low exit latency (ie sit in a loop waiting for
82 * somebody to say that they'd like to reschedule)
86 int cpu
= smp_processor_id();
89 * If we're the non-boot CPU, nothing set the stack canary up
90 * for us. CPU0 already has it initialized but no harm in
91 * doing it again. This is a good place for updating it, as
92 * we wont ever return from this function (so the invalid
93 * canaries already on the stack wont ever trigger).
95 boot_init_stack_canary();
97 current_thread_info()->status
|= TS_POLLING
;
99 /* endless idle loop with no priority at all */
101 tick_nohz_stop_sched_tick(1);
102 while (!need_resched()) {
107 if (cpu_is_offline(cpu
))
111 /* Don't trace irqs off for idle */
112 stop_critical_timings();
114 start_critical_timings();
116 tick_nohz_restart_sched_tick();
117 preempt_enable_no_resched();
123 void __show_regs(struct pt_regs
*regs
, int all
)
125 unsigned long cr0
= 0L, cr2
= 0L, cr3
= 0L, cr4
= 0L;
126 unsigned long d0
, d1
, d2
, d3
, d6
, d7
;
128 unsigned short ss
, gs
;
130 if (user_mode_vm(regs
)) {
132 ss
= regs
->ss
& 0xffff;
133 gs
= get_user_gs(regs
);
135 sp
= kernel_stack_pointer(regs
);
142 printk("EIP: %04x:[<%08lx>] EFLAGS: %08lx CPU: %d\n",
143 (u16
)regs
->cs
, regs
->ip
, regs
->flags
,
145 print_symbol("EIP is at %s\n", regs
->ip
);
147 printk("EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
148 regs
->ax
, regs
->bx
, regs
->cx
, regs
->dx
);
149 printk("ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
150 regs
->si
, regs
->di
, regs
->bp
, sp
);
151 printk(" DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x\n",
152 (u16
)regs
->ds
, (u16
)regs
->es
, (u16
)regs
->fs
, gs
, ss
);
160 cr4
= read_cr4_safe();
161 printk("CR0: %08lx CR2: %08lx CR3: %08lx CR4: %08lx\n",
168 printk("DR0: %08lx DR1: %08lx DR2: %08lx DR3: %08lx\n",
173 printk("DR6: %08lx DR7: %08lx\n",
177 void release_thread(struct task_struct
*dead_task
)
179 BUG_ON(dead_task
->mm
);
180 release_vm86_irqs(dead_task
);
184 * This gets called before we allocate a new thread and copy
185 * the current task into it.
187 void prepare_to_copy(struct task_struct
*tsk
)
192 int copy_thread(unsigned long clone_flags
, unsigned long sp
,
193 unsigned long unused
,
194 struct task_struct
*p
, struct pt_regs
*regs
)
196 struct pt_regs
*childregs
;
197 struct task_struct
*tsk
;
200 childregs
= task_pt_regs(p
);
205 p
->thread
.sp
= (unsigned long) childregs
;
206 p
->thread
.sp0
= (unsigned long) (childregs
+1);
208 p
->thread
.ip
= (unsigned long) ret_from_fork
;
210 task_user_gs(p
) = get_user_gs(regs
);
212 p
->thread
.io_bitmap_ptr
= NULL
;
216 memset(p
->thread
.ptrace_bps
, 0, sizeof(p
->thread
.ptrace_bps
));
218 if (unlikely(test_tsk_thread_flag(tsk
, TIF_IO_BITMAP
))) {
219 p
->thread
.io_bitmap_ptr
= kmemdup(tsk
->thread
.io_bitmap_ptr
,
220 IO_BITMAP_BYTES
, GFP_KERNEL
);
221 if (!p
->thread
.io_bitmap_ptr
) {
222 p
->thread
.io_bitmap_max
= 0;
225 set_tsk_thread_flag(p
, TIF_IO_BITMAP
);
231 * Set a new TLS for the child thread?
233 if (clone_flags
& CLONE_SETTLS
)
234 err
= do_set_thread_area(p
, -1,
235 (struct user_desc __user
*)childregs
->si
, 0);
237 if (err
&& p
->thread
.io_bitmap_ptr
) {
238 kfree(p
->thread
.io_bitmap_ptr
);
239 p
->thread
.io_bitmap_max
= 0;
242 clear_tsk_thread_flag(p
, TIF_DS_AREA_MSR
);
243 p
->thread
.ds_ctx
= NULL
;
245 clear_tsk_thread_flag(p
, TIF_DEBUGCTLMSR
);
246 p
->thread
.debugctlmsr
= 0;
252 start_thread(struct pt_regs
*regs
, unsigned long new_ip
, unsigned long new_sp
)
254 set_user_gs(regs
, 0);
257 regs
->ds
= __USER_DS
;
258 regs
->es
= __USER_DS
;
259 regs
->ss
= __USER_DS
;
260 regs
->cs
= __USER_CS
;
264 * Free the old FP and other extended state
266 free_thread_xstate(current
);
268 EXPORT_SYMBOL_GPL(start_thread
);
272 * switch_to(x,yn) should switch tasks from x to y.
274 * We fsave/fwait so that an exception goes off at the right time
275 * (as a call from the fsave or fwait in effect) rather than to
276 * the wrong process. Lazy FP saving no longer makes any sense
277 * with modern CPU's, and this simplifies a lot of things (SMP
278 * and UP become the same).
280 * NOTE! We used to use the x86 hardware context switching. The
281 * reason for not using it any more becomes apparent when you
282 * try to recover gracefully from saved state that is no longer
283 * valid (stale segment register values in particular). With the
284 * hardware task-switch, there is no way to fix up bad state in
285 * a reasonable manner.
287 * The fact that Intel documents the hardware task-switching to
288 * be slow is a fairly red herring - this code is not noticeably
289 * faster. However, there _is_ some room for improvement here,
290 * so the performance issues may eventually be a valid point.
291 * More important, however, is the fact that this allows us much
294 * The return value (in %ax) will be the "prev" task after
295 * the task-switch, and shows up in ret_from_fork in entry.S,
298 __notrace_funcgraph
struct task_struct
*
299 __switch_to(struct task_struct
*prev_p
, struct task_struct
*next_p
)
301 struct thread_struct
*prev
= &prev_p
->thread
,
302 *next
= &next_p
->thread
;
303 int cpu
= smp_processor_id();
304 struct tss_struct
*tss
= &per_cpu(init_tss
, cpu
);
307 /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
310 * If the task has used fpu the last 5 timeslices, just do a full
311 * restore of the math state immediately to avoid the trap; the
312 * chances of needing FPU soon are obviously high now
314 preload_fpu
= tsk_used_math(next_p
) && next_p
->fpu_counter
> 5;
316 __unlazy_fpu(prev_p
);
318 /* we're going to use this soon, after a few expensive things */
320 prefetch(next
->xstate
);
328 * Save away %gs. No need to save %fs, as it was saved on the
329 * stack on entry. No need to save %es and %ds, as those are
330 * always kernel segments while inside the kernel. Doing this
331 * before setting the new TLS descriptors avoids the situation
332 * where we temporarily have non-reloadable segments in %fs
333 * and %gs. This could be an issue if the NMI handler ever
334 * used %fs or %gs (it does not today), or if the kernel is
335 * running inside of a hypervisor layer.
337 lazy_save_gs(prev
->gs
);
340 * Load the per-thread Thread-Local Storage descriptor.
345 * Restore IOPL if needed. In normal use, the flags restore
346 * in the switch assembly will handle this. But if the kernel
347 * is running virtualized at a non-zero CPL, the popf will
348 * not restore flags, so it must be done in a separate step.
350 if (get_kernel_rpl() && unlikely(prev
->iopl
!= next
->iopl
))
351 set_iopl_mask(next
->iopl
);
354 * Now maybe handle debug registers and/or IO bitmaps
356 if (unlikely(task_thread_info(prev_p
)->flags
& _TIF_WORK_CTXSW_PREV
||
357 task_thread_info(next_p
)->flags
& _TIF_WORK_CTXSW_NEXT
))
358 __switch_to_xtra(prev_p
, next_p
, tss
);
360 /* If we're going to preload the fpu context, make sure clts
361 is run while we're batching the cpu state updates. */
366 * Leave lazy mode, flushing any hypercalls made here.
367 * This must be done before restoring TLS segments so
368 * the GDT and LDT are properly updated, and must be
369 * done before math_state_restore, so the TS bit is up
372 arch_end_context_switch(next_p
);
375 __math_state_restore();
378 * Restore %gs if needed (which is common)
380 if (prev
->gs
| next
->gs
)
381 lazy_load_gs(next
->gs
);
383 percpu_write(current_task
, next_p
);
388 #define top_esp (THREAD_SIZE - sizeof(unsigned long))
389 #define top_ebp (THREAD_SIZE - 2*sizeof(unsigned long))
391 unsigned long get_wchan(struct task_struct
*p
)
393 unsigned long bp
, sp
, ip
;
394 unsigned long stack_page
;
396 if (!p
|| p
== current
|| p
->state
== TASK_RUNNING
)
398 stack_page
= (unsigned long)task_stack_page(p
);
400 if (!stack_page
|| sp
< stack_page
|| sp
> top_esp
+stack_page
)
402 /* include/asm-i386/system.h:switch_to() pushes bp last. */
403 bp
= *(unsigned long *) sp
;
405 if (bp
< stack_page
|| bp
> top_ebp
+stack_page
)
407 ip
= *(unsigned long *) (bp
+4);
408 if (!in_sched_functions(ip
))
410 bp
= *(unsigned long *) bp
;
411 } while (count
++ < 16);