2 * Copyright (C) 1995 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * Gareth Hughes <gareth@valinux.com>, May 2000
10 * CPU hotplug support - ashok.raj@intel.com
14 * This file handles the architecture-dependent parts of process handling..
19 #include <linux/cpu.h>
20 #include <linux/errno.h>
21 #include <linux/sched.h>
23 #include <linux/kernel.h>
25 #include <linux/elfcore.h>
26 #include <linux/smp.h>
27 #include <linux/slab.h>
28 #include <linux/user.h>
29 #include <linux/interrupt.h>
30 #include <linux/utsname.h>
31 #include <linux/delay.h>
32 #include <linux/module.h>
33 #include <linux/ptrace.h>
34 #include <linux/random.h>
35 #include <linux/notifier.h>
36 #include <linux/kprobes.h>
37 #include <linux/kdebug.h>
38 #include <linux/tick.h>
39 #include <linux/prctl.h>
40 #include <linux/uaccess.h>
43 #include <asm/pgtable.h>
44 #include <asm/system.h>
45 #include <asm/processor.h>
47 #include <asm/mmu_context.h>
49 #include <asm/prctl.h>
51 #include <asm/proto.h>
54 #include <asm/syscalls.h>
56 asmlinkage
extern void ret_from_fork(void);
58 unsigned long kernel_thread_flags
= CLONE_VM
| CLONE_UNTRACED
;
60 static ATOMIC_NOTIFIER_HEAD(idle_notifier
);
62 void idle_notifier_register(struct notifier_block
*n
)
64 atomic_notifier_chain_register(&idle_notifier
, n
);
70 atomic_notifier_call_chain(&idle_notifier
, IDLE_START
, NULL
);
73 static void __exit_idle(void)
75 if (test_and_clear_bit_pda(0, isidle
) == 0)
77 atomic_notifier_call_chain(&idle_notifier
, IDLE_END
, NULL
);
80 /* Called from interrupts to signify idle end */
83 /* idle loop has pid 0 */
89 #ifdef CONFIG_HOTPLUG_CPU
90 DECLARE_PER_CPU(int, cpu_state
);
92 #include <linux/nmi.h>
93 /* We halt the CPU with physical CPU hotplug */
94 static inline void play_dead(void)
97 c1e_remove_cpu(raw_smp_processor_id());
101 __get_cpu_var(cpu_state
) = CPU_DEAD
;
104 /* mask all interrupts, flush any and all caches, and halt */
108 static inline void play_dead(void)
112 #endif /* CONFIG_HOTPLUG_CPU */
115 * The idle thread. There's no useful work to be
116 * done, so just try to conserve power and have a
117 * low exit latency (ie sit in a loop waiting for
118 * somebody to say that they'd like to reschedule)
122 current_thread_info()->status
|= TS_POLLING
;
123 /* endless idle loop with no priority at all */
125 tick_nohz_stop_sched_tick(1);
126 while (!need_resched()) {
130 if (cpu_is_offline(smp_processor_id()))
133 * Idle routines should keep interrupts disabled
134 * from here on, until they go to idle.
135 * Otherwise, idle callbacks can misfire.
139 /* Don't trace irqs off for idle */
140 stop_critical_timings();
142 start_critical_timings();
143 /* In many cases the interrupt that ended idle
144 has already called exit_idle. But some idle
145 loops can be woken up without interrupt. */
149 tick_nohz_restart_sched_tick();
150 preempt_enable_no_resched();
156 /* Prints also some state that isn't saved in the pt_regs */
157 void __show_regs(struct pt_regs
*regs
)
159 unsigned long cr0
= 0L, cr2
= 0L, cr3
= 0L, cr4
= 0L, fs
, gs
, shadowgs
;
160 unsigned long d0
, d1
, d2
, d3
, d6
, d7
;
161 unsigned int fsindex
, gsindex
;
162 unsigned int ds
, cs
, es
;
166 printk(KERN_INFO
"Pid: %d, comm: %.20s %s %s %.*s\n",
167 current
->pid
, current
->comm
, print_tainted(),
168 init_utsname()->release
,
169 (int)strcspn(init_utsname()->version
, " "),
170 init_utsname()->version
);
171 printk(KERN_INFO
"RIP: %04lx:[<%016lx>] ", regs
->cs
& 0xffff, regs
->ip
);
172 printk_address(regs
->ip
, 1);
173 printk(KERN_INFO
"RSP: %04lx:%016lx EFLAGS: %08lx\n", regs
->ss
,
174 regs
->sp
, regs
->flags
);
175 printk(KERN_INFO
"RAX: %016lx RBX: %016lx RCX: %016lx\n",
176 regs
->ax
, regs
->bx
, regs
->cx
);
177 printk(KERN_INFO
"RDX: %016lx RSI: %016lx RDI: %016lx\n",
178 regs
->dx
, regs
->si
, regs
->di
);
179 printk(KERN_INFO
"RBP: %016lx R08: %016lx R09: %016lx\n",
180 regs
->bp
, regs
->r8
, regs
->r9
);
181 printk(KERN_INFO
"R10: %016lx R11: %016lx R12: %016lx\n",
182 regs
->r10
, regs
->r11
, regs
->r12
);
183 printk(KERN_INFO
"R13: %016lx R14: %016lx R15: %016lx\n",
184 regs
->r13
, regs
->r14
, regs
->r15
);
186 asm("movl %%ds,%0" : "=r" (ds
));
187 asm("movl %%cs,%0" : "=r" (cs
));
188 asm("movl %%es,%0" : "=r" (es
));
189 asm("movl %%fs,%0" : "=r" (fsindex
));
190 asm("movl %%gs,%0" : "=r" (gsindex
));
192 rdmsrl(MSR_FS_BASE
, fs
);
193 rdmsrl(MSR_GS_BASE
, gs
);
194 rdmsrl(MSR_KERNEL_GS_BASE
, shadowgs
);
201 printk(KERN_INFO
"FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
202 fs
, fsindex
, gs
, gsindex
, shadowgs
);
203 printk(KERN_INFO
"CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs
, ds
,
205 printk(KERN_INFO
"CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2
, cr3
,
211 printk(KERN_INFO
"DR0: %016lx DR1: %016lx DR2: %016lx\n", d0
, d1
, d2
);
215 printk(KERN_INFO
"DR3: %016lx DR6: %016lx DR7: %016lx\n", d3
, d6
, d7
);
218 void show_regs(struct pt_regs
*regs
)
220 printk(KERN_INFO
"CPU %d:", smp_processor_id());
222 show_trace(NULL
, regs
, (void *)(regs
+ 1), regs
->bp
);
226 * Free current thread data structures etc..
228 void exit_thread(void)
230 struct task_struct
*me
= current
;
231 struct thread_struct
*t
= &me
->thread
;
233 if (me
->thread
.io_bitmap_ptr
) {
234 struct tss_struct
*tss
= &per_cpu(init_tss
, get_cpu());
236 kfree(t
->io_bitmap_ptr
);
237 t
->io_bitmap_ptr
= NULL
;
238 clear_thread_flag(TIF_IO_BITMAP
);
240 * Careful, clear this in the TSS too:
242 memset(tss
->io_bitmap
, 0xff, t
->io_bitmap_max
);
243 t
->io_bitmap_max
= 0;
247 /* Free any DS contexts that have not been properly released. */
248 if (unlikely(t
->ds_ctx
)) {
249 /* we clear debugctl to make sure DS is not used. */
250 update_debugctlmsr(0);
253 #endif /* CONFIG_X86_DS */
256 void flush_thread(void)
258 struct task_struct
*tsk
= current
;
260 if (test_tsk_thread_flag(tsk
, TIF_ABI_PENDING
)) {
261 clear_tsk_thread_flag(tsk
, TIF_ABI_PENDING
);
262 if (test_tsk_thread_flag(tsk
, TIF_IA32
)) {
263 clear_tsk_thread_flag(tsk
, TIF_IA32
);
265 set_tsk_thread_flag(tsk
, TIF_IA32
);
266 current_thread_info()->status
|= TS_COMPAT
;
269 clear_tsk_thread_flag(tsk
, TIF_DEBUG
);
271 tsk
->thread
.debugreg0
= 0;
272 tsk
->thread
.debugreg1
= 0;
273 tsk
->thread
.debugreg2
= 0;
274 tsk
->thread
.debugreg3
= 0;
275 tsk
->thread
.debugreg6
= 0;
276 tsk
->thread
.debugreg7
= 0;
277 memset(tsk
->thread
.tls_array
, 0, sizeof(tsk
->thread
.tls_array
));
279 * Forget coprocessor state..
281 tsk
->fpu_counter
= 0;
286 void release_thread(struct task_struct
*dead_task
)
289 if (dead_task
->mm
->context
.size
) {
290 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
292 dead_task
->mm
->context
.ldt
,
293 dead_task
->mm
->context
.size
);
299 static inline void set_32bit_tls(struct task_struct
*t
, int tls
, u32 addr
)
301 struct user_desc ud
= {
308 struct desc_struct
*desc
= t
->thread
.tls_array
;
313 static inline u32
read_32bit_tls(struct task_struct
*t
, int tls
)
315 return get_desc_base(&t
->thread
.tls_array
[tls
]);
319 * This gets called before we allocate a new thread and copy
320 * the current task into it.
322 void prepare_to_copy(struct task_struct
*tsk
)
327 int copy_thread(int nr
, unsigned long clone_flags
, unsigned long sp
,
328 unsigned long unused
,
329 struct task_struct
*p
, struct pt_regs
*regs
)
332 struct pt_regs
*childregs
;
333 struct task_struct
*me
= current
;
335 childregs
= ((struct pt_regs
*)
336 (THREAD_SIZE
+ task_stack_page(p
))) - 1;
342 childregs
->sp
= (unsigned long)childregs
;
344 p
->thread
.sp
= (unsigned long) childregs
;
345 p
->thread
.sp0
= (unsigned long) (childregs
+1);
346 p
->thread
.usersp
= me
->thread
.usersp
;
348 set_tsk_thread_flag(p
, TIF_FORK
);
350 p
->thread
.fs
= me
->thread
.fs
;
351 p
->thread
.gs
= me
->thread
.gs
;
353 savesegment(gs
, p
->thread
.gsindex
);
354 savesegment(fs
, p
->thread
.fsindex
);
355 savesegment(es
, p
->thread
.es
);
356 savesegment(ds
, p
->thread
.ds
);
358 if (unlikely(test_tsk_thread_flag(me
, TIF_IO_BITMAP
))) {
359 p
->thread
.io_bitmap_ptr
= kmalloc(IO_BITMAP_BYTES
, GFP_KERNEL
);
360 if (!p
->thread
.io_bitmap_ptr
) {
361 p
->thread
.io_bitmap_max
= 0;
364 memcpy(p
->thread
.io_bitmap_ptr
, me
->thread
.io_bitmap_ptr
,
366 set_tsk_thread_flag(p
, TIF_IO_BITMAP
);
370 * Set a new TLS for the child thread?
372 if (clone_flags
& CLONE_SETTLS
) {
373 #ifdef CONFIG_IA32_EMULATION
374 if (test_thread_flag(TIF_IA32
))
375 err
= do_set_thread_area(p
, -1,
376 (struct user_desc __user
*)childregs
->si
, 0);
379 err
= do_arch_prctl(p
, ARCH_SET_FS
, childregs
->r8
);
385 if (err
&& p
->thread
.io_bitmap_ptr
) {
386 kfree(p
->thread
.io_bitmap_ptr
);
387 p
->thread
.io_bitmap_max
= 0;
393 start_thread(struct pt_regs
*regs
, unsigned long new_ip
, unsigned long new_sp
)
401 write_pda(oldrsp
, new_sp
);
402 regs
->cs
= __USER_CS
;
403 regs
->ss
= __USER_DS
;
407 * Free the old FP and other extended state
409 free_thread_xstate(current
);
411 EXPORT_SYMBOL_GPL(start_thread
);
413 static void hard_disable_TSC(void)
415 write_cr4(read_cr4() | X86_CR4_TSD
);
418 void disable_TSC(void)
421 if (!test_and_set_thread_flag(TIF_NOTSC
))
423 * Must flip the CPU state synchronously with
424 * TIF_NOTSC in the current running context.
430 static void hard_enable_TSC(void)
432 write_cr4(read_cr4() & ~X86_CR4_TSD
);
435 static void enable_TSC(void)
438 if (test_and_clear_thread_flag(TIF_NOTSC
))
440 * Must flip the CPU state synchronously with
441 * TIF_NOTSC in the current running context.
447 int get_tsc_mode(unsigned long adr
)
451 if (test_thread_flag(TIF_NOTSC
))
452 val
= PR_TSC_SIGSEGV
;
456 return put_user(val
, (unsigned int __user
*)adr
);
459 int set_tsc_mode(unsigned int val
)
461 if (val
== PR_TSC_SIGSEGV
)
463 else if (val
== PR_TSC_ENABLE
)
472 * This special macro can be used to load a debugging register
474 #define loaddebug(thread, r) set_debugreg(thread->debugreg ## r, r)
476 static inline void __switch_to_xtra(struct task_struct
*prev_p
,
477 struct task_struct
*next_p
,
478 struct tss_struct
*tss
)
480 struct thread_struct
*prev
, *next
;
481 unsigned long debugctl
;
483 prev
= &prev_p
->thread
,
484 next
= &next_p
->thread
;
486 debugctl
= prev
->debugctlmsr
;
490 unsigned long ds_prev
= 0, ds_next
= 0;
493 ds_prev
= (unsigned long)prev
->ds_ctx
->ds
;
495 ds_next
= (unsigned long)next
->ds_ctx
->ds
;
497 if (ds_next
!= ds_prev
) {
499 * We clear debugctl to make sure DS
500 * is not in use when we change it:
503 update_debugctlmsr(0);
504 wrmsrl(MSR_IA32_DS_AREA
, ds_next
);
507 #endif /* CONFIG_X86_DS */
509 if (next
->debugctlmsr
!= debugctl
)
510 update_debugctlmsr(next
->debugctlmsr
);
512 if (test_tsk_thread_flag(next_p
, TIF_DEBUG
)) {
522 if (test_tsk_thread_flag(prev_p
, TIF_NOTSC
) ^
523 test_tsk_thread_flag(next_p
, TIF_NOTSC
)) {
524 /* prev and next are different */
525 if (test_tsk_thread_flag(next_p
, TIF_NOTSC
))
531 if (test_tsk_thread_flag(next_p
, TIF_IO_BITMAP
)) {
533 * Copy the relevant range of the IO bitmap.
534 * Normally this is 128 bytes or less:
536 memcpy(tss
->io_bitmap
, next
->io_bitmap_ptr
,
537 max(prev
->io_bitmap_max
, next
->io_bitmap_max
));
538 } else if (test_tsk_thread_flag(prev_p
, TIF_IO_BITMAP
)) {
540 * Clear any possible leftover bits:
542 memset(tss
->io_bitmap
, 0xff, prev
->io_bitmap_max
);
545 #ifdef CONFIG_X86_PTRACE_BTS
546 if (test_tsk_thread_flag(prev_p
, TIF_BTS_TRACE_TS
))
547 ptrace_bts_take_timestamp(prev_p
, BTS_TASK_DEPARTS
);
549 if (test_tsk_thread_flag(next_p
, TIF_BTS_TRACE_TS
))
550 ptrace_bts_take_timestamp(next_p
, BTS_TASK_ARRIVES
);
551 #endif /* CONFIG_X86_PTRACE_BTS */
555 * switch_to(x,y) should switch tasks from x to y.
557 * This could still be optimized:
558 * - fold all the options into a flag word and test it with a single test.
559 * - could test fs/gs bitsliced
561 * Kprobes not supported here. Set the probe on schedule instead.
564 __switch_to(struct task_struct
*prev_p
, struct task_struct
*next_p
)
566 struct thread_struct
*prev
= &prev_p
->thread
;
567 struct thread_struct
*next
= &next_p
->thread
;
568 int cpu
= smp_processor_id();
569 struct tss_struct
*tss
= &per_cpu(init_tss
, cpu
);
570 unsigned fsindex
, gsindex
;
572 /* we're going to use this soon, after a few expensive things */
573 if (next_p
->fpu_counter
> 5)
574 prefetch(next
->xstate
);
577 * Reload esp0, LDT and the page table pointer:
583 * This won't pick up thread selector changes, but I guess that is ok.
585 savesegment(es
, prev
->es
);
586 if (unlikely(next
->es
| prev
->es
))
587 loadsegment(es
, next
->es
);
589 savesegment(ds
, prev
->ds
);
590 if (unlikely(next
->ds
| prev
->ds
))
591 loadsegment(ds
, next
->ds
);
594 /* We must save %fs and %gs before load_TLS() because
595 * %fs and %gs may be cleared by load_TLS().
597 * (e.g. xen_load_tls())
599 savesegment(fs
, fsindex
);
600 savesegment(gs
, gsindex
);
605 * Leave lazy mode, flushing any hypercalls made here.
606 * This must be done before restoring TLS segments so
607 * the GDT and LDT are properly updated, and must be
608 * done before math_state_restore, so the TS bit is up
611 arch_leave_lazy_cpu_mode();
616 * Segment register != 0 always requires a reload. Also
617 * reload when it has changed. When prev process used 64bit
618 * base always reload to avoid an information leak.
620 if (unlikely(fsindex
| next
->fsindex
| prev
->fs
)) {
621 loadsegment(fs
, next
->fsindex
);
623 * Check if the user used a selector != 0; if yes
624 * clear 64bit base, since overloaded base is always
625 * mapped to the Null selector
630 /* when next process has a 64bit base use it */
632 wrmsrl(MSR_FS_BASE
, next
->fs
);
633 prev
->fsindex
= fsindex
;
635 if (unlikely(gsindex
| next
->gsindex
| prev
->gs
)) {
636 load_gs_index(next
->gsindex
);
641 wrmsrl(MSR_KERNEL_GS_BASE
, next
->gs
);
642 prev
->gsindex
= gsindex
;
644 /* Must be after DS reload */
648 * Switch the PDA and FPU contexts.
650 prev
->usersp
= read_pda(oldrsp
);
651 write_pda(oldrsp
, next
->usersp
);
652 write_pda(pcurrent
, next_p
);
654 write_pda(kernelstack
,
655 (unsigned long)task_stack_page(next_p
) +
656 THREAD_SIZE
- PDA_STACKOFFSET
);
657 #ifdef CONFIG_CC_STACKPROTECTOR
658 write_pda(stack_canary
, next_p
->stack_canary
);
660 * Build time only check to make sure the stack_canary is at
661 * offset 40 in the pda; this is a gcc ABI requirement
663 BUILD_BUG_ON(offsetof(struct x8664_pda
, stack_canary
) != 40);
667 * Now maybe reload the debug registers and handle I/O bitmaps
669 if (unlikely(task_thread_info(next_p
)->flags
& _TIF_WORK_CTXSW_NEXT
||
670 task_thread_info(prev_p
)->flags
& _TIF_WORK_CTXSW_PREV
))
671 __switch_to_xtra(prev_p
, next_p
, tss
);
673 /* If the task has used fpu the last 5 timeslices, just do a full
674 * restore of the math state immediately to avoid the trap; the
675 * chances of needing FPU soon are obviously high now
677 * tsk_used_math() checks prevent calling math_state_restore(),
678 * which can sleep in the case of !tsk_used_math()
680 if (tsk_used_math(next_p
) && next_p
->fpu_counter
> 5)
681 math_state_restore();
686 * sys_execve() executes a new program.
689 long sys_execve(char __user
*name
, char __user
* __user
*argv
,
690 char __user
* __user
*envp
, struct pt_regs
*regs
)
695 filename
= getname(name
);
696 error
= PTR_ERR(filename
);
697 if (IS_ERR(filename
))
699 error
= do_execve(filename
, argv
, envp
, regs
);
704 void set_personality_64bit(void)
706 /* inherit personality from parent */
708 /* Make sure to be in 64bit mode */
709 clear_thread_flag(TIF_IA32
);
711 /* TBD: overwrites user setup. Should have two bits.
712 But 64bit processes have always behaved this way,
713 so it's not too bad. The main problem is just that
714 32bit childs are affected again. */
715 current
->personality
&= ~READ_IMPLIES_EXEC
;
718 asmlinkage
long sys_fork(struct pt_regs
*regs
)
720 return do_fork(SIGCHLD
, regs
->sp
, regs
, 0, NULL
, NULL
);
724 sys_clone(unsigned long clone_flags
, unsigned long newsp
,
725 void __user
*parent_tid
, void __user
*child_tid
, struct pt_regs
*regs
)
729 return do_fork(clone_flags
, newsp
, regs
, 0, parent_tid
, child_tid
);
733 * This is trivial, and on the face of it looks like it
734 * could equally well be done in user mode.
736 * Not so, for quite unobvious reasons - register pressure.
737 * In user mode vfork() cannot have a stack frame, and if
738 * done by calling the "clone()" system call directly, you
739 * do not have enough call-clobbered registers to hold all
740 * the information you need.
742 asmlinkage
long sys_vfork(struct pt_regs
*regs
)
744 return do_fork(CLONE_VFORK
| CLONE_VM
| SIGCHLD
, regs
->sp
, regs
, 0,
748 unsigned long get_wchan(struct task_struct
*p
)
754 if (!p
|| p
== current
|| p
->state
== TASK_RUNNING
)
756 stack
= (unsigned long)task_stack_page(p
);
757 if (p
->thread
.sp
< stack
|| p
->thread
.sp
>= stack
+THREAD_SIZE
)
759 fp
= *(u64
*)(p
->thread
.sp
);
761 if (fp
< (unsigned long)stack
||
762 fp
>= (unsigned long)stack
+THREAD_SIZE
)
765 if (!in_sched_functions(ip
))
768 } while (count
++ < 16);
772 long do_arch_prctl(struct task_struct
*task
, int code
, unsigned long addr
)
775 int doit
= task
== current
;
780 if (addr
>= TASK_SIZE_OF(task
))
783 /* handle small bases via the GDT because that's faster to
785 if (addr
<= 0xffffffff) {
786 set_32bit_tls(task
, GS_TLS
, addr
);
788 load_TLS(&task
->thread
, cpu
);
789 load_gs_index(GS_TLS_SEL
);
791 task
->thread
.gsindex
= GS_TLS_SEL
;
794 task
->thread
.gsindex
= 0;
795 task
->thread
.gs
= addr
;
798 ret
= checking_wrmsrl(MSR_KERNEL_GS_BASE
, addr
);
804 /* Not strictly needed for fs, but do it for symmetry
806 if (addr
>= TASK_SIZE_OF(task
))
809 /* handle small bases via the GDT because that's faster to
811 if (addr
<= 0xffffffff) {
812 set_32bit_tls(task
, FS_TLS
, addr
);
814 load_TLS(&task
->thread
, cpu
);
815 loadsegment(fs
, FS_TLS_SEL
);
817 task
->thread
.fsindex
= FS_TLS_SEL
;
820 task
->thread
.fsindex
= 0;
821 task
->thread
.fs
= addr
;
823 /* set the selector to 0 to not confuse
826 ret
= checking_wrmsrl(MSR_FS_BASE
, addr
);
833 if (task
->thread
.fsindex
== FS_TLS_SEL
)
834 base
= read_32bit_tls(task
, FS_TLS
);
836 rdmsrl(MSR_FS_BASE
, base
);
838 base
= task
->thread
.fs
;
839 ret
= put_user(base
, (unsigned long __user
*)addr
);
845 if (task
->thread
.gsindex
== GS_TLS_SEL
)
846 base
= read_32bit_tls(task
, GS_TLS
);
848 savesegment(gs
, gsindex
);
850 rdmsrl(MSR_KERNEL_GS_BASE
, base
);
852 base
= task
->thread
.gs
;
854 base
= task
->thread
.gs
;
855 ret
= put_user(base
, (unsigned long __user
*)addr
);
867 long sys_arch_prctl(int code
, unsigned long addr
)
869 return do_arch_prctl(current
, code
, addr
);
872 unsigned long arch_align_stack(unsigned long sp
)
874 if (!(current
->personality
& ADDR_NO_RANDOMIZE
) && randomize_va_space
)
875 sp
-= get_random_int() % 8192;
879 unsigned long arch_randomize_brk(struct mm_struct
*mm
)
881 unsigned long range_end
= mm
->brk
+ 0x02000000;
882 return randomize_range(mm
->brk
, range_end
, 0) ? : mm
->brk
;