2 * Copyright (c) 1992 Terrence R. Lambert.
3 * Copyright (C) 1994, David Greenman
4 * Copyright (c) 1982, 1987, 1990, 1993
5 * The Regents of the University of California. All rights reserved.
7 * This code is derived from software contributed to Berkeley by
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the University of
21 * California, Berkeley and its contributors.
22 * 4. Neither the name of the University nor the names of its contributors
23 * may be used to endorse or promote products derived from this software
24 * without specific prior written permission.
26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
39 * $FreeBSD: src/sys/i386/i386/machdep.c,v 1.385.2.30 2003/05/31 08:48:05 alc Exp $
40 * $DragonFly: src/sys/platform/vkernel/i386/cpu_regs.c,v 1.1 2007/01/05 23:18:18 dillon Exp $
43 #include "use_ether.h"
46 #include "opt_atalk.h"
47 #include "opt_compat.h"
49 #include "opt_directio.h"
52 #include "opt_msgbuf.h"
55 #include <sys/param.h>
56 #include <sys/systm.h>
57 #include <sys/sysproto.h>
58 #include <sys/signalvar.h>
59 #include <sys/kernel.h>
60 #include <sys/linker.h>
61 #include <sys/malloc.h>
64 #include <sys/reboot.h>
66 #include <sys/msgbuf.h>
67 #include <sys/sysent.h>
68 #include <sys/sysctl.h>
69 #include <sys/vmmeter.h>
71 #include <sys/upcall.h>
72 #include <sys/usched.h>
76 #include <vm/vm_param.h>
78 #include <vm/vm_kern.h>
79 #include <vm/vm_object.h>
80 #include <vm/vm_page.h>
81 #include <vm/vm_map.h>
82 #include <vm/vm_pager.h>
83 #include <vm/vm_extern.h>
85 #include <sys/thread2.h>
93 #include <machine/cpu.h>
94 #include <machine/clock.h>
95 #include <machine/specialreg.h>
96 #include <machine/md_var.h>
97 #include <machine/pcb_ext.h> /* pcb.h included via sys/user.h */
98 #include <machine/globaldata.h> /* CPU_prvspace */
99 #include <machine/smp.h>
101 #include <machine/perfmon.h>
103 #include <machine/cputypes.h>
105 #include <bus/isa/rtc.h>
106 #include <machine/vm86.h>
107 #include <sys/random.h>
108 #include <sys/ptrace.h>
109 #include <machine/sigframe.h>
111 extern void dblfault_handler (void);
113 #ifndef CPU_DISABLE_SSE
114 static void set_fpregs_xmm (struct save87
*, struct savexmm
*);
115 static void fill_fpregs_xmm (struct savexmm
*, struct save87
*);
116 #endif /* CPU_DISABLE_SSE */
118 extern void ffs_rawread_setup(void);
119 #endif /* DIRECTIO */
122 int64_t tsc_offsets
[MAXCPU
];
124 int64_t tsc_offsets
[1];
127 #if defined(SWTCH_OPTIM_STATS)
128 extern int swtch_optim_stats
;
129 SYSCTL_INT(_debug
, OID_AUTO
, swtch_optim_stats
,
130 CTLFLAG_RD
, &swtch_optim_stats
, 0, "");
131 SYSCTL_INT(_debug
, OID_AUTO
, tlb_flush_count
,
132 CTLFLAG_RD
, &tlb_flush_count
, 0, "");
138 sysctl_hw_physmem(SYSCTL_HANDLER_ARGS
)
140 int error
= sysctl_handle_int(oidp
, 0, ctob(physmem
), req
);
144 SYSCTL_PROC(_hw
, HW_PHYSMEM
, physmem
, CTLTYPE_INT
|CTLFLAG_RD
,
145 0, 0, sysctl_hw_physmem
, "IU", "");
148 sysctl_hw_usermem(SYSCTL_HANDLER_ARGS
)
150 int error
= sysctl_handle_int(oidp
, 0,
151 ctob(physmem
- vmstats
.v_wire_count
), req
);
155 SYSCTL_PROC(_hw
, HW_USERMEM
, usermem
, CTLTYPE_INT
|CTLFLAG_RD
,
156 0, 0, sysctl_hw_usermem
, "IU", "");
161 sysctl_machdep_msgbuf(SYSCTL_HANDLER_ARGS
)
165 /* Unwind the buffer, so that it's linear (possibly starting with
166 * some initial nulls).
168 error
=sysctl_handle_opaque(oidp
,msgbufp
->msg_ptr
+msgbufp
->msg_bufr
,
169 msgbufp
->msg_size
-msgbufp
->msg_bufr
,req
);
170 if(error
) return(error
);
171 if(msgbufp
->msg_bufr
>0) {
172 error
=sysctl_handle_opaque(oidp
,msgbufp
->msg_ptr
,
173 msgbufp
->msg_bufr
,req
);
178 SYSCTL_PROC(_machdep
, OID_AUTO
, msgbuf
, CTLTYPE_STRING
|CTLFLAG_RD
,
179 0, 0, sysctl_machdep_msgbuf
, "A","Contents of kernel message buffer");
181 static int msgbuf_clear
;
184 sysctl_machdep_msgbuf_clear(SYSCTL_HANDLER_ARGS
)
187 error
= sysctl_handle_int(oidp
, oidp
->oid_arg1
, oidp
->oid_arg2
,
189 if (!error
&& req
->newptr
) {
190 /* Clear the buffer and reset write pointer */
191 bzero(msgbufp
->msg_ptr
,msgbufp
->msg_size
);
192 msgbufp
->msg_bufr
=msgbufp
->msg_bufx
=0;
198 SYSCTL_PROC(_machdep
, OID_AUTO
, msgbuf_clear
, CTLTYPE_INT
|CTLFLAG_RW
,
199 &msgbuf_clear
, 0, sysctl_machdep_msgbuf_clear
, "I",
200 "Clear kernel message buffer");
205 * Send an interrupt to process.
207 * Stack is set up to allow sigcode stored
208 * at top to call routine, followed by kcall
209 * to sigreturn routine below. After sigreturn
210 * resets the signal mask, the stack, and the
211 * frame pointer, it returns to the user
215 sendsig(sig_t catcher
, int sig
, sigset_t
*mask
, u_long code
)
217 struct lwp
*lp
= curthread
->td_lwp
;
218 struct proc
*p
= lp
->lwp_proc
;
219 struct trapframe
*regs
;
220 struct sigacts
*psp
= p
->p_sigacts
;
221 struct sigframe sf
, *sfp
;
224 regs
= lp
->lwp_md
.md_regs
;
225 oonstack
= (lp
->lwp_sigstk
.ss_flags
& SS_ONSTACK
) ? 1 : 0;
227 /* save user context */
228 bzero(&sf
, sizeof(struct sigframe
));
229 sf
.sf_uc
.uc_sigmask
= *mask
;
230 sf
.sf_uc
.uc_stack
= lp
->lwp_sigstk
;
231 sf
.sf_uc
.uc_mcontext
.mc_onstack
= oonstack
;
232 sf
.sf_uc
.uc_mcontext
.mc_gs
= rgs();
233 bcopy(regs
, &sf
.sf_uc
.uc_mcontext
.mc_fs
, sizeof(struct trapframe
));
235 /* Allocate and validate space for the signal handler context. */
237 if ((p
->p_flag
& P_ALTSTACK
) != 0 && !oonstack
&&
238 SIGISMEMBER(psp
->ps_sigonstack
, sig
)) {
239 sfp
= (struct sigframe
*)(lp
->lwp_sigstk
.ss_sp
+
240 lp
->lwp_sigstk
.ss_size
- sizeof(struct sigframe
));
241 lp
->lwp_sigstk
.ss_flags
|= SS_ONSTACK
;
244 sfp
= (struct sigframe
*)regs
->tf_esp
- 1;
246 /* Translate the signal is appropriate */
247 if (p
->p_sysent
->sv_sigtbl
) {
248 if (sig
<= p
->p_sysent
->sv_sigsize
)
249 sig
= p
->p_sysent
->sv_sigtbl
[_SIG_IDX(sig
)];
252 /* Build the argument list for the signal handler. */
254 sf
.sf_ucontext
= (register_t
)&sfp
->sf_uc
;
255 if (SIGISMEMBER(psp
->ps_siginfo
, sig
)) {
256 /* Signal handler installed with SA_SIGINFO. */
257 sf
.sf_siginfo
= (register_t
)&sfp
->sf_si
;
258 sf
.sf_ahu
.sf_action
= (__siginfohandler_t
*)catcher
;
260 /* fill siginfo structure */
261 sf
.sf_si
.si_signo
= sig
;
262 sf
.sf_si
.si_code
= code
;
263 sf
.sf_si
.si_addr
= (void*)regs
->tf_err
;
266 /* Old FreeBSD-style arguments. */
267 sf
.sf_siginfo
= code
;
268 sf
.sf_addr
= regs
->tf_err
;
269 sf
.sf_ahu
.sf_handler
= catcher
;
274 * If we're a vm86 process, we want to save the segment registers.
275 * We also change eflags to be our emulated eflags, not the actual
278 if (regs
->tf_eflags
& PSL_VM
) {
279 struct trapframe_vm86
*tf
= (struct trapframe_vm86
*)regs
;
280 struct vm86_kernel
*vm86
= &lp
->lwp_thread
->td_pcb
->pcb_ext
->ext_vm86
;
282 sf
.sf_uc
.uc_mcontext
.mc_gs
= tf
->tf_vm86_gs
;
283 sf
.sf_uc
.uc_mcontext
.mc_fs
= tf
->tf_vm86_fs
;
284 sf
.sf_uc
.uc_mcontext
.mc_es
= tf
->tf_vm86_es
;
285 sf
.sf_uc
.uc_mcontext
.mc_ds
= tf
->tf_vm86_ds
;
287 if (vm86
->vm86_has_vme
== 0)
288 sf
.sf_uc
.uc_mcontext
.mc_eflags
=
289 (tf
->tf_eflags
& ~(PSL_VIF
| PSL_VIP
)) |
290 (vm86
->vm86_eflags
& (PSL_VIF
| PSL_VIP
));
293 * Clear PSL_NT to inhibit T_TSSFLT faults on return from
294 * syscalls made by the signal handler. This just avoids
295 * wasting time for our lazy fixup of such faults. PSL_NT
296 * does nothing in vm86 mode, but vm86 programs can set it
297 * almost legitimately in probes for old cpu types.
299 tf
->tf_eflags
&= ~(PSL_VM
| PSL_NT
| PSL_VIF
| PSL_VIP
);
304 * Copy the sigframe out to the user's stack.
306 if (copyout(&sf
, sfp
, sizeof(struct sigframe
)) != 0) {
308 * Something is wrong with the stack pointer.
309 * ...Kill the process.
314 regs
->tf_esp
= (int)sfp
;
315 regs
->tf_eip
= PS_STRINGS
- *(p
->p_sysent
->sv_szsigcode
);
316 regs
->tf_eflags
&= ~PSL_T
;
325 * Sanitize the trapframe for a virtual kernel passing control to a custom
328 * Allow userland to set or maintain PSL_RF, the resume flag. This flag
329 * basically controls whether the return PC should skip the first instruction
330 * (as in an explicit system call) or re-execute it (as in an exception).
333 cpu_sanitize_frame(struct trapframe
*frame
)
340 frame
->tf_eflags
&= (PSL_USER
| PSL_RF
);
341 frame
->tf_eflags
|= PSL_RESERVED_DEFAULT
| PSL_I
;
346 * sigreturn(ucontext_t *sigcntxp)
348 * System call to cleanup state after a signal
349 * has been taken. Reset signal mask and
350 * stack state from context left by sendsig (above).
351 * Return to previous pc and psl as specified by
352 * context left by sendsig. Check carefully to
353 * make sure that the user has not modified the
354 * state to gain improper privileges.
356 #define EFL_SECURE(ef, oef) ((((ef) ^ (oef)) & ~PSL_USERCHANGE) == 0)
357 #define CS_SECURE(cs) (ISPL(cs) == SEL_UPL)
360 sys_sigreturn(struct sigreturn_args
*uap
)
362 struct lwp
*lp
= curthread
->td_lwp
;
363 struct trapframe
*regs
;
369 if (!useracc((caddr_t
)ucp
, sizeof(ucontext_t
), VM_PROT_READ
))
372 regs
= lp
->lwp_md
.md_regs
;
373 eflags
= ucp
->uc_mcontext
.mc_eflags
;
376 if (eflags
& PSL_VM
) {
377 struct trapframe_vm86
*tf
= (struct trapframe_vm86
*)regs
;
378 struct vm86_kernel
*vm86
;
381 * if pcb_ext == 0 or vm86_inited == 0, the user hasn't
382 * set up the vm86 area, and we can't enter vm86 mode.
384 if (lp
->lwp_thread
->td_pcb
->pcb_ext
== 0)
386 vm86
= &lp
->lwp_thread
->td_pcb
->pcb_ext
->ext_vm86
;
387 if (vm86
->vm86_inited
== 0)
390 /* go back to user mode if both flags are set */
391 if ((eflags
& PSL_VIP
) && (eflags
& PSL_VIF
))
392 trapsignal(lp
->lwp_proc
, SIGBUS
, 0);
394 if (vm86
->vm86_has_vme
) {
395 eflags
= (tf
->tf_eflags
& ~VME_USERCHANGE
) |
396 (eflags
& VME_USERCHANGE
) | PSL_VM
;
398 vm86
->vm86_eflags
= eflags
; /* save VIF, VIP */
399 eflags
= (tf
->tf_eflags
& ~VM_USERCHANGE
) | (eflags
& VM_USERCHANGE
) | PSL_VM
;
401 bcopy(&ucp
->uc_mcontext
.mc_fs
, tf
, sizeof(struct trapframe
));
402 tf
->tf_eflags
= eflags
;
403 tf
->tf_vm86_ds
= tf
->tf_ds
;
404 tf
->tf_vm86_es
= tf
->tf_es
;
405 tf
->tf_vm86_fs
= tf
->tf_fs
;
406 tf
->tf_vm86_gs
= ucp
->uc_mcontext
.mc_gs
;
414 * Don't allow users to change privileged or reserved flags.
417 * XXX do allow users to change the privileged flag PSL_RF.
418 * The cpu sets PSL_RF in tf_eflags for faults. Debuggers
419 * should sometimes set it there too. tf_eflags is kept in
420 * the signal context during signal handling and there is no
421 * other place to remember it, so the PSL_RF bit may be
422 * corrupted by the signal handler without us knowing.
423 * Corruption of the PSL_RF bit at worst causes one more or
424 * one less debugger trap, so allowing it is fairly harmless.
426 if (!EFL_SECURE(eflags
& ~PSL_RF
, regs
->tf_eflags
& ~PSL_RF
)) {
427 kprintf("sigreturn: eflags = 0x%x\n", eflags
);
432 * Don't allow users to load a valid privileged %cs. Let the
433 * hardware check for invalid selectors, excess privilege in
434 * other selectors, invalid %eip's and invalid %esp's.
436 cs
= ucp
->uc_mcontext
.mc_cs
;
437 if (!CS_SECURE(cs
)) {
438 kprintf("sigreturn: cs = 0x%x\n", cs
);
439 trapsignal(lp
->lwp_proc
, SIGBUS
, T_PROTFLT
);
442 bcopy(&ucp
->uc_mcontext
.mc_fs
, regs
, sizeof(struct trapframe
));
445 if (ucp
->uc_mcontext
.mc_onstack
& 1)
446 lp
->lwp_sigstk
.ss_flags
|= SS_ONSTACK
;
448 lp
->lwp_sigstk
.ss_flags
&= ~SS_ONSTACK
;
450 lp
->lwp_sigmask
= ucp
->uc_sigmask
;
451 SIG_CANTMASK(lp
->lwp_sigmask
);
456 * Stack frame on entry to function. %eax will contain the function vector,
457 * %ecx will contain the function data. flags, ecx, and eax will have
458 * already been pushed on the stack.
469 sendupcall(struct vmupcall
*vu
, int morepending
)
471 struct lwp
*lp
= curthread
->td_lwp
;
472 struct trapframe
*regs
;
473 struct upcall upcall
;
474 struct upc_frame upc_frame
;
478 * Get the upcall data structure
480 if (copyin(lp
->lwp_upcall
, &upcall
, sizeof(upcall
)) ||
481 copyin((char *)upcall
.upc_uthread
+ upcall
.upc_critoff
, &crit_count
, sizeof(int))
484 kprintf("bad upcall address\n");
489 * If the data structure is already marked pending or has a critical
490 * section count, mark the data structure as pending and return
491 * without doing an upcall. vu_pending is left set.
493 if (upcall
.upc_pending
|| crit_count
>= vu
->vu_pending
) {
494 if (upcall
.upc_pending
< vu
->vu_pending
) {
495 upcall
.upc_pending
= vu
->vu_pending
;
496 copyout(&upcall
.upc_pending
, &lp
->lwp_upcall
->upc_pending
,
497 sizeof(upcall
.upc_pending
));
503 * We can run this upcall now, clear vu_pending.
505 * Bump our critical section count and set or clear the
506 * user pending flag depending on whether more upcalls are
507 * pending. The user will be responsible for calling
508 * upc_dispatch(-1) to process remaining upcalls.
511 upcall
.upc_pending
= morepending
;
512 crit_count
+= TDPRI_CRIT
;
513 copyout(&upcall
.upc_pending
, &lp
->lwp_upcall
->upc_pending
,
514 sizeof(upcall
.upc_pending
));
515 copyout(&crit_count
, (char *)upcall
.upc_uthread
+ upcall
.upc_critoff
,
519 * Construct a stack frame and issue the upcall
521 regs
= lp
->lwp_md
.md_regs
;
522 upc_frame
.eax
= regs
->tf_eax
;
523 upc_frame
.ecx
= regs
->tf_ecx
;
524 upc_frame
.edx
= regs
->tf_edx
;
525 upc_frame
.flags
= regs
->tf_eflags
;
526 upc_frame
.oldip
= regs
->tf_eip
;
527 if (copyout(&upc_frame
, (void *)(regs
->tf_esp
- sizeof(upc_frame
)),
528 sizeof(upc_frame
)) != 0) {
529 kprintf("bad stack on upcall\n");
531 regs
->tf_eax
= (register_t
)vu
->vu_func
;
532 regs
->tf_ecx
= (register_t
)vu
->vu_data
;
533 regs
->tf_edx
= (register_t
)lp
->lwp_upcall
;
534 regs
->tf_eip
= (register_t
)vu
->vu_ctx
;
535 regs
->tf_esp
-= sizeof(upc_frame
);
540 * fetchupcall occurs in the context of a system call, which means that
541 * we have to return EJUSTRETURN in order to prevent eax and edx from
542 * being overwritten by the syscall return value.
544 * if vu is not NULL we return the new context in %edx, the new data in %ecx,
545 * and the function pointer in %eax.
548 fetchupcall (struct vmupcall
*vu
, int morepending
, void *rsp
)
550 struct upc_frame upc_frame
;
551 struct lwp
*lp
= curthread
->td_lwp
;
552 struct trapframe
*regs
;
554 struct upcall upcall
;
557 regs
= lp
->lwp_md
.md_regs
;
559 error
= copyout(&morepending
, &lp
->lwp_upcall
->upc_pending
, sizeof(int));
563 * This jumps us to the next ready context.
566 error
= copyin(lp
->lwp_upcall
, &upcall
, sizeof(upcall
));
569 error
= copyin((char *)upcall
.upc_uthread
+ upcall
.upc_critoff
, &crit_count
, sizeof(int));
570 crit_count
+= TDPRI_CRIT
;
572 error
= copyout(&crit_count
, (char *)upcall
.upc_uthread
+ upcall
.upc_critoff
, sizeof(int));
573 regs
->tf_eax
= (register_t
)vu
->vu_func
;
574 regs
->tf_ecx
= (register_t
)vu
->vu_data
;
575 regs
->tf_edx
= (register_t
)lp
->lwp_upcall
;
576 regs
->tf_eip
= (register_t
)vu
->vu_ctx
;
577 regs
->tf_esp
= (register_t
)rsp
;
580 * This returns us to the originally interrupted code.
582 error
= copyin(rsp
, &upc_frame
, sizeof(upc_frame
));
583 regs
->tf_eax
= upc_frame
.eax
;
584 regs
->tf_ecx
= upc_frame
.ecx
;
585 regs
->tf_edx
= upc_frame
.edx
;
586 regs
->tf_eflags
= (regs
->tf_eflags
& ~PSL_USERCHANGE
) |
587 (upc_frame
.flags
& PSL_USERCHANGE
);
588 regs
->tf_eip
= upc_frame
.oldip
;
589 regs
->tf_esp
= (register_t
)((char *)rsp
+ sizeof(upc_frame
));
598 * cpu_idle() represents the idle LWKT. You cannot return from this function
599 * (unless you want to blow things up!). Instead we look for runnable threads
600 * and loop or halt as appropriate. Giant is not held on entry to the thread.
602 * The main loop is entered with a critical section held, we must release
603 * the critical section before doing anything else. lwkt_switch() will
604 * check for pending interrupts due to entering and exiting its own
607 * Note on cpu_idle_hlt: On an SMP system we rely on a scheduler IPI
608 * to wake a HLTed cpu up. However, there are cases where the idlethread
609 * will be entered with the possibility that no IPI will occur and in such
610 * cases lwkt_switch() sets TDF_IDLE_NOHLT.
612 static int cpu_idle_hlt
= 1;
613 static int cpu_idle_hltcnt
;
614 static int cpu_idle_spincnt
;
615 SYSCTL_INT(_machdep
, OID_AUTO
, cpu_idle_hlt
, CTLFLAG_RW
,
616 &cpu_idle_hlt
, 0, "Idle loop HLT enable");
617 SYSCTL_INT(_machdep
, OID_AUTO
, cpu_idle_hltcnt
, CTLFLAG_RW
,
618 &cpu_idle_hltcnt
, 0, "Idle loop entry halts");
619 SYSCTL_INT(_machdep
, OID_AUTO
, cpu_idle_spincnt
, CTLFLAG_RW
,
620 &cpu_idle_spincnt
, 0, "Idle loop entry spins");
623 cpu_idle_default_hook(void)
626 * We must guarentee that hlt is exactly the instruction
629 __asm
__volatile("hlt"); /* sti; hlt */
632 /* Other subsystems (e.g., ACPI) can hook this later. */
633 void (*cpu_idle_hook
)(void) = cpu_idle_default_hook
;
638 struct thread
*td
= curthread
;
641 KKASSERT(td
->td_pri
< TDPRI_CRIT
);
644 * See if there are any LWKTs ready to go.
649 * If we are going to halt call splz unconditionally after
650 * CLIing to catch any interrupt races. Note that we are
651 * at SPL0 and interrupts are enabled.
653 if (cpu_idle_hlt
&& !lwkt_runnable() &&
654 (td
->td_flags
& TDF_IDLE_NOHLT
) == 0) {
655 /* __asm __volatile("cli"); */
657 if (!lwkt_runnable())
661 __asm
__volatile("pause");
665 td
->td_flags
&= ~TDF_IDLE_NOHLT
;
668 /*__asm __volatile("sti; pause");*/
669 __asm
__volatile("pause");
671 /*__asm __volatile("sti");*/
679 * Clear registers on exec
682 setregs(struct lwp
*lp
, u_long entry
, u_long stack
, u_long ps_strings
)
684 struct trapframe
*regs
= lp
->lwp_md
.md_regs
;
685 struct pcb
*pcb
= lp
->lwp_thread
->td_pcb
;
687 /* Reset pc->pcb_gs and %gs before possibly invalidating it. */
693 /* was i386_user_cleanup() in NetBSD */
696 bzero((char *)regs
, sizeof(struct trapframe
));
697 regs
->tf_eip
= entry
;
698 regs
->tf_esp
= stack
;
699 regs
->tf_eflags
= PSL_USER
| (regs
->tf_eflags
& PSL_T
);
706 /* PS_STRINGS value for BSD/OS binaries. It is 0 for non-BSD/OS. */
707 regs
->tf_ebx
= ps_strings
;
710 * Reset the hardware debug registers if they were in use.
711 * They won't have any meaning for the newly exec'd process.
713 if (pcb
->pcb_flags
& PCB_DBREGS
) {
720 if (pcb
== curthread
->td_pcb
) {
722 * Clear the debug registers on the running
723 * CPU, otherwise they will end up affecting
724 * the next process we switch to.
728 pcb
->pcb_flags
&= ~PCB_DBREGS
;
732 * Initialize the math emulator (if any) for the current process.
733 * Actually, just clear the bit that says that the emulator has
734 * been initialized. Initialization is delayed until the process
735 * traps to the emulator (if it is done at all) mainly because
736 * emulators don't provide an entry point for initialization.
738 lp
->lwp_thread
->td_pcb
->pcb_flags
&= ~FP_SOFTFP
;
741 * note: do not set CR0_TS here. npxinit() must do it after clearing
742 * gd_npxthread. Otherwise a preemptive interrupt thread may panic
747 load_cr0(rcr0() | CR0_MP
);
751 /* Initialize the npx (if any) for the current process. */
752 npxinit(__INITIAL_NPXCW__
);
757 * note: linux emulator needs edx to be 0x0 on entry, which is
758 * handled in execve simply by setting the 64 bit syscall
770 cr0
|= CR0_NE
; /* Done by npxinit() */
771 cr0
|= CR0_MP
| CR0_TS
; /* Done at every execve() too. */
773 if (cpu_class
!= CPUCLASS_386
)
775 cr0
|= CR0_WP
| CR0_AM
;
782 sysctl_machdep_adjkerntz(SYSCTL_HANDLER_ARGS
)
785 error
= sysctl_handle_int(oidp
, oidp
->oid_arg1
, oidp
->oid_arg2
,
787 if (!error
&& req
->newptr
)
792 SYSCTL_PROC(_machdep
, CPU_ADJKERNTZ
, adjkerntz
, CTLTYPE_INT
|CTLFLAG_RW
,
793 &adjkerntz
, 0, sysctl_machdep_adjkerntz
, "I", "");
795 extern u_long bootdev
; /* not a cdev_t - encoding is different */
796 SYSCTL_ULONG(_machdep
, OID_AUTO
, guessed_bootdev
,
797 CTLFLAG_RD
, &bootdev
, 0, "Boot device (not in cdev_t format)");
800 * Initialize 386 and configure to run kernel
804 * Initialize segments & interrupt table
807 extern struct user
*proc0paddr
;
812 IDTVEC(div
), IDTVEC(dbg
), IDTVEC(nmi
), IDTVEC(bpt
), IDTVEC(ofl
),
813 IDTVEC(bnd
), IDTVEC(ill
), IDTVEC(dna
), IDTVEC(fpusegm
),
814 IDTVEC(tss
), IDTVEC(missing
), IDTVEC(stk
), IDTVEC(prot
),
815 IDTVEC(page
), IDTVEC(mchk
), IDTVEC(fpu
), IDTVEC(align
),
816 IDTVEC(xmm
), IDTVEC(syscall
),
819 IDTVEC(int0x80_syscall
);
823 #ifdef DEBUG_INTERRUPTS
824 extern inthand_t
*Xrsvdary
[256];
828 ptrace_set_pc(struct proc
*p
, unsigned long addr
)
830 p
->p_md
.md_regs
->tf_eip
= addr
;
835 ptrace_single_step(struct lwp
*lp
)
837 lp
->lwp_md
.md_regs
->tf_eflags
|= PSL_T
;
842 fill_regs(struct lwp
*lp
, struct reg
*regs
)
845 struct trapframe
*tp
;
847 tp
= lp
->lwp_md
.md_regs
;
848 regs
->r_fs
= tp
->tf_fs
;
849 regs
->r_es
= tp
->tf_es
;
850 regs
->r_ds
= tp
->tf_ds
;
851 regs
->r_edi
= tp
->tf_edi
;
852 regs
->r_esi
= tp
->tf_esi
;
853 regs
->r_ebp
= tp
->tf_ebp
;
854 regs
->r_ebx
= tp
->tf_ebx
;
855 regs
->r_edx
= tp
->tf_edx
;
856 regs
->r_ecx
= tp
->tf_ecx
;
857 regs
->r_eax
= tp
->tf_eax
;
858 regs
->r_eip
= tp
->tf_eip
;
859 regs
->r_cs
= tp
->tf_cs
;
860 regs
->r_eflags
= tp
->tf_eflags
;
861 regs
->r_esp
= tp
->tf_esp
;
862 regs
->r_ss
= tp
->tf_ss
;
863 pcb
= lp
->lwp_thread
->td_pcb
;
864 regs
->r_gs
= pcb
->pcb_gs
;
869 set_regs(struct lwp
*lp
, struct reg
*regs
)
872 struct trapframe
*tp
;
874 tp
= lp
->lwp_md
.md_regs
;
875 if (!EFL_SECURE(regs
->r_eflags
, tp
->tf_eflags
) ||
876 !CS_SECURE(regs
->r_cs
))
878 tp
->tf_fs
= regs
->r_fs
;
879 tp
->tf_es
= regs
->r_es
;
880 tp
->tf_ds
= regs
->r_ds
;
881 tp
->tf_edi
= regs
->r_edi
;
882 tp
->tf_esi
= regs
->r_esi
;
883 tp
->tf_ebp
= regs
->r_ebp
;
884 tp
->tf_ebx
= regs
->r_ebx
;
885 tp
->tf_edx
= regs
->r_edx
;
886 tp
->tf_ecx
= regs
->r_ecx
;
887 tp
->tf_eax
= regs
->r_eax
;
888 tp
->tf_eip
= regs
->r_eip
;
889 tp
->tf_cs
= regs
->r_cs
;
890 tp
->tf_eflags
= regs
->r_eflags
;
891 tp
->tf_esp
= regs
->r_esp
;
892 tp
->tf_ss
= regs
->r_ss
;
893 pcb
= lp
->lwp_thread
->td_pcb
;
894 pcb
->pcb_gs
= regs
->r_gs
;
898 #ifndef CPU_DISABLE_SSE
900 fill_fpregs_xmm(struct savexmm
*sv_xmm
, struct save87
*sv_87
)
902 struct env87
*penv_87
= &sv_87
->sv_env
;
903 struct envxmm
*penv_xmm
= &sv_xmm
->sv_env
;
906 /* FPU control/status */
907 penv_87
->en_cw
= penv_xmm
->en_cw
;
908 penv_87
->en_sw
= penv_xmm
->en_sw
;
909 penv_87
->en_tw
= penv_xmm
->en_tw
;
910 penv_87
->en_fip
= penv_xmm
->en_fip
;
911 penv_87
->en_fcs
= penv_xmm
->en_fcs
;
912 penv_87
->en_opcode
= penv_xmm
->en_opcode
;
913 penv_87
->en_foo
= penv_xmm
->en_foo
;
914 penv_87
->en_fos
= penv_xmm
->en_fos
;
917 for (i
= 0; i
< 8; ++i
)
918 sv_87
->sv_ac
[i
] = sv_xmm
->sv_fp
[i
].fp_acc
;
920 sv_87
->sv_ex_sw
= sv_xmm
->sv_ex_sw
;
924 set_fpregs_xmm(struct save87
*sv_87
, struct savexmm
*sv_xmm
)
926 struct env87
*penv_87
= &sv_87
->sv_env
;
927 struct envxmm
*penv_xmm
= &sv_xmm
->sv_env
;
930 /* FPU control/status */
931 penv_xmm
->en_cw
= penv_87
->en_cw
;
932 penv_xmm
->en_sw
= penv_87
->en_sw
;
933 penv_xmm
->en_tw
= penv_87
->en_tw
;
934 penv_xmm
->en_fip
= penv_87
->en_fip
;
935 penv_xmm
->en_fcs
= penv_87
->en_fcs
;
936 penv_xmm
->en_opcode
= penv_87
->en_opcode
;
937 penv_xmm
->en_foo
= penv_87
->en_foo
;
938 penv_xmm
->en_fos
= penv_87
->en_fos
;
941 for (i
= 0; i
< 8; ++i
)
942 sv_xmm
->sv_fp
[i
].fp_acc
= sv_87
->sv_ac
[i
];
944 sv_xmm
->sv_ex_sw
= sv_87
->sv_ex_sw
;
946 #endif /* CPU_DISABLE_SSE */
949 fill_fpregs(struct lwp
*lp
, struct fpreg
*fpregs
)
951 #ifndef CPU_DISABLE_SSE
953 fill_fpregs_xmm(&lp
->lwp_thread
->td_pcb
->pcb_save
.sv_xmm
,
954 (struct save87
*)fpregs
);
957 #endif /* CPU_DISABLE_SSE */
958 bcopy(&lp
->lwp_thread
->td_pcb
->pcb_save
.sv_87
, fpregs
, sizeof *fpregs
);
963 set_fpregs(struct lwp
*lp
, struct fpreg
*fpregs
)
965 #ifndef CPU_DISABLE_SSE
967 set_fpregs_xmm((struct save87
*)fpregs
,
968 &lp
->lwp_thread
->td_pcb
->pcb_save
.sv_xmm
);
971 #endif /* CPU_DISABLE_SSE */
972 bcopy(fpregs
, &lp
->lwp_thread
->td_pcb
->pcb_save
.sv_87
, sizeof *fpregs
);
977 fill_dbregs(struct lwp
*lp
, struct dbreg
*dbregs
)
980 dbregs
->dr0
= rdr0();
981 dbregs
->dr1
= rdr1();
982 dbregs
->dr2
= rdr2();
983 dbregs
->dr3
= rdr3();
984 dbregs
->dr4
= rdr4();
985 dbregs
->dr5
= rdr5();
986 dbregs
->dr6
= rdr6();
987 dbregs
->dr7
= rdr7();
991 pcb
= lp
->lwp_thread
->td_pcb
;
992 dbregs
->dr0
= pcb
->pcb_dr0
;
993 dbregs
->dr1
= pcb
->pcb_dr1
;
994 dbregs
->dr2
= pcb
->pcb_dr2
;
995 dbregs
->dr3
= pcb
->pcb_dr3
;
998 dbregs
->dr6
= pcb
->pcb_dr6
;
999 dbregs
->dr7
= pcb
->pcb_dr7
;
1005 set_dbregs(struct lwp
*lp
, struct dbreg
*dbregs
)
1008 load_dr0(dbregs
->dr0
);
1009 load_dr1(dbregs
->dr1
);
1010 load_dr2(dbregs
->dr2
);
1011 load_dr3(dbregs
->dr3
);
1012 load_dr4(dbregs
->dr4
);
1013 load_dr5(dbregs
->dr5
);
1014 load_dr6(dbregs
->dr6
);
1015 load_dr7(dbregs
->dr7
);
1018 struct ucred
*ucred
;
1020 uint32_t mask1
, mask2
;
1023 * Don't let an illegal value for dr7 get set. Specifically,
1024 * check for undefined settings. Setting these bit patterns
1025 * result in undefined behaviour and can lead to an unexpected
1028 for (i
= 0, mask1
= 0x3<<16, mask2
= 0x2<<16; i
< 8;
1029 i
++, mask1
<<= 2, mask2
<<= 2)
1030 if ((dbregs
->dr7
& mask1
) == mask2
)
1033 pcb
= lp
->lwp_thread
->td_pcb
;
1034 ucred
= lp
->lwp_proc
->p_ucred
;
1037 * Don't let a process set a breakpoint that is not within the
1038 * process's address space. If a process could do this, it
1039 * could halt the system by setting a breakpoint in the kernel
1040 * (if ddb was enabled). Thus, we need to check to make sure
1041 * that no breakpoints are being enabled for addresses outside
1042 * process's address space, unless, perhaps, we were called by
1045 * XXX - what about when the watched area of the user's
1046 * address space is written into from within the kernel
1047 * ... wouldn't that still cause a breakpoint to be generated
1048 * from within kernel mode?
1051 if (suser_cred(ucred
, 0) != 0) {
1052 if (dbregs
->dr7
& 0x3) {
1053 /* dr0 is enabled */
1054 if (dbregs
->dr0
>= VM_MAX_USER_ADDRESS
)
1058 if (dbregs
->dr7
& (0x3<<2)) {
1059 /* dr1 is enabled */
1060 if (dbregs
->dr1
>= VM_MAX_USER_ADDRESS
)
1064 if (dbregs
->dr7
& (0x3<<4)) {
1065 /* dr2 is enabled */
1066 if (dbregs
->dr2
>= VM_MAX_USER_ADDRESS
)
1070 if (dbregs
->dr7
& (0x3<<6)) {
1071 /* dr3 is enabled */
1072 if (dbregs
->dr3
>= VM_MAX_USER_ADDRESS
)
1077 pcb
->pcb_dr0
= dbregs
->dr0
;
1078 pcb
->pcb_dr1
= dbregs
->dr1
;
1079 pcb
->pcb_dr2
= dbregs
->dr2
;
1080 pcb
->pcb_dr3
= dbregs
->dr3
;
1081 pcb
->pcb_dr6
= dbregs
->dr6
;
1082 pcb
->pcb_dr7
= dbregs
->dr7
;
1084 pcb
->pcb_flags
|= PCB_DBREGS
;
1092 * Return > 0 if a hardware breakpoint has been hit, and the
1093 * breakpoint was in user space. Return 0, otherwise.
1096 user_dbreg_trap(void)
1098 u_int32_t dr7
, dr6
; /* debug registers dr6 and dr7 */
1099 u_int32_t bp
; /* breakpoint bits extracted from dr6 */
1100 int nbp
; /* number of breakpoints that triggered */
1101 caddr_t addr
[4]; /* breakpoint addresses */
1105 if ((dr7
& 0x000000ff) == 0) {
1107 * all GE and LE bits in the dr7 register are zero,
1108 * thus the trap couldn't have been caused by the
1109 * hardware debug registers
1116 bp
= dr6
& 0x0000000f;
1120 * None of the breakpoint bits are set meaning this
1121 * trap was not caused by any of the debug registers
1127 * at least one of the breakpoints were hit, check to see
1128 * which ones and if any of them are user space addresses
1132 addr
[nbp
++] = (caddr_t
)rdr0();
1135 addr
[nbp
++] = (caddr_t
)rdr1();
1138 addr
[nbp
++] = (caddr_t
)rdr2();
1141 addr
[nbp
++] = (caddr_t
)rdr3();
1144 for (i
=0; i
<nbp
; i
++) {
1146 (caddr_t
)VM_MAX_USER_ADDRESS
) {
1148 * addr[i] is in user space
1155 * None of the breakpoints are in user space.
1165 Debugger(const char *msg
)
1167 kprintf("Debugger(\"%s\") called.\n", msg
);
1171 #include <sys/disklabel.h>
1174 * Determine the size of the transfer, and make sure it is
1175 * within the boundaries of the partition. Adjust transfer
1176 * if needed, and signal errors or early completion.
1178 * On success a new bio layer is pushed with the translated
1179 * block number, and returned.
1182 bounds_check_with_label(cdev_t dev
, struct bio
*bio
,
1183 struct disklabel
*lp
, int wlabel
)
1186 struct buf
*bp
= bio
->bio_buf
;
1187 struct partition
*p
= lp
->d_partitions
+ dkpart(dev
);
1188 int labelsect
= lp
->d_partitions
[0].p_offset
;
1189 int maxsz
= p
->p_size
,
1190 sz
= (bp
->b_bcount
+ DEV_BSIZE
- 1) >> DEV_BSHIFT
;
1191 daddr_t blkno
= (daddr_t
)(bio
->bio_offset
>> DEV_BSHIFT
);
1193 /* overwriting disk label ? */
1194 /* XXX should also protect bootstrap in first 8K */
1195 if (blkno
+ p
->p_offset
<= LABELSECTOR
+ labelsect
&&
1196 #if LABELSECTOR != 0
1197 blkno
+ p
->p_offset
+ sz
> LABELSECTOR
+ labelsect
&&
1199 bp
->b_cmd
!= BUF_CMD_READ
&& wlabel
== 0) {
1200 bp
->b_error
= EROFS
;
1204 #if defined(DOSBBSECTOR) && defined(notyet)
1205 /* overwriting master boot record? */
1206 if (blkno
+ p
->p_offset
<= DOSBBSECTOR
&&
1207 bp
->b_cmd
!= BUF_CMD_READ
&& wlabel
== 0) {
1208 bp
->b_error
= EROFS
;
1214 * Check for out of bounds, EOF, and EOF clipping.
1216 if (bio
->bio_offset
< 0)
1218 if (blkno
+ sz
> maxsz
) {
1220 * Past EOF or B_BNOCLIP flag was set, the request is bad.
1222 if (blkno
> maxsz
|| (bp
->b_flags
& B_BNOCLIP
))
1226 * If exactly on EOF just complete the I/O with no bytes
1227 * transfered. B_INVAL must be set to throw away the
1228 * contents of the buffer. Otherwise clip b_bcount.
1230 if (blkno
== maxsz
) {
1231 bp
->b_resid
= bp
->b_bcount
;
1232 bp
->b_flags
|= B_INVAL
;
1235 bp
->b_bcount
= (maxsz
- blkno
) << DEV_BSHIFT
;
1237 nbio
= push_bio(bio
);
1238 nbio
->bio_offset
= bio
->bio_offset
+ ((off_t
)p
->p_offset
<< DEV_BSHIFT
);
1242 * The caller is responsible for calling biodone() on the passed bio
1243 * when we return NULL.
1246 bp
->b_error
= EINVAL
;
1248 bp
->b_resid
= bp
->b_bcount
;
1249 bp
->b_flags
|= B_ERROR
| B_INVAL
;