4 * Copyright (c) 2006 The DragonFly Project. All rights reserved.
6 * This code is derived from software contributed to The DragonFly Project
7 * by Matthew Dillon <dillon@backplane.com>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #include <sys/param.h>
38 #include <sys/kernel.h>
39 #include <sys/systm.h>
40 #include <sys/sysproto.h>
41 #include <sys/kern_syscall.h>
43 #include <sys/thread.h>
45 #include <sys/malloc.h>
46 #include <sys/sysctl.h>
47 #include <sys/vkernel.h>
48 #include <sys/vmspace.h>
50 #include <vm/vm_extern.h>
53 #include <machine/vmparam.h>
55 #include <sys/sysref2.h>
56 #include <sys/mplock2.h>
58 static struct vmspace_entry
*vkernel_find_vmspace(struct vkernel_proc
*vkp
,
60 static void vmspace_entry_delete(struct vmspace_entry
*ve
,
61 struct vkernel_proc
*vkp
);
63 static MALLOC_DEFINE(M_VKERNEL
, "vkernel", "VKernel structures");
66 * vmspace_create (void *id, int type, void *data)
68 * Create a VMSPACE under the control of the caller with the specified id.
69 * An id of NULL cannot be used. The type and data fields must currently
72 * The vmspace starts out completely empty. Memory may be mapped into the
73 * VMSPACE with vmspace_mmap() and MAP_VPAGETABLE section(s) controlled
74 * with vmspace_mcontrol().
79 sys_vmspace_create(struct vmspace_create_args
*uap
)
81 struct vmspace_entry
*ve
;
82 struct vkernel_proc
*vkp
;
83 struct proc
*p
= curproc
;
86 if (vkernel_enable
== 0)
90 * Create a virtual kernel side-structure for the process if one
93 * Implement a simple resolution for SMP races.
95 if ((vkp
= p
->p_vkernel
) == NULL
) {
96 vkp
= kmalloc(sizeof(*vkp
), M_VKERNEL
, M_WAITOK
|M_ZERO
);
97 lwkt_gettoken(&proc_token
);
98 if (p
->p_vkernel
== NULL
) {
100 lwkt_token_init(&vkp
->token
, "vkernel");
104 kfree(vkp
, M_VKERNEL
);
107 lwkt_reltoken(&proc_token
);
113 * Create a new VMSPACE, disallow conflicting ids
115 ve
= kmalloc(sizeof(struct vmspace_entry
), M_VKERNEL
, M_WAITOK
|M_ZERO
);
116 ve
->vmspace
= vmspace_alloc(VM_MIN_USER_ADDRESS
, VM_MAX_USER_ADDRESS
);
118 pmap_pinit2(vmspace_pmap(ve
->vmspace
));
120 lwkt_gettoken(&vkp
->token
);
121 if (RB_INSERT(vmspace_rb_tree
, &vkp
->root
, ve
)) {
122 vmspace_free(ve
->vmspace
);
123 ve
->vmspace
= NULL
; /* safety */
124 kfree(ve
, M_VKERNEL
);
129 lwkt_reltoken(&vkp
->token
);
135 * Destroy a VMSPACE given its identifier.
140 sys_vmspace_destroy(struct vmspace_destroy_args
*uap
)
142 struct vkernel_proc
*vkp
;
143 struct vmspace_entry
*ve
;
147 if ((vkp
= curproc
->p_vkernel
) == NULL
) {
151 lwkt_gettoken(&vkp
->token
);
152 if ((ve
= vkernel_find_vmspace(vkp
, uap
->id
)) == NULL
) {
160 vmspace_entry_delete(ve
, vkp
);
163 lwkt_reltoken(&vkp
->token
);
170 * vmspace_ctl (void *id, int cmd, struct trapframe *tframe,
171 * struct vextframe *vframe);
173 * Transfer control to a VMSPACE. Control is returned after the specified
174 * number of microseconds or if a page fault, signal, trap, or system call
175 * occurs. The context is updated as appropriate.
180 sys_vmspace_ctl(struct vmspace_ctl_args
*uap
)
182 struct vkernel_proc
*vkp
;
183 struct vkernel_lwp
*vklp
;
184 struct vmspace_entry
*ve
;
190 lp
= curthread
->td_lwp
;
193 if ((vkp
= p
->p_vkernel
) == NULL
)
197 lwkt_gettoken(&vkp
->token
);
198 if ((ve
= vkernel_find_vmspace(vkp
, uap
->id
)) == NULL
) {
204 case VMSPACE_CTL_RUN
:
206 * Save the caller's register context, swap VM spaces, and
207 * install the passed register context. Return with
208 * EJUSTRETURN so the syscall code doesn't adjust the context.
210 atomic_add_int(&ve
->refs
, 1);
211 framesz
= sizeof(struct trapframe
);
212 if ((vklp
= lp
->lwp_vkernel
) == NULL
) {
213 vklp
= kmalloc(sizeof(*vklp
), M_VKERNEL
,
215 lp
->lwp_vkernel
= vklp
;
217 vklp
->user_trapframe
= uap
->tframe
;
218 vklp
->user_vextframe
= uap
->vframe
;
219 bcopy(uap
->sysmsg_frame
, &vklp
->save_trapframe
, framesz
);
220 bcopy(&curthread
->td_tls
, &vklp
->save_vextframe
.vx_tls
,
221 sizeof(vklp
->save_vextframe
.vx_tls
));
222 error
= copyin(uap
->tframe
, uap
->sysmsg_frame
, framesz
);
224 error
= copyin(&uap
->vframe
->vx_tls
,
226 sizeof(struct savetls
));
229 error
= cpu_sanitize_frame(uap
->sysmsg_frame
);
231 error
= cpu_sanitize_tls(&curthread
->td_tls
);
233 bcopy(&vklp
->save_trapframe
, uap
->sysmsg_frame
,
235 bcopy(&vklp
->save_vextframe
.vx_tls
, &curthread
->td_tls
,
236 sizeof(vklp
->save_vextframe
.vx_tls
));
238 atomic_subtract_int(&ve
->refs
, 1);
241 pmap_setlwpvm(lp
, ve
->vmspace
);
243 set_vkernel_fp(uap
->sysmsg_frame
);
252 lwkt_reltoken(&vkp
->token
);
258 * vmspace_mmap(id, addr, len, prot, flags, fd, offset)
260 * map memory within a VMSPACE. This function is just like a normal mmap()
261 * but operates on the vmspace's memory map. Most callers use this to create
262 * a MAP_VPAGETABLE mapping.
267 sys_vmspace_mmap(struct vmspace_mmap_args
*uap
)
269 struct vkernel_proc
*vkp
;
270 struct vmspace_entry
*ve
;
274 * We hold the vmspace token to serialize calls to vkernel_find_vmspace.
276 lwkt_gettoken(&vmspace_token
);
277 if ((vkp
= curproc
->p_vkernel
) == NULL
) {
283 * NOTE: kern_mmap() can block so we need to temporarily ref ve->refs.
285 lwkt_gettoken(&vkp
->token
);
286 if ((ve
= vkernel_find_vmspace(vkp
, uap
->id
)) != NULL
) {
287 atomic_add_int(&ve
->refs
, 1);
288 error
= kern_mmap(ve
->vmspace
, uap
->addr
, uap
->len
,
289 uap
->prot
, uap
->flags
,
290 uap
->fd
, uap
->offset
, &uap
->sysmsg_resultp
);
291 atomic_subtract_int(&ve
->refs
, 1);
295 lwkt_reltoken(&vkp
->token
);
297 lwkt_reltoken(&vmspace_token
);
302 * vmspace_munmap(id, addr, len)
304 * unmap memory within a VMSPACE.
309 sys_vmspace_munmap(struct vmspace_munmap_args
*uap
)
311 struct vkernel_proc
*vkp
;
312 struct vmspace_entry
*ve
;
315 vm_size_t size
, pageoff
;
320 if ((vkp
= curproc
->p_vkernel
) == NULL
) {
324 lwkt_gettoken(&vkp
->token
);
325 if ((ve
= vkernel_find_vmspace(vkp
, uap
->id
)) == NULL
) {
331 * NOTE: kern_munmap() can block so we need to temporarily
334 atomic_add_int(&ve
->refs
, 1);
337 * Copied from sys_munmap()
339 addr
= (vm_offset_t
)uap
->addr
;
342 pageoff
= (addr
& PAGE_MASK
);
345 size
= (vm_size_t
)round_page(size
);
346 if (size
< uap
->len
) { /* wrap */
350 tmpaddr
= addr
+ size
; /* workaround gcc4 opt */
351 if (tmpaddr
< addr
) { /* wrap */
360 if (VM_MAX_USER_ADDRESS
> 0 && tmpaddr
> VM_MAX_USER_ADDRESS
) {
364 if (VM_MIN_USER_ADDRESS
> 0 && addr
< VM_MIN_USER_ADDRESS
) {
368 map
= &ve
->vmspace
->vm_map
;
369 if (!vm_map_check_protection(map
, addr
, tmpaddr
, VM_PROT_NONE
, FALSE
)) {
373 vm_map_remove(map
, addr
, addr
+ size
);
376 atomic_subtract_int(&ve
->refs
, 1);
378 lwkt_reltoken(&vkp
->token
);
385 * vmspace_pread(id, buf, nbyte, flags, offset)
387 * Read data from a vmspace. The number of bytes read is returned or
388 * -1 if an unrecoverable error occured. If the number of bytes read is
389 * less then the request size, a page fault occured in the VMSPACE which
390 * the caller must resolve in order to proceed.
392 * (not implemented yet)
396 sys_vmspace_pread(struct vmspace_pread_args
*uap
)
398 struct vkernel_proc
*vkp
;
399 struct vmspace_entry
*ve
;
403 if ((vkp
= curproc
->p_vkernel
) == NULL
) {
407 lwkt_gettoken(&vkp
->token
);
408 if ((ve
= vkernel_find_vmspace(vkp
, uap
->id
)) == NULL
) {
414 lwkt_reltoken(&vkp
->token
);
421 * vmspace_pwrite(id, buf, nbyte, flags, offset)
423 * Write data to a vmspace. The number of bytes written is returned or
424 * -1 if an unrecoverable error occured. If the number of bytes written is
425 * less then the request size, a page fault occured in the VMSPACE which
426 * the caller must resolve in order to proceed.
428 * (not implemented yet)
432 sys_vmspace_pwrite(struct vmspace_pwrite_args
*uap
)
434 struct vkernel_proc
*vkp
;
435 struct vmspace_entry
*ve
;
439 if ((vkp
= curproc
->p_vkernel
) == NULL
) {
443 lwkt_gettoken(&vkp
->token
);
444 if ((ve
= vkernel_find_vmspace(vkp
, uap
->id
)) == NULL
) {
450 lwkt_reltoken(&vkp
->token
);
457 * vmspace_mcontrol(id, addr, len, behav, value)
459 * madvise/mcontrol support for a vmspace.
464 sys_vmspace_mcontrol(struct vmspace_mcontrol_args
*uap
)
466 struct vkernel_proc
*vkp
;
467 struct vmspace_entry
*ve
;
468 vm_offset_t start
, end
;
469 vm_offset_t tmpaddr
= (vm_offset_t
)uap
->addr
+ uap
->len
;
473 if ((vkp
= curproc
->p_vkernel
) == NULL
) {
477 lwkt_gettoken(&vkp
->token
);
478 if ((ve
= vkernel_find_vmspace(vkp
, uap
->id
)) == NULL
) {
484 * NOTE: kern_madvise() can block so we need to temporarily
487 atomic_add_int(&ve
->refs
, 1);
490 * This code is basically copied from sys_mcontrol()
492 if (uap
->behav
< 0 || uap
->behav
> MADV_CONTROL_END
) {
497 if (tmpaddr
< (vm_offset_t
)uap
->addr
) {
501 if (VM_MAX_USER_ADDRESS
> 0 && tmpaddr
> VM_MAX_USER_ADDRESS
) {
505 if (VM_MIN_USER_ADDRESS
> 0 && uap
->addr
< VM_MIN_USER_ADDRESS
) {
510 start
= trunc_page((vm_offset_t
) uap
->addr
);
511 end
= round_page(tmpaddr
);
513 error
= vm_map_madvise(&ve
->vmspace
->vm_map
, start
, end
,
514 uap
->behav
, uap
->value
);
516 atomic_subtract_int(&ve
->refs
, 1);
518 lwkt_reltoken(&vkp
->token
);
525 * Red black tree functions
527 static int rb_vmspace_compare(struct vmspace_entry
*, struct vmspace_entry
*);
528 RB_GENERATE(vmspace_rb_tree
, vmspace_entry
, rb_entry
, rb_vmspace_compare
);
531 * a->start is address, and the only field has to be initialized.
532 * The caller must hold vkp->token.
534 * The caller must hold vkp->token.
537 rb_vmspace_compare(struct vmspace_entry
*a
, struct vmspace_entry
*b
)
539 if ((char *)a
->id
< (char *)b
->id
)
541 else if ((char *)a
->id
> (char *)b
->id
)
547 * The caller must hold vkp->token.
551 rb_vmspace_delete(struct vmspace_entry
*ve
, void *data
)
553 struct vkernel_proc
*vkp
= data
;
555 KKASSERT(ve
->refs
== 0);
556 vmspace_entry_delete(ve
, vkp
);
561 * Remove a vmspace_entry from the RB tree and destroy it. We have to clean
562 * up the pmap, the vm_map, then destroy the vmspace.
564 * This function must remove the ve immediately before it might potentially
567 * The caller must hold vkp->token.
571 vmspace_entry_delete(struct vmspace_entry
*ve
, struct vkernel_proc
*vkp
)
573 RB_REMOVE(vmspace_rb_tree
, &vkp
->root
, ve
);
575 pmap_remove_pages(vmspace_pmap(ve
->vmspace
),
576 VM_MIN_USER_ADDRESS
, VM_MAX_USER_ADDRESS
);
577 vm_map_remove(&ve
->vmspace
->vm_map
,
578 VM_MIN_USER_ADDRESS
, VM_MAX_USER_ADDRESS
);
579 vmspace_free(ve
->vmspace
);
580 ve
->vmspace
= NULL
; /* safety */
581 kfree(ve
, M_VKERNEL
);
585 * Locate the ve for (id), return the ve or NULL. If found this function
586 * will bump ve->refs which prevents the ve from being immediately destroyed
587 * (but it can still be removed).
589 * The caller must hold vkp->token.
592 struct vmspace_entry
*
593 vkernel_find_vmspace(struct vkernel_proc
*vkp
, void *id
)
595 struct vmspace_entry
*ve
;
596 struct vmspace_entry key
;
599 ve
= RB_FIND(vmspace_rb_tree
, &vkp
->root
, &key
);
604 * Manage vkernel refs, used by the kernel when fork()ing or exit()ing
610 vkernel_inherit(struct proc
*p1
, struct proc
*p2
)
612 struct vkernel_proc
*vkp
;
615 KKASSERT(vkp
->refs
> 0);
616 atomic_add_int(&vkp
->refs
, 1);
624 vkernel_exit(struct proc
*p
)
626 struct vkernel_proc
*vkp
;
632 * Restore the original VM context if we are killed while running
635 * This isn't supposed to happen. What is supposed to happen is
636 * that the process should enter vkernel_trap() before the handling
639 RB_FOREACH(lp
, lwp_rb_tree
, &p
->p_lwp_tree
) {
640 vkernel_lwp_exit(lp
);
644 * Dereference the common area
647 KKASSERT(vkp
->refs
> 0);
649 if (atomic_fetchadd_int(&vkp
->refs
, -1) == 1) {
650 lwkt_gettoken(&vkp
->token
);
651 RB_SCAN(vmspace_rb_tree
, &vkp
->root
, NULL
,
652 rb_vmspace_delete
, vkp
);
653 lwkt_reltoken(&vkp
->token
);
654 kfree(vkp
, M_VKERNEL
);
662 vkernel_lwp_exit(struct lwp
*lp
)
664 struct vkernel_lwp
*vklp
;
665 struct vmspace_entry
*ve
;
667 if ((vklp
= lp
->lwp_vkernel
) != NULL
) {
668 if ((ve
= vklp
->ve
) != NULL
) {
669 kprintf("Warning, pid %d killed with "
670 "active VC!\n", lp
->lwp_proc
->p_pid
);
671 pmap_setlwpvm(lp
, lp
->lwp_proc
->p_vmspace
);
673 KKASSERT(ve
->refs
> 0);
674 atomic_subtract_int(&ve
->refs
, 1);
676 lp
->lwp_vkernel
= NULL
;
677 kfree(vklp
, M_VKERNEL
);
682 * A VM space under virtual kernel control trapped out or made a system call
683 * or otherwise needs to return control to the virtual kernel context.
688 vkernel_trap(struct lwp
*lp
, struct trapframe
*frame
)
690 struct proc
*p
= lp
->lwp_proc
;
691 struct vmspace_entry
*ve
;
692 struct vkernel_lwp
*vklp
;
696 * Which vmspace entry was running?
698 vklp
= lp
->lwp_vkernel
;
701 KKASSERT(ve
!= NULL
);
704 * Switch the LWP vmspace back to the virtual kernel's VM space.
707 pmap_setlwpvm(lp
, p
->p_vmspace
);
708 KKASSERT(ve
->refs
> 0);
709 atomic_subtract_int(&ve
->refs
, 1);
710 /* ve is invalid once we kill our ref */
713 * Copy the emulated process frame to the virtual kernel process.
714 * The emulated process cannot change TLS descriptors so don't
715 * bother saving them, we already have a copy.
717 * Restore the virtual kernel's saved context so the virtual kernel
718 * process can resume.
720 error
= copyout(frame
, vklp
->user_trapframe
, sizeof(*frame
));
721 bcopy(&vklp
->save_trapframe
, frame
, sizeof(*frame
));
722 bcopy(&vklp
->save_vextframe
.vx_tls
, &curthread
->td_tls
,
723 sizeof(vklp
->save_vextframe
.vx_tls
));
725 cpu_vkernel_trap(frame
, error
);