2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
6 * Copyright (c) 1994 David Greenman
10 * This code is derived from software contributed to Berkeley by
11 * The Mach Operating System project at Carnegie-Mellon University.
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 * must display the following acknowledgement:
23 * This product includes software developed by the University of
24 * California, Berkeley and its contributors.
25 * 4. Neither the name of the University nor the names of its contributors
26 * may be used to endorse or promote products derived from this software
27 * without specific prior written permission.
29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41 * from: @(#)vm_fault.c 8.4 (Berkeley) 1/12/94
44 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
45 * All rights reserved.
47 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
49 * Permission to use, copy, modify and distribute this software and
50 * its documentation is hereby granted, provided that both the copyright
51 * notice and this permission notice appear in all copies of the
52 * software, derivative works or modified versions, and any portions
53 * thereof, and that both notices appear in supporting documentation.
55 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
59 * Carnegie Mellon requests users of this software to return to
61 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
62 * School of Computer Science
63 * Carnegie Mellon University
64 * Pittsburgh PA 15213-3890
66 * any improvements or extensions that they make and grant Carnegie the
67 * rights to redistribute these changes.
69 * $FreeBSD: src/sys/vm/vm_fault.c,v 1.108.2.8 2002/02/26 05:49:27 silby Exp $
70 * $DragonFly: src/sys/vm/vm_fault.c,v 1.42 2007/06/07 23:00:39 dillon Exp $
74 * Page fault handling module.
77 #include <sys/param.h>
78 #include <sys/systm.h>
79 #include <sys/kernel.h>
81 #include <sys/vnode.h>
82 #include <sys/resourcevar.h>
83 #include <sys/vmmeter.h>
84 #include <sys/vkernel.h>
85 #include <sys/sfbuf.h>
89 #include <vm/vm_param.h>
91 #include <vm/vm_map.h>
92 #include <vm/vm_object.h>
93 #include <vm/vm_page.h>
94 #include <vm/vm_pageout.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_pager.h>
97 #include <vm/vnode_pager.h>
98 #include <vm/vm_extern.h>
100 #include <sys/thread2.h>
101 #include <vm/vm_page2.h>
103 #define VM_FAULT_READ_AHEAD 8
104 #define VM_FAULT_READ_BEHIND 7
105 #define VM_FAULT_READ (VM_FAULT_READ_AHEAD+VM_FAULT_READ_BEHIND+1)
113 vm_object_t first_object
;
114 vm_prot_t first_prot
;
116 vm_map_entry_t entry
;
117 int lookup_still_valid
;
126 static int vm_fault_object(struct faultstate
*, vm_pindex_t
, vm_prot_t
);
127 static int vm_fault_vpagetable(struct faultstate
*, vm_pindex_t
*, vpte_t
, int);
128 static int vm_fault_additional_pages (vm_page_t
, int, int, vm_page_t
*, int *);
129 static int vm_fault_ratelimit(struct vmspace
*);
132 release_page(struct faultstate
*fs
)
134 vm_page_wakeup(fs
->m
);
135 vm_page_deactivate(fs
->m
);
140 unlock_map(struct faultstate
*fs
)
142 if (fs
->lookup_still_valid
&& fs
->map
) {
143 vm_map_lookup_done(fs
->map
, fs
->entry
, 0);
144 fs
->lookup_still_valid
= FALSE
;
149 * Clean up after a successful call to vm_fault_object() so another call
150 * to vm_fault_object() can be made.
153 _cleanup_successful_fault(struct faultstate
*fs
, int relock
)
155 if (fs
->object
!= fs
->first_object
) {
156 vm_page_free(fs
->first_m
);
157 vm_object_pip_wakeup(fs
->object
);
160 fs
->object
= fs
->first_object
;
161 if (relock
&& fs
->lookup_still_valid
== FALSE
) {
163 vm_map_lock_read(fs
->map
);
164 fs
->lookup_still_valid
= TRUE
;
169 _unlock_things(struct faultstate
*fs
, int dealloc
)
171 vm_object_pip_wakeup(fs
->first_object
);
172 _cleanup_successful_fault(fs
, 0);
174 vm_object_deallocate(fs
->first_object
);
177 if (fs
->vp
!= NULL
) {
183 #define unlock_things(fs) _unlock_things(fs, 0)
184 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
185 #define cleanup_successful_fault(fs) _cleanup_successful_fault(fs, 1)
190 * Determine if the pager for the current object *might* contain the page.
192 * We only need to try the pager if this is not a default object (default
193 * objects are zero-fill and have no real pager), and if we are not taking
194 * a wiring fault or if the FS entry is wired.
196 #define TRYPAGER(fs) \
197 (fs->object->type != OBJT_DEFAULT && \
198 (((fs->fault_flags & VM_FAULT_WIRE_MASK) == 0) || fs->wired))
203 * Handle a page fault occuring at the given address, requiring the given
204 * permissions, in the map specified. If successful, the page is inserted
205 * into the associated physical map.
207 * NOTE: The given address should be truncated to the proper page address.
209 * KERN_SUCCESS is returned if the page fault is handled; otherwise,
210 * a standard error specifying why the fault is fatal is returned.
212 * The map in question must be referenced, and remains so.
213 * The caller may hold no locks.
216 vm_fault(vm_map_t map
, vm_offset_t vaddr
, vm_prot_t fault_type
, int fault_flags
)
219 vm_pindex_t first_pindex
;
220 struct faultstate fs
;
222 mycpu
->gd_cnt
.v_vm_faults
++;
226 fs
.fault_flags
= fault_flags
;
230 * Find the vm_map_entry representing the backing store and resolve
231 * the top level object and page index. This may have the side
232 * effect of executing a copy-on-write on the map entry and/or
233 * creating a shadow object, but will not COW any actual VM pages.
235 * On success fs.map is left read-locked and various other fields
236 * are initialized but not otherwise referenced or locked.
238 * NOTE! vm_map_lookup will try to upgrade the fault_type to
239 * VM_FAULT_WRITE if the map entry is a virtual page table and also
240 * writable, so we can set the 'A'accessed bit in the virtual page
244 result
= vm_map_lookup(&fs
.map
, vaddr
, fault_type
,
245 &fs
.entry
, &fs
.first_object
,
246 &first_pindex
, &fs
.first_prot
, &fs
.wired
);
249 * If the lookup failed or the map protections are incompatible,
250 * the fault generally fails. However, if the caller is trying
251 * to do a user wiring we have more work to do.
253 if (result
!= KERN_SUCCESS
) {
254 if (result
!= KERN_PROTECTION_FAILURE
)
256 if ((fs
.fault_flags
& VM_FAULT_WIRE_MASK
) != VM_FAULT_USER_WIRE
)
260 * If we are user-wiring a r/w segment, and it is COW, then
261 * we need to do the COW operation. Note that we don't
262 * currently COW RO sections now, because it is NOT desirable
263 * to COW .text. We simply keep .text from ever being COW'ed
264 * and take the heat that one cannot debug wired .text sections.
266 result
= vm_map_lookup(&fs
.map
, vaddr
,
267 VM_PROT_READ
|VM_PROT_WRITE
|
268 VM_PROT_OVERRIDE_WRITE
,
269 &fs
.entry
, &fs
.first_object
,
270 &first_pindex
, &fs
.first_prot
,
272 if (result
!= KERN_SUCCESS
)
276 * If we don't COW now, on a user wire, the user will never
277 * be able to write to the mapping. If we don't make this
278 * restriction, the bookkeeping would be nearly impossible.
280 if ((fs
.entry
->protection
& VM_PROT_WRITE
) == 0)
281 fs
.entry
->max_protection
&= ~VM_PROT_WRITE
;
285 * fs.map is read-locked
287 * Misc checks. Save the map generation number to detect races.
289 fs
.map_generation
= fs
.map
->timestamp
;
291 if (fs
.entry
->eflags
& MAP_ENTRY_NOFAULT
) {
292 panic("vm_fault: fault on nofault entry, addr: %lx",
297 * A system map entry may return a NULL object. No object means
298 * no pager means an unrecoverable kernel fault.
300 if (fs
.first_object
== NULL
) {
301 panic("vm_fault: unrecoverable fault at %p in entry %p",
302 (void *)vaddr
, fs
.entry
);
306 * Make a reference to this object to prevent its disposal while we
307 * are messing with it. Once we have the reference, the map is free
308 * to be diddled. Since objects reference their shadows (and copies),
309 * they will stay around as well.
311 * Bump the paging-in-progress count to prevent size changes (e.g.
312 * truncation operations) during I/O. This must be done after
313 * obtaining the vnode lock in order to avoid possible deadlocks.
315 vm_object_reference(fs
.first_object
);
316 fs
.vp
= vnode_pager_lock(fs
.first_object
);
317 vm_object_pip_add(fs
.first_object
, 1);
319 fs
.lookup_still_valid
= TRUE
;
321 fs
.object
= fs
.first_object
; /* so unlock_and_deallocate works */
324 * If the entry is wired we cannot change the page protection.
327 fault_type
= fs
.first_prot
;
330 * The page we want is at (first_object, first_pindex), but if the
331 * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
332 * page table to figure out the actual pindex.
334 * NOTE! DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
337 if (fs
.entry
->maptype
== VM_MAPTYPE_VPAGETABLE
) {
338 result
= vm_fault_vpagetable(&fs
, &first_pindex
,
339 fs
.entry
->aux
.master_pde
,
341 if (result
== KERN_TRY_AGAIN
)
343 if (result
!= KERN_SUCCESS
)
348 * Now we have the actual (object, pindex), fault in the page. If
349 * vm_fault_object() fails it will unlock and deallocate the FS
350 * data. If it succeeds everything remains locked and fs->object
351 * will have an additinal PIP count if it is not equal to
354 * vm_fault_object will set fs->prot for the pmap operation. It is
355 * allowed to set VM_PROT_WRITE if fault_type == VM_PROT_READ if the
356 * page can be safely written. However, it will force a read-only
357 * mapping for a read fault if the memory is managed by a virtual
360 result
= vm_fault_object(&fs
, first_pindex
, fault_type
);
362 if (result
== KERN_TRY_AGAIN
)
364 if (result
!= KERN_SUCCESS
)
368 * On success vm_fault_object() does not unlock or deallocate, and fs.m
369 * will contain a busied page.
371 * Enter the page into the pmap and do pmap-related adjustments.
374 pmap_enter(fs
.map
->pmap
, vaddr
, fs
.m
, fs
.prot
, fs
.wired
);
376 if (((fs
.fault_flags
& VM_FAULT_WIRE_MASK
) == 0) && (fs
.wired
== 0)) {
377 pmap_prefault(fs
.map
->pmap
, vaddr
, fs
.entry
);
380 vm_page_flag_clear(fs
.m
, PG_ZERO
);
381 vm_page_flag_set(fs
.m
, PG_MAPPED
|PG_REFERENCED
);
384 * If the page is not wired down, then put it where the pageout daemon
387 if (fs
.fault_flags
& VM_FAULT_WIRE_MASK
) {
391 vm_page_unwire(fs
.m
, 1);
393 vm_page_activate(fs
.m
);
396 if (curthread
->td_lwp
) {
398 curthread
->td_lwp
->lwp_ru
.ru_majflt
++;
400 curthread
->td_lwp
->lwp_ru
.ru_minflt
++;
405 * Unlock everything, and return
407 vm_page_wakeup(fs
.m
);
408 vm_object_deallocate(fs
.first_object
);
410 return (KERN_SUCCESS
);
414 * Fault in the specified virtual address in the current process map,
415 * returning a held VM page or NULL. See vm_fault_page() for more
419 vm_fault_page_quick(vm_offset_t va
, vm_prot_t fault_type
, int *errorp
)
423 m
= vm_fault_page(&curproc
->p_vmspace
->vm_map
, va
,
424 fault_type
, VM_FAULT_NORMAL
, errorp
);
429 * Fault in the specified virtual address in the specified map, doing all
430 * necessary manipulation of the object store and all necessary I/O. Return
431 * a held VM page or NULL, and set *errorp. The related pmap is not
434 * The returned page will be properly dirtied if VM_PROT_WRITE was specified,
435 * and marked PG_REFERENCED as well.
438 vm_fault_page(vm_map_t map
, vm_offset_t vaddr
, vm_prot_t fault_type
,
439 int fault_flags
, int *errorp
)
442 vm_pindex_t first_pindex
;
443 struct faultstate fs
;
445 mycpu
->gd_cnt
.v_vm_faults
++;
449 fs
.fault_flags
= fault_flags
;
450 KKASSERT((fault_flags
& VM_FAULT_WIRE_MASK
) == 0);
454 * Find the vm_map_entry representing the backing store and resolve
455 * the top level object and page index. This may have the side
456 * effect of executing a copy-on-write on the map entry and/or
457 * creating a shadow object, but will not COW any actual VM pages.
459 * On success fs.map is left read-locked and various other fields
460 * are initialized but not otherwise referenced or locked.
462 * NOTE! vm_map_lookup will upgrade the fault_type to VM_FAULT_WRITE
463 * if the map entry is a virtual page table and also writable,
464 * so we can set the 'A'accessed bit in the virtual page table entry.
467 result
= vm_map_lookup(&fs
.map
, vaddr
, fault_type
,
468 &fs
.entry
, &fs
.first_object
,
469 &first_pindex
, &fs
.first_prot
, &fs
.wired
);
471 if (result
!= KERN_SUCCESS
) {
477 * fs.map is read-locked
479 * Misc checks. Save the map generation number to detect races.
481 fs
.map_generation
= fs
.map
->timestamp
;
483 if (fs
.entry
->eflags
& MAP_ENTRY_NOFAULT
) {
484 panic("vm_fault: fault on nofault entry, addr: %lx",
489 * A system map entry may return a NULL object. No object means
490 * no pager means an unrecoverable kernel fault.
492 if (fs
.first_object
== NULL
) {
493 panic("vm_fault: unrecoverable fault at %p in entry %p",
494 (void *)vaddr
, fs
.entry
);
498 * Make a reference to this object to prevent its disposal while we
499 * are messing with it. Once we have the reference, the map is free
500 * to be diddled. Since objects reference their shadows (and copies),
501 * they will stay around as well.
503 * Bump the paging-in-progress count to prevent size changes (e.g.
504 * truncation operations) during I/O. This must be done after
505 * obtaining the vnode lock in order to avoid possible deadlocks.
507 vm_object_reference(fs
.first_object
);
508 fs
.vp
= vnode_pager_lock(fs
.first_object
);
509 vm_object_pip_add(fs
.first_object
, 1);
511 fs
.lookup_still_valid
= TRUE
;
513 fs
.object
= fs
.first_object
; /* so unlock_and_deallocate works */
516 * If the entry is wired we cannot change the page protection.
519 fault_type
= fs
.first_prot
;
522 * The page we want is at (first_object, first_pindex), but if the
523 * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
524 * page table to figure out the actual pindex.
526 * NOTE! DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
529 if (fs
.entry
->maptype
== VM_MAPTYPE_VPAGETABLE
) {
530 result
= vm_fault_vpagetable(&fs
, &first_pindex
,
531 fs
.entry
->aux
.master_pde
,
533 if (result
== KERN_TRY_AGAIN
)
535 if (result
!= KERN_SUCCESS
) {
542 * Now we have the actual (object, pindex), fault in the page. If
543 * vm_fault_object() fails it will unlock and deallocate the FS
544 * data. If it succeeds everything remains locked and fs->object
545 * will have an additinal PIP count if it is not equal to
548 result
= vm_fault_object(&fs
, first_pindex
, fault_type
);
550 if (result
== KERN_TRY_AGAIN
)
552 if (result
!= KERN_SUCCESS
) {
558 * On success vm_fault_object() does not unlock or deallocate, and fs.m
559 * will contain a busied page.
564 * Return a held page. We are not doing any pmap manipulation so do
565 * not set PG_MAPPED. However, adjust the page flags according to
566 * the fault type because the caller may not use a managed pmapping
567 * (so we don't want to lose the fact that the page will be dirtied
568 * if a write fault was specified).
571 vm_page_flag_clear(fs
.m
, PG_ZERO
);
572 if (fault_type
& VM_PROT_WRITE
)
576 * Update the pmap. We really only have to do this if a COW
577 * occured to replace the read-only page with the new page. For
578 * now just do it unconditionally. XXX
580 pmap_enter(fs
.map
->pmap
, vaddr
, fs
.m
, fs
.prot
, fs
.wired
);
581 vm_page_flag_set(fs
.m
, PG_REFERENCED
|PG_MAPPED
);
584 * Unbusy the page by activating it. It remains held and will not
587 vm_page_activate(fs
.m
);
589 if (curthread
->td_lwp
) {
591 curthread
->td_lwp
->lwp_ru
.ru_majflt
++;
593 curthread
->td_lwp
->lwp_ru
.ru_minflt
++;
598 * Unlock everything, and return the held page.
600 vm_page_wakeup(fs
.m
);
601 vm_object_deallocate(fs
.first_object
);
608 * Fault in the specified
611 vm_fault_object_page(vm_object_t object
, vm_ooffset_t offset
,
612 vm_prot_t fault_type
, int fault_flags
, int *errorp
)
615 vm_pindex_t first_pindex
;
616 struct faultstate fs
;
617 struct vm_map_entry entry
;
619 bzero(&entry
, sizeof(entry
));
620 entry
.object
.vm_object
= object
;
621 entry
.maptype
= VM_MAPTYPE_NORMAL
;
622 entry
.protection
= entry
.max_protection
= fault_type
;
626 fs
.fault_flags
= fault_flags
;
628 KKASSERT((fault_flags
& VM_FAULT_WIRE_MASK
) == 0);
632 fs
.first_object
= object
;
633 first_pindex
= OFF_TO_IDX(offset
);
635 fs
.first_prot
= fault_type
;
637 /*fs.map_generation = 0; unused */
640 * Make a reference to this object to prevent its disposal while we
641 * are messing with it. Once we have the reference, the map is free
642 * to be diddled. Since objects reference their shadows (and copies),
643 * they will stay around as well.
645 * Bump the paging-in-progress count to prevent size changes (e.g.
646 * truncation operations) during I/O. This must be done after
647 * obtaining the vnode lock in order to avoid possible deadlocks.
649 vm_object_reference(fs
.first_object
);
650 fs
.vp
= vnode_pager_lock(fs
.first_object
);
651 vm_object_pip_add(fs
.first_object
, 1);
653 fs
.lookup_still_valid
= TRUE
;
655 fs
.object
= fs
.first_object
; /* so unlock_and_deallocate works */
658 /* XXX future - ability to operate on VM object using vpagetable */
659 if (fs
.entry
->maptype
== VM_MAPTYPE_VPAGETABLE
) {
660 result
= vm_fault_vpagetable(&fs
, &first_pindex
,
661 fs
.entry
->aux
.master_pde
,
663 if (result
== KERN_TRY_AGAIN
)
665 if (result
!= KERN_SUCCESS
) {
673 * Now we have the actual (object, pindex), fault in the page. If
674 * vm_fault_object() fails it will unlock and deallocate the FS
675 * data. If it succeeds everything remains locked and fs->object
676 * will have an additinal PIP count if it is not equal to
679 result
= vm_fault_object(&fs
, first_pindex
, fault_type
);
681 if (result
== KERN_TRY_AGAIN
)
683 if (result
!= KERN_SUCCESS
) {
689 * On success vm_fault_object() does not unlock or deallocate, and fs.m
690 * will contain a busied page.
695 * Return a held page. We are not doing any pmap manipulation so do
696 * not set PG_MAPPED. However, adjust the page flags according to
697 * the fault type because the caller may not use a managed pmapping
698 * (so we don't want to lose the fact that the page will be dirtied
699 * if a write fault was specified).
702 vm_page_flag_clear(fs
.m
, PG_ZERO
);
703 if (fault_type
& VM_PROT_WRITE
)
707 * Indicate that the page was accessed.
709 vm_page_flag_set(fs
.m
, PG_REFERENCED
);
712 * Unbusy the page by activating it. It remains held and will not
715 vm_page_activate(fs
.m
);
717 if (curthread
->td_lwp
) {
719 mycpu
->gd_cnt
.v_vm_faults
++;
720 curthread
->td_lwp
->lwp_ru
.ru_majflt
++;
722 curthread
->td_lwp
->lwp_ru
.ru_minflt
++;
727 * Unlock everything, and return the held page.
729 vm_page_wakeup(fs
.m
);
730 vm_object_deallocate(fs
.first_object
);
737 * Translate the virtual page number (first_pindex) that is relative
738 * to the address space into a logical page number that is relative to the
739 * backing object. Use the virtual page table pointed to by (vpte).
741 * This implements an N-level page table. Any level can terminate the
742 * scan by setting VPTE_PS. A linear mapping is accomplished by setting
743 * VPTE_PS in the master page directory entry set via mcontrol(MADV_SETMAP).
747 vm_fault_vpagetable(struct faultstate
*fs
, vm_pindex_t
*pindex
,
748 vpte_t vpte
, int fault_type
)
751 int vshift
= 32 - PAGE_SHIFT
; /* page index bits remaining */
752 int result
= KERN_SUCCESS
;
757 * We cannot proceed if the vpte is not valid, not readable
758 * for a read fault, or not writable for a write fault.
760 if ((vpte
& VPTE_V
) == 0) {
761 unlock_and_deallocate(fs
);
762 return (KERN_FAILURE
);
764 if ((fault_type
& VM_PROT_READ
) && (vpte
& VPTE_R
) == 0) {
765 unlock_and_deallocate(fs
);
766 return (KERN_FAILURE
);
768 if ((fault_type
& VM_PROT_WRITE
) && (vpte
& VPTE_W
) == 0) {
769 unlock_and_deallocate(fs
);
770 return (KERN_FAILURE
);
772 if ((vpte
& VPTE_PS
) || vshift
== 0)
774 KKASSERT(vshift
>= VPTE_PAGE_BITS
);
777 * Get the page table page. Nominally we only read the page
778 * table, but since we are actively setting VPTE_M and VPTE_A,
779 * tell vm_fault_object() that we are writing it.
781 * There is currently no real need to optimize this.
783 result
= vm_fault_object(fs
, vpte
>> PAGE_SHIFT
,
784 VM_PROT_READ
|VM_PROT_WRITE
);
785 if (result
!= KERN_SUCCESS
)
789 * Process the returned fs.m and look up the page table
790 * entry in the page table page.
792 vshift
-= VPTE_PAGE_BITS
;
793 sf
= sf_buf_alloc(fs
->m
, SFB_CPUPRIVATE
);
794 ptep
= ((vpte_t
*)sf_buf_kva(sf
) +
795 ((*pindex
>> vshift
) & VPTE_PAGE_MASK
));
799 * Page table write-back. If the vpte is valid for the
800 * requested operation, do a write-back to the page table.
802 * XXX VPTE_M is not set properly for page directory pages.
803 * It doesn't get set in the page directory if the page table
804 * is modified during a read access.
806 if ((fault_type
& VM_PROT_WRITE
) && (vpte
& VPTE_V
) &&
808 if ((vpte
& (VPTE_M
|VPTE_A
)) != (VPTE_M
|VPTE_A
)) {
809 atomic_set_int(ptep
, VPTE_M
|VPTE_A
);
810 vm_page_dirty(fs
->m
);
813 if ((fault_type
& VM_PROT_READ
) && (vpte
& VPTE_V
) &&
815 if ((vpte
& VPTE_A
) == 0) {
816 atomic_set_int(ptep
, VPTE_A
);
817 vm_page_dirty(fs
->m
);
821 vm_page_flag_set(fs
->m
, PG_REFERENCED
);
822 vm_page_activate(fs
->m
);
823 vm_page_wakeup(fs
->m
);
824 cleanup_successful_fault(fs
);
827 * Combine remaining address bits with the vpte.
829 *pindex
= (vpte
>> PAGE_SHIFT
) +
830 (*pindex
& ((1 << vshift
) - 1));
831 return (KERN_SUCCESS
);
836 * Do all operations required to fault-in (fs.first_object, pindex). Run
837 * through the shadow chain as necessary and do required COW or virtual
838 * copy operations. The caller has already fully resolved the vm_map_entry
839 * and, if appropriate, has created a copy-on-write layer. All we need to
840 * do is iterate the object chain.
842 * On failure (fs) is unlocked and deallocated and the caller may return or
843 * retry depending on the failure code. On success (fs) is NOT unlocked or
844 * deallocated, fs.m will contained a resolved, busied page, and fs.object
845 * will have an additional PIP count if it is not equal to fs.first_object.
849 vm_fault_object(struct faultstate
*fs
,
850 vm_pindex_t first_pindex
, vm_prot_t fault_type
)
852 vm_object_t next_object
;
853 vm_page_t marray
[VM_FAULT_READ
];
857 fs
->prot
= fs
->first_prot
;
858 fs
->object
= fs
->first_object
;
859 pindex
= first_pindex
;
862 * If a read fault occurs we try to make the page writable if
863 * possible. There are three cases where we cannot make the
864 * page mapping writable:
866 * (1) The mapping is read-only or the VM object is read-only,
867 * fs->prot above will simply not have VM_PROT_WRITE set.
869 * (2) If the mapping is a virtual page table we need to be able
870 * to detect writes so we can set VPTE_M in the virtual page
873 * (3) If the VM page is read-only or copy-on-write, upgrading would
874 * just result in an unnecessary COW fault.
876 * VM_PROT_VPAGED is set if faulting via a virtual page table and
877 * causes adjustments to the 'M'odify bit to also turn off write
878 * access to force a re-fault.
880 if (fs
->entry
->maptype
== VM_MAPTYPE_VPAGETABLE
) {
881 if ((fault_type
& VM_PROT_WRITE
) == 0)
882 fs
->prot
&= ~VM_PROT_WRITE
;
887 * If the object is dead, we stop here
889 if (fs
->object
->flags
& OBJ_DEAD
) {
890 unlock_and_deallocate(fs
);
891 return (KERN_PROTECTION_FAILURE
);
895 * See if page is resident. spl protection is required
896 * to avoid an interrupt unbusy/free race against our
897 * lookup. We must hold the protection through a page
898 * allocation or busy.
901 fs
->m
= vm_page_lookup(fs
->object
, pindex
);
905 * Wait/Retry if the page is busy. We have to do this
906 * if the page is busy via either PG_BUSY or
907 * vm_page_t->busy because the vm_pager may be using
908 * vm_page_t->busy for pageouts ( and even pageins if
909 * it is the vnode pager ), and we could end up trying
910 * to pagein and pageout the same page simultaneously.
912 * We can theoretically allow the busy case on a read
913 * fault if the page is marked valid, but since such
914 * pages are typically already pmap'd, putting that
915 * special case in might be more effort then it is
916 * worth. We cannot under any circumstances mess
917 * around with a vm_page_t->busy page except, perhaps,
920 if ((fs
->m
->flags
& PG_BUSY
) || fs
->m
->busy
) {
922 vm_page_sleep_busy(fs
->m
, TRUE
, "vmpfw");
923 mycpu
->gd_cnt
.v_intrans
++;
924 vm_object_deallocate(fs
->first_object
);
926 return (KERN_TRY_AGAIN
);
930 * If reactivating a page from PQ_CACHE we may have
933 queue
= fs
->m
->queue
;
934 vm_page_unqueue_nowakeup(fs
->m
);
936 if ((queue
- fs
->m
->pc
) == PQ_CACHE
&&
937 vm_page_count_severe()) {
938 vm_page_activate(fs
->m
);
939 unlock_and_deallocate(fs
);
942 return (KERN_TRY_AGAIN
);
946 * Mark page busy for other processes, and the
947 * pagedaemon. If it still isn't completely valid
948 * (readable), jump to readrest, else we found the
949 * page and can return.
951 * We can release the spl once we have marked the
957 if (((fs
->m
->valid
& VM_PAGE_BITS_ALL
) != VM_PAGE_BITS_ALL
) &&
958 fs
->m
->object
!= &kernel_object
) {
961 break; /* break to PAGE HAS BEEN FOUND */
965 * Page is not resident, If this is the search termination
966 * or the pager might contain the page, allocate a new page.
968 * NOTE: We are still in a critical section.
970 if (TRYPAGER(fs
) || fs
->object
== fs
->first_object
) {
972 * If the page is beyond the object size we fail
974 if (pindex
>= fs
->object
->size
) {
976 unlock_and_deallocate(fs
);
977 return (KERN_PROTECTION_FAILURE
);
983 if (fs
->didlimit
== 0 && curproc
!= NULL
) {
986 limticks
= vm_fault_ratelimit(curproc
->p_vmspace
);
989 unlock_and_deallocate(fs
);
990 tsleep(curproc
, 0, "vmrate", limticks
);
992 return (KERN_TRY_AGAIN
);
997 * Allocate a new page for this object/offset pair.
1000 if (!vm_page_count_severe()) {
1001 fs
->m
= vm_page_alloc(fs
->object
, pindex
,
1002 (fs
->vp
|| fs
->object
->backing_object
) ? VM_ALLOC_NORMAL
: VM_ALLOC_NORMAL
| VM_ALLOC_ZERO
);
1004 if (fs
->m
== NULL
) {
1006 unlock_and_deallocate(fs
);
1008 return (KERN_TRY_AGAIN
);
1015 * We have found a valid page or we have allocated a new page.
1016 * The page thus may not be valid or may not be entirely
1019 * Attempt to fault-in the page if there is a chance that the
1020 * pager has it, and potentially fault in additional pages
1023 * We are NOT in splvm here and if TRYPAGER is true then
1024 * fs.m will be non-NULL and will be PG_BUSY for us.
1031 u_char behavior
= vm_map_entry_behavior(fs
->entry
);
1033 if (behavior
== MAP_ENTRY_BEHAV_RANDOM
) {
1038 if (behind
> VM_FAULT_READ_BEHIND
)
1039 behind
= VM_FAULT_READ_BEHIND
;
1041 ahead
= fs
->object
->size
- pindex
;
1044 if (ahead
> VM_FAULT_READ_AHEAD
)
1045 ahead
= VM_FAULT_READ_AHEAD
;
1048 if ((fs
->first_object
->type
!= OBJT_DEVICE
) &&
1049 (behavior
== MAP_ENTRY_BEHAV_SEQUENTIAL
||
1050 (behavior
!= MAP_ENTRY_BEHAV_RANDOM
&&
1051 pindex
>= fs
->entry
->lastr
&&
1052 pindex
< fs
->entry
->lastr
+ VM_FAULT_READ
))
1054 vm_pindex_t firstpindex
, tmppindex
;
1056 if (first_pindex
< 2 * VM_FAULT_READ
)
1059 firstpindex
= first_pindex
- 2 * VM_FAULT_READ
;
1062 * note: partially valid pages cannot be
1063 * included in the lookahead - NFS piecemeal
1064 * writes will barf on it badly.
1066 * spl protection is required to avoid races
1067 * between the lookup and an interrupt
1068 * unbusy/free sequence occuring prior to
1072 for (tmppindex
= first_pindex
- 1;
1073 tmppindex
>= firstpindex
;
1078 mt
= vm_page_lookup(fs
->first_object
, tmppindex
);
1079 if (mt
== NULL
|| (mt
->valid
!= VM_PAGE_BITS_ALL
))
1082 (mt
->flags
& (PG_BUSY
| PG_FICTITIOUS
| PG_UNMANAGED
)) ||
1087 vm_page_test_dirty(mt
);
1089 vm_page_protect(mt
, VM_PROT_NONE
);
1090 vm_page_deactivate(mt
);
1102 * now we find out if any other pages should be paged
1103 * in at this time this routine checks to see if the
1104 * pages surrounding this fault reside in the same
1105 * object as the page for this fault. If they do,
1106 * then they are faulted in also into the object. The
1107 * array "marray" returned contains an array of
1108 * vm_page_t structs where one of them is the
1109 * vm_page_t passed to the routine. The reqpage
1110 * return value is the index into the marray for the
1111 * vm_page_t passed to the routine.
1113 * fs.m plus the additional pages are PG_BUSY'd.
1115 faultcount
= vm_fault_additional_pages(
1116 fs
->m
, behind
, ahead
, marray
, &reqpage
);
1119 * update lastr imperfectly (we do not know how much
1120 * getpages will actually read), but good enough.
1122 fs
->entry
->lastr
= pindex
+ faultcount
- behind
;
1125 * Call the pager to retrieve the data, if any, after
1126 * releasing the lock on the map. We hold a ref on
1127 * fs.object and the pages are PG_BUSY'd.
1132 rv
= vm_pager_get_pages(fs
->object
, marray
,
1133 faultcount
, reqpage
);
1138 if (rv
== VM_PAGER_OK
) {
1140 * Found the page. Leave it busy while we play
1145 * Relookup in case pager changed page. Pager
1146 * is responsible for disposition of old page
1149 * XXX other code segments do relookups too.
1150 * It's a bad abstraction that needs to be
1153 fs
->m
= vm_page_lookup(fs
->object
, pindex
);
1154 if (fs
->m
== NULL
) {
1155 unlock_and_deallocate(fs
);
1156 return (KERN_TRY_AGAIN
);
1160 break; /* break to PAGE HAS BEEN FOUND */
1164 * Remove the bogus page (which does not exist at this
1165 * object/offset); before doing so, we must get back
1166 * our object lock to preserve our invariant.
1168 * Also wake up any other process that may want to bring
1171 * If this is the top-level object, we must leave the
1172 * busy page to prevent another process from rushing
1173 * past us, and inserting the page in that object at
1174 * the same time that we are.
1176 if (rv
== VM_PAGER_ERROR
) {
1178 kprintf("vm_fault: pager read error, pid %d (%s)\n", curproc
->p_pid
, curproc
->p_comm
);
1180 kprintf("vm_fault: pager read error, thread %p (%s)\n", curthread
, curproc
->p_comm
);
1183 * Data outside the range of the pager or an I/O error
1186 * XXX - the check for kernel_map is a kludge to work
1187 * around having the machine panic on a kernel space
1188 * fault w/ I/O error.
1190 if (((fs
->map
!= &kernel_map
) && (rv
== VM_PAGER_ERROR
)) ||
1191 (rv
== VM_PAGER_BAD
)) {
1192 vm_page_free(fs
->m
);
1194 unlock_and_deallocate(fs
);
1195 if (rv
== VM_PAGER_ERROR
)
1196 return (KERN_FAILURE
);
1198 return (KERN_PROTECTION_FAILURE
);
1201 if (fs
->object
!= fs
->first_object
) {
1202 vm_page_free(fs
->m
);
1205 * XXX - we cannot just fall out at this
1206 * point, m has been freed and is invalid!
1212 * We get here if the object has a default pager (or unwiring)
1213 * or the pager doesn't have the page.
1215 if (fs
->object
== fs
->first_object
)
1216 fs
->first_m
= fs
->m
;
1219 * Move on to the next object. Lock the next object before
1220 * unlocking the current one.
1222 pindex
+= OFF_TO_IDX(fs
->object
->backing_object_offset
);
1223 next_object
= fs
->object
->backing_object
;
1224 if (next_object
== NULL
) {
1226 * If there's no object left, fill the page in the top
1227 * object with zeros.
1229 if (fs
->object
!= fs
->first_object
) {
1230 vm_object_pip_wakeup(fs
->object
);
1232 fs
->object
= fs
->first_object
;
1233 pindex
= first_pindex
;
1234 fs
->m
= fs
->first_m
;
1239 * Zero the page if necessary and mark it valid.
1241 if ((fs
->m
->flags
& PG_ZERO
) == 0) {
1242 vm_page_zero_fill(fs
->m
);
1244 mycpu
->gd_cnt
.v_ozfod
++;
1246 mycpu
->gd_cnt
.v_zfod
++;
1247 fs
->m
->valid
= VM_PAGE_BITS_ALL
;
1248 break; /* break to PAGE HAS BEEN FOUND */
1250 if (fs
->object
!= fs
->first_object
) {
1251 vm_object_pip_wakeup(fs
->object
);
1253 KASSERT(fs
->object
!= next_object
, ("object loop %p", next_object
));
1254 fs
->object
= next_object
;
1255 vm_object_pip_add(fs
->object
, 1);
1259 KASSERT((fs
->m
->flags
& PG_BUSY
) != 0,
1260 ("vm_fault: not busy after main loop"));
1263 * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
1268 * If the page is being written, but isn't already owned by the
1269 * top-level object, we have to copy it into a new page owned by the
1272 if (fs
->object
!= fs
->first_object
) {
1274 * We only really need to copy if we want to write it.
1276 if (fault_type
& VM_PROT_WRITE
) {
1278 * This allows pages to be virtually copied from a
1279 * backing_object into the first_object, where the
1280 * backing object has no other refs to it, and cannot
1281 * gain any more refs. Instead of a bcopy, we just
1282 * move the page from the backing object to the
1283 * first object. Note that we must mark the page
1284 * dirty in the first object so that it will go out
1285 * to swap when needed.
1289 * Map, if present, has not changed
1292 fs
->map_generation
== fs
->map
->timestamp
) &&
1294 * Only one shadow object
1296 (fs
->object
->shadow_count
== 1) &&
1298 * No COW refs, except us
1300 (fs
->object
->ref_count
== 1) &&
1302 * No one else can look this object up
1304 (fs
->object
->handle
== NULL
) &&
1306 * No other ways to look the object up
1308 ((fs
->object
->type
== OBJT_DEFAULT
) ||
1309 (fs
->object
->type
== OBJT_SWAP
)) &&
1311 * We don't chase down the shadow chain
1313 (fs
->object
== fs
->first_object
->backing_object
) &&
1316 * grab the lock if we need to
1318 (fs
->lookup_still_valid
||
1320 lockmgr(&fs
->map
->lock
, LK_EXCLUSIVE
|LK_NOWAIT
) == 0)
1323 fs
->lookup_still_valid
= 1;
1325 * get rid of the unnecessary page
1327 vm_page_protect(fs
->first_m
, VM_PROT_NONE
);
1328 vm_page_free(fs
->first_m
);
1332 * grab the page and put it into the
1333 * process'es object. The page is
1334 * automatically made dirty.
1336 vm_page_rename(fs
->m
, fs
->first_object
, first_pindex
);
1337 fs
->first_m
= fs
->m
;
1338 vm_page_busy(fs
->first_m
);
1340 mycpu
->gd_cnt
.v_cow_optim
++;
1343 * Oh, well, lets copy it.
1345 vm_page_copy(fs
->m
, fs
->first_m
);
1350 * We no longer need the old page or object.
1356 * fs->object != fs->first_object due to above
1359 vm_object_pip_wakeup(fs
->object
);
1362 * Only use the new page below...
1365 mycpu
->gd_cnt
.v_cow_faults
++;
1366 fs
->m
= fs
->first_m
;
1367 fs
->object
= fs
->first_object
;
1368 pindex
= first_pindex
;
1371 * If it wasn't a write fault avoid having to copy
1372 * the page by mapping it read-only.
1374 fs
->prot
&= ~VM_PROT_WRITE
;
1379 * We may have had to unlock a map to do I/O. If we did then
1380 * lookup_still_valid will be FALSE. If the map generation count
1381 * also changed then all sorts of things could have happened while
1382 * we were doing the I/O and we need to retry.
1385 if (!fs
->lookup_still_valid
&&
1387 (fs
->map
->timestamp
!= fs
->map_generation
)) {
1389 unlock_and_deallocate(fs
);
1390 return (KERN_TRY_AGAIN
);
1394 * Put this page into the physical map. We had to do the unlock above
1395 * because pmap_enter may cause other faults. We don't put the page
1396 * back on the active queue until later so that the page-out daemon
1397 * won't find us (yet).
1399 if (fs
->prot
& VM_PROT_WRITE
) {
1400 vm_page_flag_set(fs
->m
, PG_WRITEABLE
);
1401 vm_object_set_writeable_dirty(fs
->m
->object
);
1404 * If the fault is a write, we know that this page is being
1405 * written NOW so dirty it explicitly to save on
1406 * pmap_is_modified() calls later.
1408 * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
1409 * if the page is already dirty to prevent data written with
1410 * the expectation of being synced from not being synced.
1411 * Likewise if this entry does not request NOSYNC then make
1412 * sure the page isn't marked NOSYNC. Applications sharing
1413 * data should use the same flags to avoid ping ponging.
1415 * Also tell the backing pager, if any, that it should remove
1416 * any swap backing since the page is now dirty.
1418 if (fs
->entry
->eflags
& MAP_ENTRY_NOSYNC
) {
1419 if (fs
->m
->dirty
== 0)
1420 vm_page_flag_set(fs
->m
, PG_NOSYNC
);
1422 vm_page_flag_clear(fs
->m
, PG_NOSYNC
);
1424 if (fs
->fault_flags
& VM_FAULT_DIRTY
) {
1426 vm_page_dirty(fs
->m
);
1427 vm_pager_page_unswapped(fs
->m
);
1433 * Page had better still be busy. We are still locked up and
1434 * fs->object will have another PIP reference if it is not equal
1435 * to fs->first_object.
1437 KASSERT(fs
->m
->flags
& PG_BUSY
,
1438 ("vm_fault: page %p not busy!", fs
->m
));
1441 * Sanity check: page must be completely valid or it is not fit to
1442 * map into user space. vm_pager_get_pages() ensures this.
1444 if (fs
->m
->valid
!= VM_PAGE_BITS_ALL
) {
1445 vm_page_zero_invalid(fs
->m
, TRUE
);
1446 kprintf("Warning: page %p partially invalid on fault\n", fs
->m
);
1449 return (KERN_SUCCESS
);
1453 * Wire down a range of virtual addresses in a map. The entry in question
1454 * should be marked in-transition and the map must be locked. We must
1455 * release the map temporarily while faulting-in the page to avoid a
1456 * deadlock. Note that the entry may be clipped while we are blocked but
1457 * will never be freed.
1460 vm_fault_wire(vm_map_t map
, vm_map_entry_t entry
, boolean_t user_wire
)
1462 boolean_t fictitious
;
1470 pmap
= vm_map_pmap(map
);
1471 start
= entry
->start
;
1473 fictitious
= entry
->object
.vm_object
&&
1474 (entry
->object
.vm_object
->type
== OBJT_DEVICE
);
1480 * We simulate a fault to get the page and enter it in the physical
1483 for (va
= start
; va
< end
; va
+= PAGE_SIZE
) {
1485 rv
= vm_fault(map
, va
, VM_PROT_READ
,
1486 VM_FAULT_USER_WIRE
);
1488 rv
= vm_fault(map
, va
, VM_PROT_READ
|VM_PROT_WRITE
,
1489 VM_FAULT_CHANGE_WIRING
);
1492 while (va
> start
) {
1494 if ((pa
= pmap_extract(pmap
, va
)) == 0)
1496 pmap_change_wiring(pmap
, va
, FALSE
);
1498 vm_page_unwire(PHYS_TO_VM_PAGE(pa
), 1);
1505 return (KERN_SUCCESS
);
1509 * Unwire a range of virtual addresses in a map. The map should be
1513 vm_fault_unwire(vm_map_t map
, vm_map_entry_t entry
)
1515 boolean_t fictitious
;
1522 pmap
= vm_map_pmap(map
);
1523 start
= entry
->start
;
1525 fictitious
= entry
->object
.vm_object
&&
1526 (entry
->object
.vm_object
->type
== OBJT_DEVICE
);
1529 * Since the pages are wired down, we must be able to get their
1530 * mappings from the physical map system.
1532 for (va
= start
; va
< end
; va
+= PAGE_SIZE
) {
1533 pa
= pmap_extract(pmap
, va
);
1535 pmap_change_wiring(pmap
, va
, FALSE
);
1537 vm_page_unwire(PHYS_TO_VM_PAGE(pa
), 1);
1543 * Reduce the rate at which memory is allocated to a process based
1544 * on the perceived load on the VM system. As the load increases
1545 * the allocation burst rate goes down and the delay increases.
1547 * Rate limiting does not apply when faulting active or inactive
1548 * pages. When faulting 'cache' pages, rate limiting only applies
1549 * if the system currently has a severe page deficit.
1551 * XXX vm_pagesupply should be increased when a page is freed.
1553 * We sleep up to 1/10 of a second.
1556 vm_fault_ratelimit(struct vmspace
*vmspace
)
1558 if (vm_load_enable
== 0)
1560 if (vmspace
->vm_pagesupply
> 0) {
1561 --vmspace
->vm_pagesupply
;
1565 if (vm_load_debug
) {
1566 kprintf("load %-4d give %d pgs, wait %d, pid %-5d (%s)\n",
1568 (1000 - vm_load
) / 10, vm_load
* hz
/ 10000,
1569 curproc
->p_pid
, curproc
->p_comm
);
1572 vmspace
->vm_pagesupply
= (1000 - vm_load
) / 10;
1573 return(vm_load
* hz
/ 10000);
1578 * vm_fault_copy_entry
1580 * Copy all of the pages from a wired-down map entry to another.
1582 * In/out conditions:
1583 * The source and destination maps must be locked for write.
1584 * The source map entry must be wired down (or be a sharing map
1585 * entry corresponding to a main map entry that is wired down).
1589 vm_fault_copy_entry(vm_map_t dst_map
, vm_map_t src_map
,
1590 vm_map_entry_t dst_entry
, vm_map_entry_t src_entry
)
1592 vm_object_t dst_object
;
1593 vm_object_t src_object
;
1594 vm_ooffset_t dst_offset
;
1595 vm_ooffset_t src_offset
;
1605 src_object
= src_entry
->object
.vm_object
;
1606 src_offset
= src_entry
->offset
;
1609 * Create the top-level object for the destination entry. (Doesn't
1610 * actually shadow anything - we copy the pages directly.)
1612 vm_map_entry_allocate_object(dst_entry
);
1613 dst_object
= dst_entry
->object
.vm_object
;
1615 prot
= dst_entry
->max_protection
;
1618 * Loop through all of the pages in the entry's range, copying each
1619 * one from the source object (it should be there) to the destination
1622 for (vaddr
= dst_entry
->start
, dst_offset
= 0;
1623 vaddr
< dst_entry
->end
;
1624 vaddr
+= PAGE_SIZE
, dst_offset
+= PAGE_SIZE
) {
1627 * Allocate a page in the destination object
1630 dst_m
= vm_page_alloc(dst_object
,
1631 OFF_TO_IDX(dst_offset
), VM_ALLOC_NORMAL
);
1632 if (dst_m
== NULL
) {
1635 } while (dst_m
== NULL
);
1638 * Find the page in the source object, and copy it in.
1639 * (Because the source is wired down, the page will be in
1642 src_m
= vm_page_lookup(src_object
,
1643 OFF_TO_IDX(dst_offset
+ src_offset
));
1645 panic("vm_fault_copy_wired: page missing");
1647 vm_page_copy(src_m
, dst_m
);
1650 * Enter it in the pmap...
1653 vm_page_flag_clear(dst_m
, PG_ZERO
);
1654 pmap_enter(dst_map
->pmap
, vaddr
, dst_m
, prot
, FALSE
);
1655 vm_page_flag_set(dst_m
, PG_WRITEABLE
|PG_MAPPED
);
1658 * Mark it no longer busy, and put it on the active list.
1660 vm_page_activate(dst_m
);
1661 vm_page_wakeup(dst_m
);
1667 * This routine checks around the requested page for other pages that
1668 * might be able to be faulted in. This routine brackets the viable
1669 * pages for the pages to be paged in.
1672 * m, rbehind, rahead
1675 * marray (array of vm_page_t), reqpage (index of requested page)
1678 * number of pages in marray
1681 vm_fault_additional_pages(vm_page_t m
, int rbehind
, int rahead
,
1682 vm_page_t
*marray
, int *reqpage
)
1686 vm_pindex_t pindex
, startpindex
, endpindex
, tpindex
;
1688 int cbehind
, cahead
;
1694 * we don't fault-ahead for device pager
1696 if (object
->type
== OBJT_DEVICE
) {
1703 * if the requested page is not available, then give up now
1706 if (!vm_pager_has_page(object
, pindex
, &cbehind
, &cahead
)) {
1710 if ((cbehind
== 0) && (cahead
== 0)) {
1716 if (rahead
> cahead
) {
1720 if (rbehind
> cbehind
) {
1725 * try to do any readahead that we might have free pages for.
1727 if ((rahead
+ rbehind
) >
1728 ((vmstats
.v_free_count
+ vmstats
.v_cache_count
) - vmstats
.v_free_reserved
)) {
1729 pagedaemon_wakeup();
1736 * scan backward for the read behind pages -- in memory
1738 * Assume that if the page is not found an interrupt will not
1739 * create it. Theoretically interrupts can only remove (busy)
1740 * pages, not create new associations.
1743 if (rbehind
> pindex
) {
1747 startpindex
= pindex
- rbehind
;
1751 for ( tpindex
= pindex
- 1; tpindex
>= startpindex
; tpindex
-= 1) {
1752 if (vm_page_lookup( object
, tpindex
)) {
1753 startpindex
= tpindex
+ 1;
1760 for(i
= 0, tpindex
= startpindex
; tpindex
< pindex
; i
++, tpindex
++) {
1762 rtm
= vm_page_alloc(object
, tpindex
, VM_ALLOC_NORMAL
);
1765 for (j
= 0; j
< i
; j
++) {
1766 vm_page_free(marray
[j
]);
1782 /* page offset of the required page */
1785 tpindex
= pindex
+ 1;
1789 * scan forward for the read ahead pages
1791 endpindex
= tpindex
+ rahead
;
1792 if (endpindex
> object
->size
)
1793 endpindex
= object
->size
;
1796 for( ; tpindex
< endpindex
; i
++, tpindex
++) {
1798 if (vm_page_lookup(object
, tpindex
)) {
1802 rtm
= vm_page_alloc(object
, tpindex
, VM_ALLOC_NORMAL
);
1811 /* return number of bytes of pages */