4 * Copyright (c) 2010 The DragonFly Project. All rights reserved.
6 * This code is derived from software contributed to The DragonFly Project
7 * by Matthew Dillon <dillon@backplane.com>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * Implement the swapcache daemon. When enabled swap is assumed to be
39 * configured on a fast storage device such as a SSD. Swap is assigned
40 * to clean vnode-backed pages in the inactive queue, clustered by object
41 * if possible, and written out. The swap assignment sticks around even
42 * after the underlying pages have been recycled.
44 * The daemon manages write bandwidth based on sysctl settings to control
47 * The vnode strategy code will check for the swap assignments and divert
48 * reads to the swap device when the data is present in the swapcache.
50 * This operates on both regular files and the block device vnodes used by
51 * filesystems to manage meta-data.
55 #include <sys/param.h>
56 #include <sys/systm.h>
57 #include <sys/kernel.h>
59 #include <sys/kthread.h>
60 #include <sys/resourcevar.h>
61 #include <sys/signalvar.h>
62 #include <sys/vnode.h>
63 #include <sys/vmmeter.h>
64 #include <sys/sysctl.h>
65 #include <sys/eventhandler.h>
68 #include <vm/vm_param.h>
70 #include <vm/vm_object.h>
71 #include <vm/vm_page.h>
72 #include <vm/vm_map.h>
73 #include <vm/vm_pageout.h>
74 #include <vm/vm_pager.h>
75 #include <vm/swap_pager.h>
76 #include <vm/vm_extern.h>
78 #include <sys/thread2.h>
79 #include <sys/spinlock2.h>
80 #include <vm/vm_page2.h>
82 /* the kernel process "vm_pageout"*/
83 static int vm_swapcached_flush (vm_page_t m
, int isblkdev
);
84 static int vm_swapcache_test(vm_page_t m
);
85 static int vm_swapcache_writing_heuristic(void);
86 static int vm_swapcache_writing(vm_page_t marker
, int count
, int scount
);
87 static void vm_swapcache_cleaning(vm_object_t marker
, int *swindexp
);
88 static void vm_swapcache_movemarker(vm_object_t marker
, int swindex
,
90 struct thread
*swapcached_thread
;
92 SYSCTL_NODE(_vm
, OID_AUTO
, swapcache
, CTLFLAG_RW
, NULL
, NULL
);
94 int vm_swapcache_read_enable
;
95 int vm_swapcache_inactive_heuristic
;
96 static int vm_swapcache_sleep
;
97 static int vm_swapcache_maxscan
= PQ_L2_SIZE
* 8;
98 static int vm_swapcache_maxlaunder
= PQ_L2_SIZE
* 4;
99 static int vm_swapcache_data_enable
= 0;
100 static int vm_swapcache_meta_enable
= 0;
101 static int vm_swapcache_maxswappct
= 75;
102 static int vm_swapcache_hysteresis
;
103 static int vm_swapcache_min_hysteresis
;
104 int vm_swapcache_use_chflags
= 1; /* require chflags cache */
105 static int64_t vm_swapcache_minburst
= 10000000LL; /* 10MB */
106 static int64_t vm_swapcache_curburst
= 4000000000LL; /* 4G after boot */
107 static int64_t vm_swapcache_maxburst
= 2000000000LL; /* 2G nominal max */
108 static int64_t vm_swapcache_accrate
= 100000LL; /* 100K/s */
109 static int64_t vm_swapcache_write_count
;
110 static int64_t vm_swapcache_maxfilesize
;
111 static int64_t vm_swapcache_cleanperobj
= 16*1024*1024;
113 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, maxlaunder
,
114 CTLFLAG_RW
, &vm_swapcache_maxlaunder
, 0, "");
115 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, maxscan
,
116 CTLFLAG_RW
, &vm_swapcache_maxscan
, 0, "");
118 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, data_enable
,
119 CTLFLAG_RW
, &vm_swapcache_data_enable
, 0, "");
120 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, meta_enable
,
121 CTLFLAG_RW
, &vm_swapcache_meta_enable
, 0, "");
122 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, read_enable
,
123 CTLFLAG_RW
, &vm_swapcache_read_enable
, 0, "");
124 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, maxswappct
,
125 CTLFLAG_RW
, &vm_swapcache_maxswappct
, 0, "");
126 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, hysteresis
,
127 CTLFLAG_RD
, &vm_swapcache_hysteresis
, 0, "");
128 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, min_hysteresis
,
129 CTLFLAG_RW
, &vm_swapcache_min_hysteresis
, 0, "");
130 SYSCTL_INT(_vm_swapcache
, OID_AUTO
, use_chflags
,
131 CTLFLAG_RW
, &vm_swapcache_use_chflags
, 0, "");
133 SYSCTL_QUAD(_vm_swapcache
, OID_AUTO
, minburst
,
134 CTLFLAG_RW
, &vm_swapcache_minburst
, 0, "");
135 SYSCTL_QUAD(_vm_swapcache
, OID_AUTO
, curburst
,
136 CTLFLAG_RW
, &vm_swapcache_curburst
, 0, "");
137 SYSCTL_QUAD(_vm_swapcache
, OID_AUTO
, maxburst
,
138 CTLFLAG_RW
, &vm_swapcache_maxburst
, 0, "");
139 SYSCTL_QUAD(_vm_swapcache
, OID_AUTO
, maxfilesize
,
140 CTLFLAG_RW
, &vm_swapcache_maxfilesize
, 0, "");
141 SYSCTL_QUAD(_vm_swapcache
, OID_AUTO
, accrate
,
142 CTLFLAG_RW
, &vm_swapcache_accrate
, 0, "");
143 SYSCTL_QUAD(_vm_swapcache
, OID_AUTO
, write_count
,
144 CTLFLAG_RW
, &vm_swapcache_write_count
, 0, "");
145 SYSCTL_QUAD(_vm_swapcache
, OID_AUTO
, cleanperobj
,
146 CTLFLAG_RW
, &vm_swapcache_cleanperobj
, 0, "");
148 #define SWAPMAX(adj) \
149 ((int64_t)vm_swap_max * (vm_swapcache_maxswappct + (adj)) / 100)
152 * When shutting down the machine we want to stop swapcache operation
153 * immediately so swap is not accessed after devices have been shuttered.
156 shutdown_swapcache(void *arg __unused
)
158 vm_swapcache_read_enable
= 0;
159 vm_swapcache_data_enable
= 0;
160 vm_swapcache_meta_enable
= 0;
161 wakeup(&vm_swapcache_sleep
); /* shortcut 5-second wait */
165 * vm_swapcached is the high level pageout daemon.
170 vm_swapcached_thread(void)
172 enum { SWAPC_WRITING
, SWAPC_CLEANING
} state
= SWAPC_WRITING
;
173 enum { SWAPB_BURSTING
, SWAPB_RECOVERING
} burst
= SWAPB_BURSTING
;
174 static struct vm_page page_marker
[PQ_L2_SIZE
];
175 static struct vm_object swmarker
;
182 curthread
->td_flags
|= TDF_SYSTHREAD
;
183 EVENTHANDLER_REGISTER(shutdown_pre_sync
, shutdown_kproc
,
184 swapcached_thread
, SHUTDOWN_PRI_FIRST
);
185 EVENTHANDLER_REGISTER(shutdown_pre_sync
, shutdown_swapcache
,
186 NULL
, SHUTDOWN_PRI_SECOND
);
189 * Initialize our marker for the inactive scan (SWAPC_WRITING)
191 bzero(&page_marker
, sizeof(page_marker
));
192 for (q
= 0; q
< PQ_L2_SIZE
; ++q
) {
193 page_marker
[q
].flags
= PG_BUSY
| PG_FICTITIOUS
| PG_MARKER
;
194 page_marker
[q
].queue
= PQ_INACTIVE
+ q
;
195 page_marker
[q
].pc
= q
;
196 page_marker
[q
].wire_count
= 1;
197 vm_page_queues_spin_lock(PQ_INACTIVE
+ q
);
199 &vm_page_queues
[PQ_INACTIVE
+ q
].pl
,
200 &page_marker
[q
], pageq
);
201 vm_page_queues_spin_unlock(PQ_INACTIVE
+ q
);
204 vm_swapcache_min_hysteresis
= 1024;
205 vm_swapcache_hysteresis
= vm_swapcache_min_hysteresis
;
206 vm_swapcache_inactive_heuristic
= -vm_swapcache_hysteresis
;
209 * Initialize our marker for the vm_object scan (SWAPC_CLEANING)
211 bzero(&swmarker
, sizeof(swmarker
));
212 swmarker
.type
= OBJT_MARKER
;
214 lwkt_gettoken(&vmobj_tokens
[swindex
]);
215 TAILQ_INSERT_HEAD(&vm_object_lists
[swindex
],
216 &swmarker
, object_list
);
217 lwkt_reltoken(&vmobj_tokens
[swindex
]);
227 kproc_suspend_loop();
230 * Check every 5 seconds when not enabled or if no swap
233 if ((vm_swapcache_data_enable
== 0 &&
234 vm_swapcache_meta_enable
== 0 &&
235 vm_swap_cache_use
<= SWAPMAX(0)) ||
237 tsleep(&vm_swapcache_sleep
, 0, "csleep", hz
* 5);
242 * Polling rate when enabled is approximately 10 hz.
244 tsleep(&vm_swapcache_sleep
, 0, "csleep", hz
/ 10);
247 * State hysteresis. Generate write activity up to 75% of
248 * swap, then clean out swap assignments down to 70%, then
251 if (state
== SWAPC_WRITING
) {
252 if (vm_swap_cache_use
> SWAPMAX(0))
253 state
= SWAPC_CLEANING
;
255 if (vm_swap_cache_use
< SWAPMAX(-10))
256 state
= SWAPC_WRITING
;
260 * We are allowed to continue accumulating burst value
261 * in either state. Allow the user to set curburst > maxburst
262 * for the initial load-in.
264 if (vm_swapcache_curburst
< vm_swapcache_maxburst
) {
265 vm_swapcache_curburst
+= vm_swapcache_accrate
/ 10;
266 if (vm_swapcache_curburst
> vm_swapcache_maxburst
)
267 vm_swapcache_curburst
= vm_swapcache_maxburst
;
271 * We don't want to nickle-and-dime the scan as that will
272 * create unnecessary fragmentation. The minimum burst
273 * is one-seconds worth of accumulation.
275 if (state
!= SWAPC_WRITING
) {
276 vm_swapcache_cleaning(&swmarker
, &swindex
);
279 if (vm_swapcache_curburst
< vm_swapcache_accrate
)
283 count
= vm_swapcache_maxlaunder
/ PQ_L2_SIZE
+ 2;
284 scount
= vm_swapcache_maxscan
/ PQ_L2_SIZE
+ 2;
286 if (burst
== SWAPB_BURSTING
) {
287 if (vm_swapcache_writing_heuristic()) {
288 for (q
= 0; q
< PQ_L2_SIZE
; ++q
) {
290 vm_swapcache_writing(
296 if (vm_swapcache_curburst
<= 0)
297 burst
= SWAPB_RECOVERING
;
298 } else if (vm_swapcache_curburst
> vm_swapcache_minburst
) {
299 if (vm_swapcache_writing_heuristic()) {
300 for (q
= 0; q
< PQ_L2_SIZE
; ++q
) {
302 vm_swapcache_writing(
308 burst
= SWAPB_BURSTING
;
310 if (reached_end
== PQ_L2_SIZE
) {
311 vm_swapcache_inactive_heuristic
=
312 -vm_swapcache_hysteresis
;
317 * Cleanup (NOT REACHED)
319 for (q
= 0; q
< PQ_L2_SIZE
; ++q
) {
320 vm_page_queues_spin_lock(PQ_INACTIVE
+ q
);
322 &vm_page_queues
[PQ_INACTIVE
+ q
].pl
,
323 &page_marker
[q
], pageq
);
324 vm_page_queues_spin_unlock(PQ_INACTIVE
+ q
);
327 lwkt_gettoken(&vmobj_tokens
[swindex
]);
328 TAILQ_REMOVE(&vm_object_lists
[swindex
], &swmarker
, object_list
);
329 lwkt_reltoken(&vmobj_tokens
[swindex
]);
332 static struct kproc_desc swpc_kp
= {
334 vm_swapcached_thread
,
337 SYSINIT(swapcached
, SI_SUB_KTHREAD_PAGE
, SI_ORDER_SECOND
, kproc_start
, &swpc_kp
);
340 * Deal with an overflow of the heuristic counter or if the user
341 * manually changes the hysteresis.
343 * Try to avoid small incremental pageouts by waiting for enough
344 * pages to buildup in the inactive queue to hopefully get a good
345 * burst in. This heuristic is bumped by the VM system and reset
346 * when our scan hits the end of the queue.
348 * Return TRUE if we need to take a writing pass.
351 vm_swapcache_writing_heuristic(void)
355 hyst
= vmstats
.v_inactive_count
/ 4;
356 if (hyst
< vm_swapcache_min_hysteresis
)
357 hyst
= vm_swapcache_min_hysteresis
;
359 vm_swapcache_hysteresis
= hyst
;
361 if (vm_swapcache_inactive_heuristic
< -hyst
)
362 vm_swapcache_inactive_heuristic
= -hyst
;
364 return (vm_swapcache_inactive_heuristic
>= 0);
368 * Take a writing pass on one of the inactive queues, return non-zero if
369 * we hit the end of the queue.
372 vm_swapcache_writing(vm_page_t marker
, int count
, int scount
)
380 * Scan the inactive queue from our marker to locate
381 * suitable pages to push to the swap cache.
383 * We are looking for clean vnode-backed pages.
385 vm_page_queues_spin_lock(marker
->queue
);
386 while ((m
= TAILQ_NEXT(marker
, pageq
)) != NULL
&&
387 count
> 0 && scount
-- > 0) {
388 KKASSERT(m
->queue
== marker
->queue
);
391 * Stop using swap if paniced, dumping, or dumped.
392 * Don't try to write if our curburst has been exhausted.
394 if (panicstr
|| dumping
)
396 if (vm_swapcache_curburst
< 0)
403 &vm_page_queues
[marker
->queue
].pl
, marker
, pageq
);
405 &vm_page_queues
[marker
->queue
].pl
, m
, marker
, pageq
);
408 * Ignore markers and ignore pages that already have a swap
411 if (m
->flags
& (PG_MARKER
| PG_SWAPPED
))
413 if (vm_page_busy_try(m
, TRUE
))
415 vm_page_queues_spin_unlock(marker
->queue
);
417 if ((object
= m
->object
) == NULL
) {
419 vm_page_queues_spin_lock(marker
->queue
);
422 vm_object_hold(object
);
423 if (m
->object
!= object
) {
424 vm_object_drop(object
);
426 vm_page_queues_spin_lock(marker
->queue
);
429 if (vm_swapcache_test(m
)) {
430 vm_object_drop(object
);
432 vm_page_queues_spin_lock(marker
->queue
);
438 vm_object_drop(object
);
440 vm_page_queues_spin_lock(marker
->queue
);
447 * PG_NOTMETA generically means 'don't swapcache this',
448 * and HAMMER will set this for regular data buffers
449 * (and leave it unset for meta-data buffers) as
450 * appropriate when double buffering is enabled.
452 if (m
->flags
& PG_NOTMETA
) {
453 vm_object_drop(object
);
455 vm_page_queues_spin_lock(marker
->queue
);
460 * If data_enable is 0 do not try to swapcache data.
461 * If use_chflags is set then only swapcache data for
462 * VSWAPCACHE marked vnodes, otherwise any vnode.
464 if (vm_swapcache_data_enable
== 0 ||
465 ((vp
->v_flag
& VSWAPCACHE
) == 0 &&
466 vm_swapcache_use_chflags
)) {
467 vm_object_drop(object
);
469 vm_page_queues_spin_lock(marker
->queue
);
472 if (vm_swapcache_maxfilesize
&&
474 (vm_swapcache_maxfilesize
>> PAGE_SHIFT
)) {
475 vm_object_drop(object
);
477 vm_page_queues_spin_lock(marker
->queue
);
484 * PG_NOTMETA generically means 'don't swapcache this',
485 * and HAMMER will set this for regular data buffers
486 * (and leave it unset for meta-data buffers) as
487 * appropriate when double buffering is enabled.
489 if (m
->flags
& PG_NOTMETA
) {
490 vm_object_drop(object
);
492 vm_page_queues_spin_lock(marker
->queue
);
495 if (vm_swapcache_meta_enable
== 0) {
496 vm_object_drop(object
);
498 vm_page_queues_spin_lock(marker
->queue
);
504 vm_object_drop(object
);
506 vm_page_queues_spin_lock(marker
->queue
);
512 * Assign swap and initiate I/O.
514 * (adjust for the --count which also occurs in the loop)
516 count
-= vm_swapcached_flush(m
, isblkdev
);
519 * Setup for next loop using marker.
521 vm_object_drop(object
);
522 vm_page_queues_spin_lock(marker
->queue
);
526 * The marker could wind up at the end, which is ok. If we hit the
527 * end of the list adjust the heuristic.
529 * Earlier inactive pages that were dirty and become clean
530 * are typically moved to the end of PQ_INACTIVE by virtue
531 * of vfs_vmio_release() when they become unwired from the
534 vm_page_queues_spin_unlock(marker
->queue
);
537 * m invalid but can be used to test for NULL
543 * Flush the specified page using the swap_pager. The page
544 * must be busied by the caller and its disposition will become
545 * the responsibility of this function.
547 * Try to collect surrounding pages, including pages which may
548 * have already been assigned swap. Try to cluster within a
549 * contiguous aligned SMAP_META_PAGES (typ 16 x PAGE_SIZE) block
550 * to match what swap_pager_putpages() can do.
552 * We also want to try to match against the buffer cache blocksize
553 * but we don't really know what it is here. Since the buffer cache
554 * wires and unwires pages in groups the fact that we skip wired pages
555 * should be sufficient.
557 * Returns a count of pages we might have flushed (minimum 1)
561 vm_swapcached_flush(vm_page_t m
, int isblkdev
)
564 vm_page_t marray
[SWAP_META_PAGES
];
566 int rtvals
[SWAP_META_PAGES
];
574 vm_page_protect(m
, VM_PROT_READ
);
576 vm_object_hold(object
);
579 * Try to cluster around (m), keeping in mind that the swap pager
580 * can only do SMAP_META_PAGES worth of continguous write.
582 x
= (int)m
->pindex
& SWAP_META_MASK
;
587 for (i
= x
- 1; i
>= 0; --i
) {
588 m
= vm_page_lookup_busy_try(object
, basei
- x
+ i
,
590 if (error
|| m
== NULL
)
592 if (vm_swapcache_test(m
)) {
596 if (isblkdev
&& (m
->flags
& PG_NOTMETA
)) {
601 vm_page_protect(m
, VM_PROT_READ
);
602 if (m
->queue
- m
->pc
== PQ_CACHE
) {
603 vm_page_unqueue_nowakeup(m
);
604 vm_page_deactivate(m
);
611 for (j
= x
+ 1; j
< SWAP_META_PAGES
; ++j
) {
612 m
= vm_page_lookup_busy_try(object
, basei
- x
+ j
,
614 if (error
|| m
== NULL
)
616 if (vm_swapcache_test(m
)) {
620 if (isblkdev
&& (m
->flags
& PG_NOTMETA
)) {
625 vm_page_protect(m
, VM_PROT_READ
);
626 if (m
->queue
- m
->pc
== PQ_CACHE
) {
627 vm_page_unqueue_nowakeup(m
);
628 vm_page_deactivate(m
);
635 vm_object_pip_add(object
, count
);
636 swap_pager_putpages(object
, marray
+ i
, count
, FALSE
, rtvals
+ i
);
637 vm_swapcache_write_count
+= count
* PAGE_SIZE
;
638 vm_swapcache_curburst
-= count
* PAGE_SIZE
;
641 if (rtvals
[i
] != VM_PAGER_PEND
) {
642 vm_page_busy_wait(marray
[i
], FALSE
, "swppgfd");
643 vm_page_io_finish(marray
[i
]);
644 vm_page_wakeup(marray
[i
]);
645 vm_object_pip_wakeup(object
);
649 vm_object_drop(object
);
654 * Test whether a VM page is suitable for writing to the swapcache.
655 * Does not test m->queue, PG_MARKER, or PG_SWAPPED.
657 * Returns 0 on success, 1 on failure
660 vm_swapcache_test(vm_page_t m
)
664 if (m
->flags
& PG_UNMANAGED
)
666 if (m
->hold_count
|| m
->wire_count
)
668 if (m
->valid
!= VM_PAGE_BITS_ALL
)
670 if (m
->dirty
& m
->valid
)
672 if ((object
= m
->object
) == NULL
)
674 if (object
->type
!= OBJT_VNODE
||
675 (object
->flags
& OBJ_DEAD
)) {
678 vm_page_test_dirty(m
);
679 if (m
->dirty
& m
->valid
)
687 * We clean whole objects up to 16MB
691 vm_swapcache_cleaning(vm_object_t marker
, int *swindexp
)
699 count
= vm_swapcache_maxlaunder
;
700 scount
= vm_swapcache_maxscan
;
703 * Look for vnode objects
705 lwkt_gettoken(&vmobj_tokens
[*swindexp
]);
708 while ((object
= TAILQ_NEXT(marker
, object_list
)) != NULL
) {
710 * We have to skip markers. We cannot hold/drop marker
713 if (object
->type
== OBJT_MARKER
) {
714 vm_swapcache_movemarker(marker
, *swindexp
, object
);
719 * Safety, or in case there are millions of VM objects
720 * without swapcache backing.
726 * We must hold the object before potentially yielding.
728 vm_object_hold(object
);
732 * Only operate on live VNODE objects that are either
733 * VREG or VCHR (VCHR for meta-data).
735 if ((object
->type
!= OBJT_VNODE
) ||
736 ((object
->flags
& OBJ_DEAD
) ||
737 object
->swblock_count
== 0) ||
738 ((vp
= object
->handle
) == NULL
) ||
739 (vp
->v_type
!= VREG
&& vp
->v_type
!= VCHR
)) {
740 vm_object_drop(object
);
741 /* object may be invalid now */
742 vm_swapcache_movemarker(marker
, *swindexp
, object
);
747 * Reset the object pindex stored in the marker if the
748 * working object has changed.
750 if (marker
->backing_object
!= object
) {
752 marker
->backing_object_offset
= 0;
753 marker
->backing_object
= object
;
757 * Look for swblocks starting at our iterator.
759 * The swap_pager_condfree() function attempts to free
760 * swap space starting at the specified index. The index
761 * will be updated on return. The function will return
762 * a scan factor (NOT the number of blocks freed).
764 * If it must cut its scan of the object short due to an
765 * excessive number of swblocks, or is able to free the
766 * requested number of blocks, it will return n >= count
767 * and we break and pick it back up on a future attempt.
769 * Scan the object linearly and try to batch large sets of
770 * blocks that are likely to clean out entire swap radix
774 lwkt_reltoken(&vmobj_tokens
[*swindexp
]);
776 n
= swap_pager_condfree(object
, &marker
->size
,
777 (count
+ SWAP_META_MASK
) & ~SWAP_META_MASK
);
779 vm_object_drop(object
); /* object may be invalid now */
780 lwkt_gettoken(&vmobj_tokens
[*swindexp
]);
783 * If we have exhausted the object or deleted our per-pass
784 * page limit then move us to the next object. Note that
785 * the current object may no longer be on the vm_object_list.
788 marker
->backing_object_offset
> vm_swapcache_cleanperobj
) {
789 vm_swapcache_movemarker(marker
, *swindexp
, object
);
793 * If we have exhausted our max-launder stop for now.
796 marker
->backing_object_offset
+= n
* PAGE_SIZE
;
802 * Iterate vm_object_lists[] hash table
804 TAILQ_REMOVE(&vm_object_lists
[*swindexp
], marker
, object_list
);
805 lwkt_reltoken(&vmobj_tokens
[*swindexp
]);
806 if (++*swindexp
>= VMOBJ_HSIZE
)
808 lwkt_gettoken(&vmobj_tokens
[*swindexp
]);
809 TAILQ_INSERT_HEAD(&vm_object_lists
[*swindexp
], marker
, object_list
);
815 lwkt_reltoken(&vmobj_tokens
[*swindexp
]);
819 * Move the marker past the current object. Object can be stale, but we
820 * still need it to determine if the marker has to be moved. If the object
821 * is still the 'current object' (object after the marker), we hop-scotch
822 * the marker past it.
825 vm_swapcache_movemarker(vm_object_t marker
, int swindex
, vm_object_t object
)
827 if (TAILQ_NEXT(marker
, object_list
) == object
) {
828 TAILQ_REMOVE(&vm_object_lists
[swindex
], marker
, object_list
);
829 TAILQ_INSERT_AFTER(&vm_object_lists
[swindex
], object
,
830 marker
, object_list
);