2 * Copyright (c) 1994,1997 John S. Dyson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice immediately at the beginning of the file, without modification,
10 * this list of conditions, and the following disclaimer.
11 * 2. Absolutely no warranty of function or purpose is made by the author
14 * $FreeBSD: src/sys/kern/vfs_bio.c,v 1.242.2.20 2003/05/28 18:38:10 alc Exp $
15 * $DragonFly: src/sys/kern/vfs_bio.c,v 1.36 2005/05/08 00:12:22 dillon Exp $
19 * this file contains a new buffer I/O scheme implementing a coherent
20 * VM object and buffer cache scheme. Pains have been taken to make
21 * sure that the performance degradation associated with schemes such
22 * as this is not realized.
24 * Author: John S. Dyson
25 * Significant help during the development and debugging phases
26 * had been provided by David Greenman, also of the FreeBSD core team.
28 * see man buf(9) for more info.
31 #include <sys/param.h>
32 #include <sys/systm.h>
35 #include <sys/eventhandler.h>
37 #include <sys/malloc.h>
38 #include <sys/mount.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
42 #include <sys/reboot.h>
43 #include <sys/resourcevar.h>
44 #include <sys/sysctl.h>
45 #include <sys/vmmeter.h>
46 #include <sys/vnode.h>
49 #include <vm/vm_param.h>
50 #include <vm/vm_kern.h>
51 #include <vm/vm_pageout.h>
52 #include <vm/vm_page.h>
53 #include <vm/vm_object.h>
54 #include <vm/vm_extern.h>
55 #include <vm/vm_map.h>
58 #include <sys/thread2.h>
59 #include <vm/vm_page2.h>
61 static MALLOC_DEFINE(M_BIOBUF
, "BIO buffer", "BIO buffer");
63 struct bio_ops bioops
; /* I/O operation notification */
65 struct buf
*buf
; /* buffer header pool */
66 struct swqueue bswlist
;
68 static void vm_hold_free_pages(struct buf
* bp
, vm_offset_t from
,
70 static void vm_hold_load_pages(struct buf
* bp
, vm_offset_t from
,
72 static void vfs_page_set_valid(struct buf
*bp
, vm_ooffset_t off
,
73 int pageno
, vm_page_t m
);
74 static void vfs_clean_pages(struct buf
* bp
);
75 static void vfs_setdirty(struct buf
*bp
);
76 static void vfs_vmio_release(struct buf
*bp
);
78 static void vfs_backgroundwritedone(struct buf
*bp
);
80 static int flushbufqueues(void);
82 static int bd_request
;
84 static void buf_daemon (void);
86 * bogus page -- for I/O to/from partially complete buffers
87 * this is a temporary solution to the problem, but it is not
88 * really that bad. it would be better to split the buffer
89 * for input in the case of buffers partially already in memory,
90 * but the code is intricate enough already.
93 int vmiodirenable
= TRUE
;
95 struct lwkt_token buftimetoken
; /* Interlock on setting prio and timo */
97 static vm_offset_t bogus_offset
;
99 static int bufspace
, maxbufspace
,
100 bufmallocspace
, maxbufmallocspace
, lobufspace
, hibufspace
;
101 static int bufreusecnt
, bufdefragcnt
, buffreekvacnt
;
102 static int needsbuffer
;
103 static int lorunningspace
, hirunningspace
, runningbufreq
;
104 static int numdirtybuffers
, lodirtybuffers
, hidirtybuffers
;
105 static int numfreebuffers
, lofreebuffers
, hifreebuffers
;
106 static int getnewbufcalls
;
107 static int getnewbufrestarts
;
109 SYSCTL_INT(_vfs
, OID_AUTO
, numdirtybuffers
, CTLFLAG_RD
,
110 &numdirtybuffers
, 0, "");
111 SYSCTL_INT(_vfs
, OID_AUTO
, lodirtybuffers
, CTLFLAG_RW
,
112 &lodirtybuffers
, 0, "");
113 SYSCTL_INT(_vfs
, OID_AUTO
, hidirtybuffers
, CTLFLAG_RW
,
114 &hidirtybuffers
, 0, "");
115 SYSCTL_INT(_vfs
, OID_AUTO
, numfreebuffers
, CTLFLAG_RD
,
116 &numfreebuffers
, 0, "");
117 SYSCTL_INT(_vfs
, OID_AUTO
, lofreebuffers
, CTLFLAG_RW
,
118 &lofreebuffers
, 0, "");
119 SYSCTL_INT(_vfs
, OID_AUTO
, hifreebuffers
, CTLFLAG_RW
,
120 &hifreebuffers
, 0, "");
121 SYSCTL_INT(_vfs
, OID_AUTO
, runningbufspace
, CTLFLAG_RD
,
122 &runningbufspace
, 0, "");
123 SYSCTL_INT(_vfs
, OID_AUTO
, lorunningspace
, CTLFLAG_RW
,
124 &lorunningspace
, 0, "");
125 SYSCTL_INT(_vfs
, OID_AUTO
, hirunningspace
, CTLFLAG_RW
,
126 &hirunningspace
, 0, "");
127 SYSCTL_INT(_vfs
, OID_AUTO
, maxbufspace
, CTLFLAG_RD
,
128 &maxbufspace
, 0, "");
129 SYSCTL_INT(_vfs
, OID_AUTO
, hibufspace
, CTLFLAG_RD
,
131 SYSCTL_INT(_vfs
, OID_AUTO
, lobufspace
, CTLFLAG_RD
,
133 SYSCTL_INT(_vfs
, OID_AUTO
, bufspace
, CTLFLAG_RD
,
135 SYSCTL_INT(_vfs
, OID_AUTO
, maxmallocbufspace
, CTLFLAG_RW
,
136 &maxbufmallocspace
, 0, "");
137 SYSCTL_INT(_vfs
, OID_AUTO
, bufmallocspace
, CTLFLAG_RD
,
138 &bufmallocspace
, 0, "");
139 SYSCTL_INT(_vfs
, OID_AUTO
, getnewbufcalls
, CTLFLAG_RW
,
140 &getnewbufcalls
, 0, "");
141 SYSCTL_INT(_vfs
, OID_AUTO
, getnewbufrestarts
, CTLFLAG_RW
,
142 &getnewbufrestarts
, 0, "");
143 SYSCTL_INT(_vfs
, OID_AUTO
, vmiodirenable
, CTLFLAG_RW
,
144 &vmiodirenable
, 0, "");
145 SYSCTL_INT(_vfs
, OID_AUTO
, bufdefragcnt
, CTLFLAG_RW
,
146 &bufdefragcnt
, 0, "");
147 SYSCTL_INT(_vfs
, OID_AUTO
, buffreekvacnt
, CTLFLAG_RW
,
148 &buffreekvacnt
, 0, "");
149 SYSCTL_INT(_vfs
, OID_AUTO
, bufreusecnt
, CTLFLAG_RW
,
150 &bufreusecnt
, 0, "");
154 * Disable background writes for now. There appear to be races in the
155 * flags tests and locking operations as well as races in the completion
156 * code modifying the original bp (origbp) without holding a lock, assuming
157 * splbio protection when there might not be splbio protection.
159 * XXX disable also because the RB tree can't handle multiple blocks with
162 static int dobkgrdwrite
= 0;
163 SYSCTL_INT(_debug
, OID_AUTO
, dobkgrdwrite
, CTLFLAG_RW
, &dobkgrdwrite
, 0,
164 "Do background writes (honoring the BV_BKGRDWRITE flag)?");
167 static int bufhashmask
;
168 static int bufhashshift
;
169 static LIST_HEAD(bufhashhdr
, buf
) *bufhashtbl
, invalhash
;
170 struct bqueues bufqueues
[BUFFER_QUEUES
] = { { 0 } };
171 char *buf_wmesg
= BUF_WMESG
;
173 extern int vm_swap_size
;
175 #define VFS_BIO_NEED_ANY 0x01 /* any freeable buffer */
176 #define VFS_BIO_NEED_DIRTYFLUSH 0x02 /* waiting for dirty buffer flush */
177 #define VFS_BIO_NEED_FREE 0x04 /* wait for free bufs, hi hysteresis */
178 #define VFS_BIO_NEED_BUFSPACE 0x08 /* wait for buf space, lo hysteresis */
181 * Buffer hash table code. Note that the logical block scans linearly, which
182 * gives us some L1 cache locality.
187 bufhash(struct vnode
*vnp
, daddr_t bn
)
193 * A variation on the Fibonacci hash that Knuth credits to
194 * R. W. Floyd, see Knuth's _Art of Computer Programming,
195 * Volume 3 / Sorting and Searching_
197 * We reduce the argument to 32 bits before doing the hash to
198 * avoid the need for a slow 64x64 multiply on 32 bit platforms.
200 * sizeof(struct vnode) is 168 on i386, so toss some of the lower
201 * bits of the vnode address to reduce the key range, which
202 * improves the distribution of keys across buckets.
204 * The file system cylinder group blocks are very heavily
205 * used. They are located at invervals of fbg, which is
206 * on the order of 89 to 94 * 2^10, depending on other
207 * filesystem parameters, for a 16k block size. Smaller block
208 * sizes will reduce fpg approximately proportionally. This
209 * will cause the cylinder group index to be hashed using the
210 * lower bits of the hash multiplier, which will not distribute
211 * the keys as uniformly in a classic Fibonacci hash where a
212 * relatively small number of the upper bits of the result
213 * are used. Using 2^16 as a close-enough approximation to
214 * fpg, split the hash multiplier in half, with the upper 16
215 * bits being the inverse of the golden ratio, and the lower
216 * 16 bits being a fraction between 1/3 and 3/7 (closer to
217 * 3/7 in this case), that gives good experimental results.
219 hashkey64
= ((u_int64_t
)(uintptr_t)vnp
>> 3) + (u_int64_t
)bn
;
220 hashkey
= (((u_int32_t
)(hashkey64
+ (hashkey64
>> 32)) * 0x9E376DB1u
) >>
221 bufhashshift
) & bufhashmask
;
222 return(&bufhashtbl
[hashkey
]);
228 * If someone is blocked due to there being too many dirty buffers,
229 * and numdirtybuffers is now reasonable, wake them up.
233 numdirtywakeup(int level
)
235 if (numdirtybuffers
<= level
) {
236 if (needsbuffer
& VFS_BIO_NEED_DIRTYFLUSH
) {
237 needsbuffer
&= ~VFS_BIO_NEED_DIRTYFLUSH
;
238 wakeup(&needsbuffer
);
246 * Called when buffer space is potentially available for recovery.
247 * getnewbuf() will block on this flag when it is unable to free
248 * sufficient buffer space. Buffer space becomes recoverable when
249 * bp's get placed back in the queues.
256 * If someone is waiting for BUF space, wake them up. Even
257 * though we haven't freed the kva space yet, the waiting
258 * process will be able to now.
260 if (needsbuffer
& VFS_BIO_NEED_BUFSPACE
) {
261 needsbuffer
&= ~VFS_BIO_NEED_BUFSPACE
;
262 wakeup(&needsbuffer
);
267 * runningbufwakeup() - in-progress I/O accounting.
271 runningbufwakeup(struct buf
*bp
)
273 if (bp
->b_runningbufspace
) {
274 runningbufspace
-= bp
->b_runningbufspace
;
275 bp
->b_runningbufspace
= 0;
276 if (runningbufreq
&& runningbufspace
<= lorunningspace
) {
278 wakeup(&runningbufreq
);
286 * Called when a buffer has been added to one of the free queues to
287 * account for the buffer and to wakeup anyone waiting for free buffers.
288 * This typically occurs when large amounts of metadata are being handled
289 * by the buffer cache ( else buffer space runs out first, usually ).
297 needsbuffer
&= ~VFS_BIO_NEED_ANY
;
298 if (numfreebuffers
>= hifreebuffers
)
299 needsbuffer
&= ~VFS_BIO_NEED_FREE
;
300 wakeup(&needsbuffer
);
305 * waitrunningbufspace()
307 * runningbufspace is a measure of the amount of I/O currently
308 * running. This routine is used in async-write situations to
309 * prevent creating huge backups of pending writes to a device.
310 * Only asynchronous writes are governed by this function.
312 * Reads will adjust runningbufspace, but will not block based on it.
313 * The read load has a side effect of reducing the allowed write load.
315 * This does NOT turn an async write into a sync write. It waits
316 * for earlier writes to complete and generally returns before the
317 * caller's write has reached the device.
320 waitrunningbufspace(void)
322 while (runningbufspace
> hirunningspace
) {
325 s
= splbio(); /* fix race against interrupt/biodone() */
327 tsleep(&runningbufreq
, 0, "wdrain", 0);
333 * vfs_buf_test_cache:
335 * Called when a buffer is extended. This function clears the B_CACHE
336 * bit if the newly extended portion of the buffer does not contain
341 vfs_buf_test_cache(struct buf
*bp
,
342 vm_ooffset_t foff
, vm_offset_t off
, vm_offset_t size
,
345 if (bp
->b_flags
& B_CACHE
) {
346 int base
= (foff
+ off
) & PAGE_MASK
;
347 if (vm_page_is_valid(m
, base
, size
) == 0)
348 bp
->b_flags
&= ~B_CACHE
;
354 bd_wakeup(int dirtybuflevel
)
356 if (bd_request
== 0 && numdirtybuffers
>= dirtybuflevel
) {
363 * bd_speedup - speedup the buffer cache flushing code
374 * Initialize buffer headers and related structures.
378 bufhashinit(caddr_t vaddr
)
380 /* first, make a null hash table */
382 for (bufhashmask
= 8; bufhashmask
< nbuf
/ 4; bufhashmask
<<= 1)
384 bufhashtbl
= (void *)vaddr
;
385 vaddr
= vaddr
+ sizeof(*bufhashtbl
) * bufhashmask
;
396 TAILQ_INIT(&bswlist
);
397 LIST_INIT(&invalhash
);
398 lwkt_token_init(&buftimetoken
);
400 for (i
= 0; i
<= bufhashmask
; i
++)
401 LIST_INIT(&bufhashtbl
[i
]);
403 /* next, make a null set of free lists */
404 for (i
= 0; i
< BUFFER_QUEUES
; i
++)
405 TAILQ_INIT(&bufqueues
[i
]);
407 /* finally, initialize each buffer header and stick on empty q */
408 for (i
= 0; i
< nbuf
; i
++) {
410 bzero(bp
, sizeof *bp
);
411 bp
->b_flags
= B_INVAL
; /* we're just an empty header */
413 bp
->b_qindex
= QUEUE_EMPTY
;
415 xio_init(&bp
->b_xio
);
416 LIST_INIT(&bp
->b_dep
);
418 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_EMPTY
], bp
, b_freelist
);
419 LIST_INSERT_HEAD(&invalhash
, bp
, b_hash
);
423 * maxbufspace is the absolute maximum amount of buffer space we are
424 * allowed to reserve in KVM and in real terms. The absolute maximum
425 * is nominally used by buf_daemon. hibufspace is the nominal maximum
426 * used by most other processes. The differential is required to
427 * ensure that buf_daemon is able to run when other processes might
428 * be blocked waiting for buffer space.
430 * maxbufspace is based on BKVASIZE. Allocating buffers larger then
431 * this may result in KVM fragmentation which is not handled optimally
434 maxbufspace
= nbuf
* BKVASIZE
;
435 hibufspace
= imax(3 * maxbufspace
/ 4, maxbufspace
- MAXBSIZE
* 10);
436 lobufspace
= hibufspace
- MAXBSIZE
;
438 lorunningspace
= 512 * 1024;
439 hirunningspace
= 1024 * 1024;
442 * Limit the amount of malloc memory since it is wired permanently into
443 * the kernel space. Even though this is accounted for in the buffer
444 * allocation, we don't want the malloced region to grow uncontrolled.
445 * The malloc scheme improves memory utilization significantly on average
446 * (small) directories.
448 maxbufmallocspace
= hibufspace
/ 20;
451 * Reduce the chance of a deadlock occuring by limiting the number
452 * of delayed-write dirty buffers we allow to stack up.
454 hidirtybuffers
= nbuf
/ 4 + 20;
457 * To support extreme low-memory systems, make sure hidirtybuffers cannot
458 * eat up all available buffer space. This occurs when our minimum cannot
459 * be met. We try to size hidirtybuffers to 3/4 our buffer space assuming
460 * BKVASIZE'd (8K) buffers.
462 while (hidirtybuffers
* BKVASIZE
> 3 * hibufspace
/ 4) {
463 hidirtybuffers
>>= 1;
465 lodirtybuffers
= hidirtybuffers
/ 2;
468 * Try to keep the number of free buffers in the specified range,
469 * and give special processes (e.g. like buf_daemon) access to an
472 lofreebuffers
= nbuf
/ 18 + 5;
473 hifreebuffers
= 2 * lofreebuffers
;
474 numfreebuffers
= nbuf
;
477 * Maximum number of async ops initiated per buf_daemon loop. This is
478 * somewhat of a hack at the moment, we really need to limit ourselves
479 * based on the number of bytes of I/O in-transit that were initiated
483 bogus_offset
= kmem_alloc_pageable(kernel_map
, PAGE_SIZE
);
484 bogus_page
= vm_page_alloc(kernel_object
,
485 ((bogus_offset
- VM_MIN_KERNEL_ADDRESS
) >> PAGE_SHIFT
),
487 vmstats
.v_wire_count
++;
492 * bfreekva() - free the kva allocation for a buffer.
494 * Must be called at splbio() or higher as this is the only locking for
497 * Since this call frees up buffer space, we call bufspacewakeup().
500 bfreekva(struct buf
* bp
)
506 count
= vm_map_entry_reserve(MAP_RESERVE_COUNT
);
507 vm_map_lock(buffer_map
);
508 bufspace
-= bp
->b_kvasize
;
509 vm_map_delete(buffer_map
,
510 (vm_offset_t
) bp
->b_kvabase
,
511 (vm_offset_t
) bp
->b_kvabase
+ bp
->b_kvasize
,
514 vm_map_unlock(buffer_map
);
515 vm_map_entry_release(count
);
524 * Remove the buffer from the appropriate free list.
527 bremfree(struct buf
* bp
)
530 int old_qindex
= bp
->b_qindex
;
532 if (bp
->b_qindex
!= QUEUE_NONE
) {
533 KASSERT(BUF_REFCNTNB(bp
) == 1,
534 ("bremfree: bp %p not locked",bp
));
535 TAILQ_REMOVE(&bufqueues
[bp
->b_qindex
], bp
, b_freelist
);
536 bp
->b_qindex
= QUEUE_NONE
;
538 if (BUF_REFCNTNB(bp
) <= 1)
539 panic("bremfree: removing a buffer not on a queue");
543 * Fixup numfreebuffers count. If the buffer is invalid or not
544 * delayed-write, and it was on the EMPTY, LRU, or AGE queues,
545 * the buffer was free and we must decrement numfreebuffers.
547 if ((bp
->b_flags
& B_INVAL
) || (bp
->b_flags
& B_DELWRI
) == 0) {
564 * Get a buffer with the specified data. Look in the cache first. We
565 * must clear B_ERROR and B_INVAL prior to initiating I/O. If B_CACHE
566 * is set, the buffer is valid and we do not have to do anything ( see
570 bread(struct vnode
* vp
, daddr_t blkno
, int size
, struct buf
** bpp
)
574 bp
= getblk(vp
, blkno
, size
, 0, 0);
577 /* if not found in cache, do some I/O */
578 if ((bp
->b_flags
& B_CACHE
) == 0) {
579 KASSERT(!(bp
->b_flags
& B_ASYNC
), ("bread: illegal async bp %p", bp
));
580 bp
->b_flags
|= B_READ
;
581 bp
->b_flags
&= ~(B_ERROR
| B_INVAL
);
582 vfs_busy_pages(bp
, 0);
583 VOP_STRATEGY(vp
, bp
);
584 return (biowait(bp
));
590 * Operates like bread, but also starts asynchronous I/O on
591 * read-ahead blocks. We must clear B_ERROR and B_INVAL prior
592 * to initiating I/O . If B_CACHE is set, the buffer is valid
593 * and we do not have to do anything.
596 breadn(struct vnode
* vp
, daddr_t blkno
, int size
, daddr_t
* rablkno
,
597 int *rabsize
, int cnt
, struct buf
** bpp
)
599 struct buf
*bp
, *rabp
;
601 int rv
= 0, readwait
= 0;
603 *bpp
= bp
= getblk(vp
, blkno
, size
, 0, 0);
605 /* if not found in cache, do some I/O */
606 if ((bp
->b_flags
& B_CACHE
) == 0) {
607 bp
->b_flags
|= B_READ
;
608 bp
->b_flags
&= ~(B_ERROR
| B_INVAL
);
609 vfs_busy_pages(bp
, 0);
610 VOP_STRATEGY(vp
, bp
);
614 for (i
= 0; i
< cnt
; i
++, rablkno
++, rabsize
++) {
615 if (inmem(vp
, *rablkno
))
617 rabp
= getblk(vp
, *rablkno
, *rabsize
, 0, 0);
619 if ((rabp
->b_flags
& B_CACHE
) == 0) {
620 rabp
->b_flags
|= B_READ
| B_ASYNC
;
621 rabp
->b_flags
&= ~(B_ERROR
| B_INVAL
);
622 vfs_busy_pages(rabp
, 0);
624 VOP_STRATEGY(vp
, rabp
);
637 * Write, release buffer on completion. (Done by iodone
638 * if async). Do not bother writing anything if the buffer
641 * Note that we set B_CACHE here, indicating that buffer is
642 * fully valid and thus cacheable. This is true even of NFS
643 * now so we set it generally. This could be set either here
644 * or in biodone() since the I/O is synchronous. We put it
648 bwrite(struct buf
* bp
)
655 if (bp
->b_flags
& B_INVAL
) {
660 oldflags
= bp
->b_flags
;
662 if (BUF_REFCNTNB(bp
) == 0)
663 panic("bwrite: buffer is not busy???");
666 * If a background write is already in progress, delay
667 * writing this block if it is asynchronous. Otherwise
668 * wait for the background write to complete.
670 if (bp
->b_xflags
& BX_BKGRDINPROG
) {
671 if (bp
->b_flags
& B_ASYNC
) {
676 bp
->b_xflags
|= BX_BKGRDWAIT
;
677 tsleep(&bp
->b_xflags
, 0, "biord", 0);
678 if (bp
->b_xflags
& BX_BKGRDINPROG
)
679 panic("bwrite: still writing");
682 /* Mark the buffer clean */
687 * If this buffer is marked for background writing and we
688 * do not have to wait for it, make a copy and write the
689 * copy so as to leave this buffer ready for further use.
691 * This optimization eats a lot of memory. If we have a page
692 * or buffer shortfull we can't do it.
694 * XXX DISABLED! This had to be removed to support the RB_TREE
695 * work and, really, this isn't the best place to do this sort
696 * of thing anyway. We really need a device copy-on-write feature.
699 (bp
->b_xflags
& BX_BKGRDWRITE
) &&
700 (bp
->b_flags
& B_ASYNC
) &&
701 !vm_page_count_severe() &&
702 !buf_dirty_count_severe()) {
703 if (bp
->b_flags
& B_CALL
)
704 panic("bwrite: need chained iodone");
706 /* get a new block */
707 newbp
= geteblk(bp
->b_bufsize
);
709 /* set it to be identical to the old block */
710 memcpy(newbp
->b_data
, bp
->b_data
, bp
->b_bufsize
);
711 newbp
->b_lblkno
= bp
->b_lblkno
;
712 newbp
->b_blkno
= bp
->b_blkno
;
713 newbp
->b_offset
= bp
->b_offset
;
714 newbp
->b_iodone
= vfs_backgroundwritedone
;
715 newbp
->b_flags
|= B_ASYNC
| B_CALL
;
716 newbp
->b_flags
&= ~B_INVAL
;
717 bgetvp(bp
->b_vp
, newbp
);
719 /* move over the dependencies */
720 if (LIST_FIRST(&bp
->b_dep
) != NULL
&& bioops
.io_movedeps
)
721 (*bioops
.io_movedeps
)(bp
, newbp
);
724 * Initiate write on the copy, release the original to
725 * the B_LOCKED queue so that it cannot go away until
726 * the background write completes. If not locked it could go
727 * away and then be reconstituted while it was being written.
728 * If the reconstituted buffer were written, we could end up
729 * with two background copies being written at the same time.
731 bp
->b_xflags
|= BX_BKGRDINPROG
;
732 bp
->b_flags
|= B_LOCKED
;
738 bp
->b_flags
&= ~(B_READ
| B_DONE
| B_ERROR
);
739 bp
->b_flags
|= B_CACHE
;
741 bp
->b_vp
->v_numoutput
++;
742 vfs_busy_pages(bp
, 1);
745 * Normal bwrites pipeline writes
747 bp
->b_runningbufspace
= bp
->b_bufsize
;
748 runningbufspace
+= bp
->b_runningbufspace
;
751 if (oldflags
& B_ASYNC
)
753 VOP_STRATEGY(bp
->b_vp
, bp
);
755 if ((oldflags
& B_ASYNC
) == 0) {
756 int rtval
= biowait(bp
);
759 } else if ((oldflags
& B_NOWDRAIN
) == 0) {
761 * don't allow the async write to saturate the I/O
762 * system. Deadlocks can occur only if a device strategy
763 * routine (like in VN) turns around and issues another
764 * high-level write, in which case B_NOWDRAIN is expected
765 * to be set. Otherwise we will not deadlock here because
766 * we are blocking waiting for I/O that is already in-progress
769 waitrunningbufspace();
777 * Complete a background write started from bwrite.
780 vfs_backgroundwritedone(struct buf
*bp
)
785 * Find the original buffer that we are writing.
787 if ((origbp
= gbincore(bp
->b_vp
, bp
->b_lblkno
)) == NULL
)
788 panic("backgroundwritedone: lost buffer");
790 * Process dependencies then return any unfinished ones.
792 if (LIST_FIRST(&bp
->b_dep
) != NULL
&& bioops
.io_complete
)
793 (*bioops
.io_complete
)(bp
);
794 if (LIST_FIRST(&bp
->b_dep
) != NULL
&& bioops
.io_movedeps
)
795 (*bioops
.io_movedeps
)(bp
, origbp
);
797 * Clear the BX_BKGRDINPROG flag in the original buffer
798 * and awaken it if it is waiting for the write to complete.
799 * If BX_BKGRDINPROG is not set in the original buffer it must
800 * have been released and re-instantiated - which is not legal.
802 KASSERT((origbp
->b_xflags
& BX_BKGRDINPROG
), ("backgroundwritedone: lost buffer2"));
803 origbp
->b_xflags
&= ~BX_BKGRDINPROG
;
804 if (origbp
->b_xflags
& BX_BKGRDWAIT
) {
805 origbp
->b_xflags
&= ~BX_BKGRDWAIT
;
806 wakeup(&origbp
->b_xflags
);
809 * Clear the B_LOCKED flag and remove it from the locked
810 * queue if it currently resides there.
812 origbp
->b_flags
&= ~B_LOCKED
;
813 if (BUF_LOCK(origbp
, LK_EXCLUSIVE
| LK_NOWAIT
) == 0) {
818 * This buffer is marked B_NOCACHE, so when it is released
819 * by biodone, it will be tossed. We mark it with B_READ
820 * to avoid biodone doing a second vwakeup.
822 bp
->b_flags
|= B_NOCACHE
| B_READ
;
823 bp
->b_flags
&= ~(B_CACHE
| B_CALL
| B_DONE
);
830 * Delayed write. (Buffer is marked dirty). Do not bother writing
831 * anything if the buffer is marked invalid.
833 * Note that since the buffer must be completely valid, we can safely
834 * set B_CACHE. In fact, we have to set B_CACHE here rather then in
835 * biodone() in order to prevent getblk from writing the buffer
839 bdwrite(struct buf
*bp
)
841 if (BUF_REFCNTNB(bp
) == 0)
842 panic("bdwrite: buffer is not busy");
844 if (bp
->b_flags
& B_INVAL
) {
851 * Set B_CACHE, indicating that the buffer is fully valid. This is
852 * true even of NFS now.
854 bp
->b_flags
|= B_CACHE
;
857 * This bmap keeps the system from needing to do the bmap later,
858 * perhaps when the system is attempting to do a sync. Since it
859 * is likely that the indirect block -- or whatever other datastructure
860 * that the filesystem needs is still in memory now, it is a good
861 * thing to do this. Note also, that if the pageout daemon is
862 * requesting a sync -- there might not be enough memory to do
863 * the bmap then... So, this is important to do.
865 if (bp
->b_lblkno
== bp
->b_blkno
) {
866 VOP_BMAP(bp
->b_vp
, bp
->b_lblkno
, NULL
, &bp
->b_blkno
, NULL
, NULL
);
870 * Set the *dirty* buffer range based upon the VM system dirty pages.
875 * We need to do this here to satisfy the vnode_pager and the
876 * pageout daemon, so that it thinks that the pages have been
877 * "cleaned". Note that since the pages are in a delayed write
878 * buffer -- the VFS layer "will" see that the pages get written
879 * out on the next sync, or perhaps the cluster will be completed.
885 * Wakeup the buffer flushing daemon if we have a lot of dirty
886 * buffers (midpoint between our recovery point and our stall
889 bd_wakeup((lodirtybuffers
+ hidirtybuffers
) / 2);
892 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
893 * due to the softdep code.
900 * Turn buffer into delayed write request. We must clear B_READ and
901 * B_RELBUF, and we must set B_DELWRI. We reassign the buffer to
902 * itself to properly update it in the dirty/clean lists. We mark it
903 * B_DONE to ensure that any asynchronization of the buffer properly
904 * clears B_DONE ( else a panic will occur later ).
906 * bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
907 * might have been set pre-getblk(). Unlike bwrite/bdwrite, bdirty()
908 * should only be called if the buffer is known-good.
910 * Since the buffer is not on a queue, we do not update the numfreebuffers
913 * Must be called at splbio().
914 * The buffer must be on QUEUE_NONE.
917 bdirty(struct buf
*bp
)
919 KASSERT(bp
->b_qindex
== QUEUE_NONE
, ("bdirty: buffer %p still on queue %d", bp
, bp
->b_qindex
));
920 bp
->b_flags
&= ~(B_READ
|B_RELBUF
);
922 if ((bp
->b_flags
& B_DELWRI
) == 0) {
923 bp
->b_flags
|= B_DONE
| B_DELWRI
;
924 reassignbuf(bp
, bp
->b_vp
);
926 bd_wakeup((lodirtybuffers
+ hidirtybuffers
) / 2);
933 * Clear B_DELWRI for buffer.
935 * Since the buffer is not on a queue, we do not update the numfreebuffers
938 * Must be called at splbio().
940 * The buffer is typically on QUEUE_NONE but there is one case in
941 * brelse() that calls this function after placing the buffer on
946 bundirty(struct buf
*bp
)
948 if (bp
->b_flags
& B_DELWRI
) {
949 bp
->b_flags
&= ~B_DELWRI
;
950 reassignbuf(bp
, bp
->b_vp
);
952 numdirtywakeup(lodirtybuffers
);
955 * Since it is now being written, we can clear its deferred write flag.
957 bp
->b_flags
&= ~B_DEFERRED
;
963 * Asynchronous write. Start output on a buffer, but do not wait for
964 * it to complete. The buffer is released when the output completes.
966 * bwrite() ( or the VOP routine anyway ) is responsible for handling
967 * B_INVAL buffers. Not us.
970 bawrite(struct buf
* bp
)
972 bp
->b_flags
|= B_ASYNC
;
973 (void) VOP_BWRITE(bp
->b_vp
, bp
);
979 * Ordered write. Start output on a buffer, and flag it so that the
980 * device will write it in the order it was queued. The buffer is
981 * released when the output completes. bwrite() ( or the VOP routine
982 * anyway ) is responsible for handling B_INVAL buffers.
985 bowrite(struct buf
* bp
)
987 bp
->b_flags
|= B_ORDERED
| B_ASYNC
;
988 return (VOP_BWRITE(bp
->b_vp
, bp
));
994 * Called prior to the locking of any vnodes when we are expecting to
995 * write. We do not want to starve the buffer cache with too many
996 * dirty buffers so we block here. By blocking prior to the locking
997 * of any vnodes we attempt to avoid the situation where a locked vnode
998 * prevents the various system daemons from flushing related buffers.
1004 if (numdirtybuffers
>= hidirtybuffers
) {
1008 while (numdirtybuffers
>= hidirtybuffers
) {
1010 needsbuffer
|= VFS_BIO_NEED_DIRTYFLUSH
;
1011 tsleep(&needsbuffer
, 0, "flswai", 0);
1018 * Return true if we have too many dirty buffers.
1021 buf_dirty_count_severe(void)
1023 return(numdirtybuffers
>= hidirtybuffers
);
1029 * Release a busy buffer and, if requested, free its resources. The
1030 * buffer will be stashed in the appropriate bufqueue[] allowing it
1031 * to be accessed later as a cache entity or reused for other purposes.
1034 brelse(struct buf
* bp
)
1038 KASSERT(!(bp
->b_flags
& (B_CLUSTER
|B_PAGING
)), ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp
));
1042 if (bp
->b_flags
& B_LOCKED
)
1043 bp
->b_flags
&= ~B_ERROR
;
1045 if ((bp
->b_flags
& (B_READ
| B_ERROR
| B_INVAL
)) == B_ERROR
) {
1047 * Failed write, redirty. Must clear B_ERROR to prevent
1048 * pages from being scrapped. If B_INVAL is set then
1049 * this case is not run and the next case is run to
1050 * destroy the buffer. B_INVAL can occur if the buffer
1051 * is outside the range supported by the underlying device.
1053 bp
->b_flags
&= ~B_ERROR
;
1055 } else if ((bp
->b_flags
& (B_NOCACHE
| B_INVAL
| B_ERROR
| B_FREEBUF
)) ||
1056 (bp
->b_bufsize
<= 0)) {
1058 * Either a failed I/O or we were asked to free or not
1061 bp
->b_flags
|= B_INVAL
;
1062 if (LIST_FIRST(&bp
->b_dep
) != NULL
&& bioops
.io_deallocate
)
1063 (*bioops
.io_deallocate
)(bp
);
1064 if (bp
->b_flags
& B_DELWRI
) {
1066 numdirtywakeup(lodirtybuffers
);
1068 bp
->b_flags
&= ~(B_DELWRI
| B_CACHE
| B_FREEBUF
);
1069 if ((bp
->b_flags
& B_VMIO
) == 0) {
1078 * We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_release()
1079 * is called with B_DELWRI set, the underlying pages may wind up
1080 * getting freed causing a previous write (bdwrite()) to get 'lost'
1081 * because pages associated with a B_DELWRI bp are marked clean.
1083 * We still allow the B_INVAL case to call vfs_vmio_release(), even
1084 * if B_DELWRI is set.
1086 * If B_DELWRI is not set we may have to set B_RELBUF if we are low
1087 * on pages to return pages to the VM page queues.
1089 if (bp
->b_flags
& B_DELWRI
)
1090 bp
->b_flags
&= ~B_RELBUF
;
1091 else if (vm_page_count_severe() && !(bp
->b_xflags
& BX_BKGRDINPROG
))
1092 bp
->b_flags
|= B_RELBUF
;
1095 * VMIO buffer rundown. It is not very necessary to keep a VMIO buffer
1096 * constituted, not even NFS buffers now. Two flags effect this. If
1097 * B_INVAL, the struct buf is invalidated but the VM object is kept
1098 * around ( i.e. so it is trivial to reconstitute the buffer later ).
1100 * If B_ERROR or B_NOCACHE is set, pages in the VM object will be
1101 * invalidated. B_ERROR cannot be set for a failed write unless the
1102 * buffer is also B_INVAL because it hits the re-dirtying code above.
1104 * Normally we can do this whether a buffer is B_DELWRI or not. If
1105 * the buffer is an NFS buffer, it is tracking piecemeal writes or
1106 * the commit state and we cannot afford to lose the buffer. If the
1107 * buffer has a background write in progress, we need to keep it
1108 * around to prevent it from being reconstituted and starting a second
1111 if ((bp
->b_flags
& B_VMIO
)
1112 && !(bp
->b_vp
->v_tag
== VT_NFS
&&
1113 !vn_isdisk(bp
->b_vp
, NULL
) &&
1114 (bp
->b_flags
& B_DELWRI
))
1127 * Get the base offset and length of the buffer. Note that
1128 * in the VMIO case if the buffer block size is not
1129 * page-aligned then b_data pointer may not be page-aligned.
1130 * But our b_xio.xio_pages array *IS* page aligned.
1132 * block sizes less then DEV_BSIZE (usually 512) are not
1133 * supported due to the page granularity bits (m->valid,
1134 * m->dirty, etc...).
1136 * See man buf(9) for more information
1139 resid
= bp
->b_bufsize
;
1140 foff
= bp
->b_offset
;
1142 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
1143 m
= bp
->b_xio
.xio_pages
[i
];
1144 vm_page_flag_clear(m
, PG_ZERO
);
1146 * If we hit a bogus page, fixup *all* of them
1147 * now. Note that we left these pages wired
1148 * when we removed them so they had better exist,
1149 * and they cannot be ripped out from under us so
1150 * no splvm() protection is necessary.
1152 if (m
== bogus_page
) {
1153 VOP_GETVOBJECT(vp
, &obj
);
1154 poff
= OFF_TO_IDX(bp
->b_offset
);
1156 for (j
= i
; j
< bp
->b_xio
.xio_npages
; j
++) {
1159 mtmp
= bp
->b_xio
.xio_pages
[j
];
1160 if (mtmp
== bogus_page
) {
1161 mtmp
= vm_page_lookup(obj
, poff
+ j
);
1163 panic("brelse: page missing");
1165 bp
->b_xio
.xio_pages
[j
] = mtmp
;
1169 if ((bp
->b_flags
& B_INVAL
) == 0) {
1170 pmap_qenter(trunc_page((vm_offset_t
)bp
->b_data
),
1171 bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
1173 m
= bp
->b_xio
.xio_pages
[i
];
1177 * Invalidate the backing store if B_NOCACHE is set
1178 * (e.g. used with vinvalbuf()). If this is NFS
1179 * we impose a requirement that the block size be
1180 * a multiple of PAGE_SIZE and create a temporary
1181 * hack to basically invalidate the whole page. The
1182 * problem is that NFS uses really odd buffer sizes
1183 * especially when tracking piecemeal writes and
1184 * it also vinvalbuf()'s a lot, which would result
1185 * in only partial page validation and invalidation
1186 * here. If the file page is mmap()'d, however,
1187 * all the valid bits get set so after we invalidate
1188 * here we would end up with weird m->valid values
1189 * like 0xfc. nfs_getpages() can't handle this so
1190 * we clear all the valid bits for the NFS case
1191 * instead of just some of them.
1193 * The real bug is the VM system having to set m->valid
1194 * to VM_PAGE_BITS_ALL for faulted-in pages, which
1195 * itself is an artifact of the whole 512-byte
1196 * granular mess that exists to support odd block
1197 * sizes and UFS meta-data block sizes (e.g. 6144).
1198 * A complete rewrite is required.
1200 if (bp
->b_flags
& (B_NOCACHE
|B_ERROR
)) {
1201 int poffset
= foff
& PAGE_MASK
;
1204 presid
= PAGE_SIZE
- poffset
;
1205 if (bp
->b_vp
->v_tag
== VT_NFS
&&
1206 bp
->b_vp
->v_type
== VREG
) {
1208 } else if (presid
> resid
) {
1211 KASSERT(presid
>= 0, ("brelse: extra page"));
1212 vm_page_set_invalid(m
, poffset
, presid
);
1214 resid
-= PAGE_SIZE
- (foff
& PAGE_MASK
);
1215 foff
= (foff
+ PAGE_SIZE
) & ~(off_t
)PAGE_MASK
;
1218 if (bp
->b_flags
& (B_INVAL
| B_RELBUF
))
1219 vfs_vmio_release(bp
);
1221 } else if (bp
->b_flags
& B_VMIO
) {
1223 if (bp
->b_flags
& (B_INVAL
| B_RELBUF
))
1224 vfs_vmio_release(bp
);
1228 if (bp
->b_qindex
!= QUEUE_NONE
)
1229 panic("brelse: free buffer onto another queue???");
1230 if (BUF_REFCNTNB(bp
) > 1) {
1231 /* Temporary panic to verify exclusive locking */
1232 /* This panic goes away when we allow shared refs */
1233 panic("brelse: multiple refs");
1234 /* do not release to free list */
1242 /* buffers with no memory */
1243 if (bp
->b_bufsize
== 0) {
1244 bp
->b_flags
|= B_INVAL
;
1245 bp
->b_xflags
&= ~BX_BKGRDWRITE
;
1246 if (bp
->b_xflags
& BX_BKGRDINPROG
)
1247 panic("losing buffer 1");
1248 if (bp
->b_kvasize
) {
1249 bp
->b_qindex
= QUEUE_EMPTYKVA
;
1251 bp
->b_qindex
= QUEUE_EMPTY
;
1253 TAILQ_INSERT_HEAD(&bufqueues
[bp
->b_qindex
], bp
, b_freelist
);
1254 LIST_REMOVE(bp
, b_hash
);
1255 LIST_INSERT_HEAD(&invalhash
, bp
, b_hash
);
1257 /* buffers with junk contents */
1258 } else if (bp
->b_flags
& (B_ERROR
| B_INVAL
| B_NOCACHE
| B_RELBUF
)) {
1259 bp
->b_flags
|= B_INVAL
;
1260 bp
->b_xflags
&= ~BX_BKGRDWRITE
;
1261 if (bp
->b_xflags
& BX_BKGRDINPROG
)
1262 panic("losing buffer 2");
1263 bp
->b_qindex
= QUEUE_CLEAN
;
1264 TAILQ_INSERT_HEAD(&bufqueues
[QUEUE_CLEAN
], bp
, b_freelist
);
1265 LIST_REMOVE(bp
, b_hash
);
1266 LIST_INSERT_HEAD(&invalhash
, bp
, b_hash
);
1269 /* buffers that are locked */
1270 } else if (bp
->b_flags
& B_LOCKED
) {
1271 bp
->b_qindex
= QUEUE_LOCKED
;
1272 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_LOCKED
], bp
, b_freelist
);
1274 /* remaining buffers */
1276 switch(bp
->b_flags
& (B_DELWRI
|B_AGE
)) {
1277 case B_DELWRI
| B_AGE
:
1278 bp
->b_qindex
= QUEUE_DIRTY
;
1279 TAILQ_INSERT_HEAD(&bufqueues
[QUEUE_DIRTY
], bp
, b_freelist
);
1282 bp
->b_qindex
= QUEUE_DIRTY
;
1283 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_DIRTY
], bp
, b_freelist
);
1286 bp
->b_qindex
= QUEUE_CLEAN
;
1287 TAILQ_INSERT_HEAD(&bufqueues
[QUEUE_CLEAN
], bp
, b_freelist
);
1290 bp
->b_qindex
= QUEUE_CLEAN
;
1291 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_CLEAN
], bp
, b_freelist
);
1297 * If B_INVAL, clear B_DELWRI. We've already placed the buffer
1298 * on the correct queue.
1300 if ((bp
->b_flags
& (B_INVAL
|B_DELWRI
)) == (B_INVAL
|B_DELWRI
))
1304 * Fixup numfreebuffers count. The bp is on an appropriate queue
1305 * unless locked. We then bump numfreebuffers if it is not B_DELWRI.
1306 * We've already handled the B_INVAL case ( B_DELWRI will be clear
1307 * if B_INVAL is set ).
1310 if ((bp
->b_flags
& B_LOCKED
) == 0 && !(bp
->b_flags
& B_DELWRI
))
1314 * Something we can maybe free or reuse
1316 if (bp
->b_bufsize
|| bp
->b_kvasize
)
1321 bp
->b_flags
&= ~(B_ORDERED
| B_ASYNC
| B_NOCACHE
| B_AGE
| B_RELBUF
|
1322 B_DIRECT
| B_NOWDRAIN
);
1327 * Release a buffer back to the appropriate queue but do not try to free
1328 * it. The buffer is expected to be used again soon.
1330 * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
1331 * biodone() to requeue an async I/O on completion. It is also used when
1332 * known good buffers need to be requeued but we think we may need the data
1335 * XXX we should be able to leave the B_RELBUF hint set on completion.
1338 bqrelse(struct buf
* bp
)
1344 KASSERT(!(bp
->b_flags
& (B_CLUSTER
|B_PAGING
)), ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp
));
1346 if (bp
->b_qindex
!= QUEUE_NONE
)
1347 panic("bqrelse: free buffer onto another queue???");
1348 if (BUF_REFCNTNB(bp
) > 1) {
1349 /* do not release to free list */
1350 panic("bqrelse: multiple refs");
1355 if (bp
->b_flags
& B_LOCKED
) {
1356 bp
->b_flags
&= ~B_ERROR
;
1357 bp
->b_qindex
= QUEUE_LOCKED
;
1358 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_LOCKED
], bp
, b_freelist
);
1359 /* buffers with stale but valid contents */
1360 } else if (bp
->b_flags
& B_DELWRI
) {
1361 bp
->b_qindex
= QUEUE_DIRTY
;
1362 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_DIRTY
], bp
, b_freelist
);
1363 } else if (vm_page_count_severe()) {
1365 * We are too low on memory, we have to try to free the
1366 * buffer (most importantly: the wired pages making up its
1367 * backing store) *now*.
1373 bp
->b_qindex
= QUEUE_CLEAN
;
1374 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_CLEAN
], bp
, b_freelist
);
1377 if ((bp
->b_flags
& B_LOCKED
) == 0 &&
1378 ((bp
->b_flags
& B_INVAL
) || !(bp
->b_flags
& B_DELWRI
))) {
1383 * Something we can maybe free or reuse.
1385 if (bp
->b_bufsize
&& !(bp
->b_flags
& B_DELWRI
))
1390 bp
->b_flags
&= ~(B_ORDERED
| B_ASYNC
| B_NOCACHE
| B_AGE
| B_RELBUF
);
1395 vfs_vmio_release(struct buf
*bp
)
1401 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
1402 m
= bp
->b_xio
.xio_pages
[i
];
1403 bp
->b_xio
.xio_pages
[i
] = NULL
;
1405 * In order to keep page LRU ordering consistent, put
1406 * everything on the inactive queue.
1408 vm_page_unwire(m
, 0);
1410 * We don't mess with busy pages, it is
1411 * the responsibility of the process that
1412 * busied the pages to deal with them.
1414 if ((m
->flags
& PG_BUSY
) || (m
->busy
!= 0))
1417 if (m
->wire_count
== 0) {
1418 vm_page_flag_clear(m
, PG_ZERO
);
1420 * Might as well free the page if we can and it has
1421 * no valid data. We also free the page if the
1422 * buffer was used for direct I/O.
1424 if ((bp
->b_flags
& B_ASYNC
) == 0 && !m
->valid
&& m
->hold_count
== 0) {
1426 vm_page_protect(m
, VM_PROT_NONE
);
1428 } else if (bp
->b_flags
& B_DIRECT
) {
1429 vm_page_try_to_free(m
);
1430 } else if (vm_page_count_severe()) {
1431 vm_page_try_to_cache(m
);
1436 pmap_qremove(trunc_page((vm_offset_t
) bp
->b_data
), bp
->b_xio
.xio_npages
);
1437 if (bp
->b_bufsize
) {
1441 bp
->b_xio
.xio_npages
= 0;
1442 bp
->b_flags
&= ~B_VMIO
;
1448 * Check to see if a block is currently memory resident.
1451 gbincore(struct vnode
* vp
, daddr_t blkno
)
1454 struct bufhashhdr
*bh
;
1456 bh
= bufhash(vp
, blkno
);
1458 /* Search hash chain */
1459 LIST_FOREACH(bp
, bh
, b_hash
) {
1461 if (bp
->b_vp
== vp
&& bp
->b_lblkno
== blkno
)
1470 * Implement clustered async writes for clearing out B_DELWRI buffers.
1471 * This is much better then the old way of writing only one buffer at
1472 * a time. Note that we may not be presented with the buffers in the
1473 * correct order, so we search for the cluster in both directions.
1476 vfs_bio_awrite(struct buf
* bp
)
1480 daddr_t lblkno
= bp
->b_lblkno
;
1481 struct vnode
*vp
= bp
->b_vp
;
1491 * right now we support clustered writing only to regular files. If
1492 * we find a clusterable block we could be in the middle of a cluster
1493 * rather then at the beginning.
1495 if ((vp
->v_type
== VREG
) &&
1496 (vp
->v_mount
!= 0) && /* Only on nodes that have the size info */
1497 (bp
->b_flags
& (B_CLUSTEROK
| B_INVAL
)) == B_CLUSTEROK
) {
1499 size
= vp
->v_mount
->mnt_stat
.f_iosize
;
1500 maxcl
= MAXPHYS
/ size
;
1502 for (i
= 1; i
< maxcl
; i
++) {
1503 if ((bpa
= gbincore(vp
, lblkno
+ i
)) &&
1504 BUF_REFCNT(bpa
) == 0 &&
1505 ((bpa
->b_flags
& (B_DELWRI
| B_CLUSTEROK
| B_INVAL
)) ==
1506 (B_DELWRI
| B_CLUSTEROK
)) &&
1507 (bpa
->b_bufsize
== size
)) {
1508 if ((bpa
->b_blkno
== bpa
->b_lblkno
) ||
1510 bp
->b_blkno
+ ((i
* size
) >> DEV_BSHIFT
)))
1516 for (j
= 1; i
+ j
<= maxcl
&& j
<= lblkno
; j
++) {
1517 if ((bpa
= gbincore(vp
, lblkno
- j
)) &&
1518 BUF_REFCNT(bpa
) == 0 &&
1519 ((bpa
->b_flags
& (B_DELWRI
| B_CLUSTEROK
| B_INVAL
)) ==
1520 (B_DELWRI
| B_CLUSTEROK
)) &&
1521 (bpa
->b_bufsize
== size
)) {
1522 if ((bpa
->b_blkno
== bpa
->b_lblkno
) ||
1524 bp
->b_blkno
- ((j
* size
) >> DEV_BSHIFT
)))
1533 * this is a possible cluster write
1536 nwritten
= cluster_wbuild(vp
, size
, lblkno
- j
, ncl
);
1542 BUF_LOCK(bp
, LK_EXCLUSIVE
);
1544 bp
->b_flags
|= B_ASYNC
;
1548 * default (old) behavior, writing out only one block
1550 * XXX returns b_bufsize instead of b_bcount for nwritten?
1552 nwritten
= bp
->b_bufsize
;
1553 (void) VOP_BWRITE(bp
->b_vp
, bp
);
1561 * Find and initialize a new buffer header, freeing up existing buffers
1562 * in the bufqueues as necessary. The new buffer is returned locked.
1564 * Important: B_INVAL is not set. If the caller wishes to throw the
1565 * buffer away, the caller must set B_INVAL prior to calling brelse().
1568 * We have insufficient buffer headers
1569 * We have insufficient buffer space
1570 * buffer_map is too fragmented ( space reservation fails )
1571 * If we have to flush dirty buffers ( but we try to avoid this )
1573 * To avoid VFS layer recursion we do not flush dirty buffers ourselves.
1574 * Instead we ask the buf daemon to do it for us. We attempt to
1575 * avoid piecemeal wakeups of the pageout daemon.
1579 getnewbuf(int slpflag
, int slptimeo
, int size
, int maxsize
)
1585 static int flushingbufs
;
1588 * We can't afford to block since we might be holding a vnode lock,
1589 * which may prevent system daemons from running. We deal with
1590 * low-memory situations by proactively returning memory and running
1591 * async I/O rather then sync I/O.
1595 --getnewbufrestarts
;
1597 ++getnewbufrestarts
;
1600 * Setup for scan. If we do not have enough free buffers,
1601 * we setup a degenerate case that immediately fails. Note
1602 * that if we are specially marked process, we are allowed to
1603 * dip into our reserves.
1605 * The scanning sequence is nominally: EMPTY->EMPTYKVA->CLEAN
1607 * We start with EMPTYKVA. If the list is empty we backup to EMPTY.
1608 * However, there are a number of cases (defragging, reusing, ...)
1609 * where we cannot backup.
1611 nqindex
= QUEUE_EMPTYKVA
;
1612 nbp
= TAILQ_FIRST(&bufqueues
[QUEUE_EMPTYKVA
]);
1616 * If no EMPTYKVA buffers and we are either
1617 * defragging or reusing, locate a CLEAN buffer
1618 * to free or reuse. If bufspace useage is low
1619 * skip this step so we can allocate a new buffer.
1621 if (defrag
|| bufspace
>= lobufspace
) {
1622 nqindex
= QUEUE_CLEAN
;
1623 nbp
= TAILQ_FIRST(&bufqueues
[QUEUE_CLEAN
]);
1627 * If we could not find or were not allowed to reuse a
1628 * CLEAN buffer, check to see if it is ok to use an EMPTY
1629 * buffer. We can only use an EMPTY buffer if allocating
1630 * its KVA would not otherwise run us out of buffer space.
1632 if (nbp
== NULL
&& defrag
== 0 &&
1633 bufspace
+ maxsize
< hibufspace
) {
1634 nqindex
= QUEUE_EMPTY
;
1635 nbp
= TAILQ_FIRST(&bufqueues
[QUEUE_EMPTY
]);
1640 * Run scan, possibly freeing data and/or kva mappings on the fly
1644 while ((bp
= nbp
) != NULL
) {
1645 int qindex
= nqindex
;
1648 * Calculate next bp ( we can only use it if we do not block
1649 * or do other fancy things ).
1651 if ((nbp
= TAILQ_NEXT(bp
, b_freelist
)) == NULL
) {
1654 nqindex
= QUEUE_EMPTYKVA
;
1655 if ((nbp
= TAILQ_FIRST(&bufqueues
[QUEUE_EMPTYKVA
])))
1658 case QUEUE_EMPTYKVA
:
1659 nqindex
= QUEUE_CLEAN
;
1660 if ((nbp
= TAILQ_FIRST(&bufqueues
[QUEUE_CLEAN
])))
1674 KASSERT(bp
->b_qindex
== qindex
, ("getnewbuf: inconsistant queue %d bp %p", qindex
, bp
));
1677 * Note: we no longer distinguish between VMIO and non-VMIO
1681 KASSERT((bp
->b_flags
& B_DELWRI
) == 0, ("delwri buffer %p found in queue %d", bp
, qindex
));
1684 * If we are defragging then we need a buffer with
1685 * b_kvasize != 0. XXX this situation should no longer
1686 * occur, if defrag is non-zero the buffer's b_kvasize
1687 * should also be non-zero at this point. XXX
1689 if (defrag
&& bp
->b_kvasize
== 0) {
1690 printf("Warning: defrag empty buffer %p\n", bp
);
1695 * Start freeing the bp. This is somewhat involved. nbp
1696 * remains valid only for QUEUE_EMPTY[KVA] bp's.
1699 if (BUF_LOCK(bp
, LK_EXCLUSIVE
| LK_NOWAIT
) != 0)
1700 panic("getnewbuf: locked buf");
1703 if (qindex
== QUEUE_CLEAN
) {
1704 if (bp
->b_flags
& B_VMIO
) {
1705 bp
->b_flags
&= ~B_ASYNC
;
1706 vfs_vmio_release(bp
);
1713 * NOTE: nbp is now entirely invalid. We can only restart
1714 * the scan from this point on.
1716 * Get the rest of the buffer freed up. b_kva* is still
1717 * valid after this operation.
1720 if (LIST_FIRST(&bp
->b_dep
) != NULL
&& bioops
.io_deallocate
)
1721 (*bioops
.io_deallocate
)(bp
);
1722 if (bp
->b_xflags
& BX_BKGRDINPROG
)
1723 panic("losing buffer 3");
1724 LIST_REMOVE(bp
, b_hash
);
1725 LIST_INSERT_HEAD(&invalhash
, bp
, b_hash
);
1728 * spl protection not required when scrapping a buffer's
1729 * contents because it is already wired.
1738 bp
->b_blkno
= bp
->b_lblkno
= 0;
1739 bp
->b_offset
= NOOFFSET
;
1744 bp
->b_xio
.xio_npages
= 0;
1745 bp
->b_dirtyoff
= bp
->b_dirtyend
= 0;
1747 LIST_INIT(&bp
->b_dep
);
1750 * If we are defragging then free the buffer.
1753 bp
->b_flags
|= B_INVAL
;
1761 * If we are overcomitted then recover the buffer and its
1762 * KVM space. This occurs in rare situations when multiple
1763 * processes are blocked in getnewbuf() or allocbuf().
1765 if (bufspace
>= hibufspace
)
1767 if (flushingbufs
&& bp
->b_kvasize
!= 0) {
1768 bp
->b_flags
|= B_INVAL
;
1773 if (bufspace
< lobufspace
)
1779 * If we exhausted our list, sleep as appropriate. We may have to
1780 * wakeup various daemons and write out some dirty buffers.
1782 * Generally we are sleeping due to insufficient buffer space.
1790 flags
= VFS_BIO_NEED_BUFSPACE
;
1792 } else if (bufspace
>= hibufspace
) {
1794 flags
= VFS_BIO_NEED_BUFSPACE
;
1797 flags
= VFS_BIO_NEED_ANY
;
1800 bd_speedup(); /* heeeelp */
1802 needsbuffer
|= flags
;
1803 while (needsbuffer
& flags
) {
1804 if (tsleep(&needsbuffer
, slpflag
, waitmsg
, slptimeo
))
1809 * We finally have a valid bp. We aren't quite out of the
1810 * woods, we still have to reserve kva space. In order
1811 * to keep fragmentation sane we only allocate kva in
1814 maxsize
= (maxsize
+ BKVAMASK
) & ~BKVAMASK
;
1816 if (maxsize
!= bp
->b_kvasize
) {
1817 vm_offset_t addr
= 0;
1822 count
= vm_map_entry_reserve(MAP_RESERVE_COUNT
);
1823 vm_map_lock(buffer_map
);
1825 if (vm_map_findspace(buffer_map
,
1826 vm_map_min(buffer_map
), maxsize
,
1829 * Uh oh. Buffer map is to fragmented. We
1830 * must defragment the map.
1832 vm_map_unlock(buffer_map
);
1833 vm_map_entry_release(count
);
1836 bp
->b_flags
|= B_INVAL
;
1841 vm_map_insert(buffer_map
, &count
,
1843 addr
, addr
+ maxsize
,
1844 VM_PROT_ALL
, VM_PROT_ALL
, MAP_NOFAULT
);
1846 bp
->b_kvabase
= (caddr_t
) addr
;
1847 bp
->b_kvasize
= maxsize
;
1848 bufspace
+= bp
->b_kvasize
;
1851 vm_map_unlock(buffer_map
);
1852 vm_map_entry_release(count
);
1854 bp
->b_data
= bp
->b_kvabase
;
1862 * buffer flushing daemon. Buffers are normally flushed by the
1863 * update daemon but if it cannot keep up this process starts to
1864 * take the load in an attempt to prevent getnewbuf() from blocking.
1867 static struct thread
*bufdaemonthread
;
1869 static struct kproc_desc buf_kp
= {
1874 SYSINIT(bufdaemon
, SI_SUB_KTHREAD_BUF
, SI_ORDER_FIRST
, kproc_start
, &buf_kp
)
1882 * This process needs to be suspended prior to shutdown sync.
1884 EVENTHANDLER_REGISTER(shutdown_pre_sync
, shutdown_kproc
,
1885 bufdaemonthread
, SHUTDOWN_PRI_LAST
);
1888 * This process is allowed to take the buffer cache to the limit
1893 kproc_suspend_loop();
1896 * Do the flush. Limit the amount of in-transit I/O we
1897 * allow to build up, otherwise we would completely saturate
1898 * the I/O system. Wakeup any waiting processes before we
1899 * normally would so they can run in parallel with our drain.
1901 while (numdirtybuffers
> lodirtybuffers
) {
1902 if (flushbufqueues() == 0)
1904 waitrunningbufspace();
1905 numdirtywakeup((lodirtybuffers
+ hidirtybuffers
) / 2);
1909 * Only clear bd_request if we have reached our low water
1910 * mark. The buf_daemon normally waits 5 seconds and
1911 * then incrementally flushes any dirty buffers that have
1912 * built up, within reason.
1914 * If we were unable to hit our low water mark and couldn't
1915 * find any flushable buffers, we sleep half a second.
1916 * Otherwise we loop immediately.
1918 if (numdirtybuffers
<= lodirtybuffers
) {
1920 * We reached our low water mark, reset the
1921 * request and sleep until we are needed again.
1922 * The sleep is just so the suspend code works.
1925 tsleep(&bd_request
, 0, "psleep", hz
);
1928 * We couldn't find any flushable dirty buffers but
1929 * still have too many dirty buffers, we
1930 * have to sleep and try again. (rare)
1932 tsleep(&bd_request
, 0, "qsleep", hz
/ 2);
1940 * Try to flush a buffer in the dirty queue. We must be careful to
1941 * free up B_INVAL buffers instead of write them, which NFS is
1942 * particularly sensitive to.
1946 flushbufqueues(void)
1951 bp
= TAILQ_FIRST(&bufqueues
[QUEUE_DIRTY
]);
1954 KASSERT((bp
->b_flags
& B_DELWRI
), ("unexpected clean buffer %p", bp
));
1955 if ((bp
->b_flags
& B_DELWRI
) != 0 &&
1956 (bp
->b_xflags
& BX_BKGRDINPROG
) == 0) {
1957 if (bp
->b_flags
& B_INVAL
) {
1958 if (BUF_LOCK(bp
, LK_EXCLUSIVE
| LK_NOWAIT
) != 0)
1959 panic("flushbufqueues: locked buf");
1965 if (LIST_FIRST(&bp
->b_dep
) != NULL
&&
1966 bioops
.io_countdeps
&&
1967 (bp
->b_flags
& B_DEFERRED
) == 0 &&
1968 (*bioops
.io_countdeps
)(bp
, 0)) {
1969 TAILQ_REMOVE(&bufqueues
[QUEUE_DIRTY
],
1971 TAILQ_INSERT_TAIL(&bufqueues
[QUEUE_DIRTY
],
1973 bp
->b_flags
|= B_DEFERRED
;
1974 bp
= TAILQ_FIRST(&bufqueues
[QUEUE_DIRTY
]);
1981 bp
= TAILQ_NEXT(bp
, b_freelist
);
1987 * Check to see if a block is currently memory resident.
1990 incore(struct vnode
* vp
, daddr_t blkno
)
1995 bp
= gbincore(vp
, blkno
);
2001 * Returns true if no I/O is needed to access the associated VM object.
2002 * This is like incore except it also hunts around in the VM system for
2005 * Note that we ignore vm_page_free() races from interrupts against our
2006 * lookup, since if the caller is not protected our return value will not
2007 * be any more valid then otherwise once we splx().
2010 inmem(struct vnode
* vp
, daddr_t blkno
)
2013 vm_offset_t toff
, tinc
, size
;
2017 if (incore(vp
, blkno
))
2019 if (vp
->v_mount
== NULL
)
2021 if (VOP_GETVOBJECT(vp
, &obj
) != 0 || (vp
->v_flag
& VOBJBUF
) == 0)
2025 if (size
> vp
->v_mount
->mnt_stat
.f_iosize
)
2026 size
= vp
->v_mount
->mnt_stat
.f_iosize
;
2027 off
= (vm_ooffset_t
)blkno
* (vm_ooffset_t
)vp
->v_mount
->mnt_stat
.f_iosize
;
2029 for (toff
= 0; toff
< vp
->v_mount
->mnt_stat
.f_iosize
; toff
+= tinc
) {
2030 m
= vm_page_lookup(obj
, OFF_TO_IDX(off
+ toff
));
2034 if (tinc
> PAGE_SIZE
- ((toff
+ off
) & PAGE_MASK
))
2035 tinc
= PAGE_SIZE
- ((toff
+ off
) & PAGE_MASK
);
2036 if (vm_page_is_valid(m
,
2037 (vm_offset_t
) ((toff
+ off
) & PAGE_MASK
), tinc
) == 0)
2046 * Sets the dirty range for a buffer based on the status of the dirty
2047 * bits in the pages comprising the buffer.
2049 * The range is limited to the size of the buffer.
2051 * This routine is primarily used by NFS, but is generalized for the
2055 vfs_setdirty(struct buf
*bp
)
2061 * Degenerate case - empty buffer
2064 if (bp
->b_bufsize
== 0)
2068 * We qualify the scan for modified pages on whether the
2069 * object has been flushed yet. The OBJ_WRITEABLE flag
2070 * is not cleared simply by protecting pages off.
2073 if ((bp
->b_flags
& B_VMIO
) == 0)
2076 object
= bp
->b_xio
.xio_pages
[0]->object
;
2078 if ((object
->flags
& OBJ_WRITEABLE
) && !(object
->flags
& OBJ_MIGHTBEDIRTY
))
2079 printf("Warning: object %p writeable but not mightbedirty\n", object
);
2080 if (!(object
->flags
& OBJ_WRITEABLE
) && (object
->flags
& OBJ_MIGHTBEDIRTY
))
2081 printf("Warning: object %p mightbedirty but not writeable\n", object
);
2083 if (object
->flags
& (OBJ_MIGHTBEDIRTY
|OBJ_CLEANING
)) {
2084 vm_offset_t boffset
;
2085 vm_offset_t eoffset
;
2088 * test the pages to see if they have been modified directly
2089 * by users through the VM system.
2091 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
2092 vm_page_flag_clear(bp
->b_xio
.xio_pages
[i
], PG_ZERO
);
2093 vm_page_test_dirty(bp
->b_xio
.xio_pages
[i
]);
2097 * Calculate the encompassing dirty range, boffset and eoffset,
2098 * (eoffset - boffset) bytes.
2101 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
2102 if (bp
->b_xio
.xio_pages
[i
]->dirty
)
2105 boffset
= (i
<< PAGE_SHIFT
) - (bp
->b_offset
& PAGE_MASK
);
2107 for (i
= bp
->b_xio
.xio_npages
- 1; i
>= 0; --i
) {
2108 if (bp
->b_xio
.xio_pages
[i
]->dirty
) {
2112 eoffset
= ((i
+ 1) << PAGE_SHIFT
) - (bp
->b_offset
& PAGE_MASK
);
2115 * Fit it to the buffer.
2118 if (eoffset
> bp
->b_bcount
)
2119 eoffset
= bp
->b_bcount
;
2122 * If we have a good dirty range, merge with the existing
2126 if (boffset
< eoffset
) {
2127 if (bp
->b_dirtyoff
> boffset
)
2128 bp
->b_dirtyoff
= boffset
;
2129 if (bp
->b_dirtyend
< eoffset
)
2130 bp
->b_dirtyend
= eoffset
;
2138 * Get a block given a specified block and offset into a file/device.
2139 * The buffers B_DONE bit will be cleared on return, making it almost
2140 * ready for an I/O initiation. B_INVAL may or may not be set on
2141 * return. The caller should clear B_INVAL prior to initiating a
2144 * IT IS IMPORTANT TO UNDERSTAND THAT IF YOU CALL GETBLK() AND B_CACHE
2145 * IS NOT SET, YOU MUST INITIALIZE THE RETURNED BUFFER, ISSUE A READ,
2146 * OR SET B_INVAL BEFORE RETIRING IT. If you retire a getblk'd buffer
2147 * without doing any of those things the system will likely believe
2148 * the buffer to be valid (especially if it is not B_VMIO), and the
2149 * next getblk() will return the buffer with B_CACHE set.
2151 * For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
2152 * an existing buffer.
2154 * For a VMIO buffer, B_CACHE is modified according to the backing VM.
2155 * If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
2156 * and then cleared based on the backing VM. If the previous buffer is
2157 * non-0-sized but invalid, B_CACHE will be cleared.
2159 * If getblk() must create a new buffer, the new buffer is returned with
2160 * both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
2161 * case it is returned with B_INVAL clear and B_CACHE set based on the
2164 * getblk() also forces a VOP_BWRITE() for any B_DELWRI buffer whos
2165 * B_CACHE bit is clear.
2167 * What this means, basically, is that the caller should use B_CACHE to
2168 * determine whether the buffer is fully valid or not and should clear
2169 * B_INVAL prior to issuing a read. If the caller intends to validate
2170 * the buffer by loading its data area with something, the caller needs
2171 * to clear B_INVAL. If the caller does this without issuing an I/O,
2172 * the caller should set B_CACHE ( as an optimization ), else the caller
2173 * should issue the I/O and biodone() will set B_CACHE if the I/O was
2174 * a write attempt or if it was a successfull read. If the caller
2175 * intends to issue a READ, the caller must clear B_INVAL and B_ERROR
2176 * prior to issuing the READ. biodone() will *not* clear B_INVAL.
2179 getblk(struct vnode
* vp
, daddr_t blkno
, int size
, int slpflag
, int slptimeo
)
2183 struct bufhashhdr
*bh
;
2185 if (size
> MAXBSIZE
)
2186 panic("getblk: size(%d) > MAXBSIZE(%d)", size
, MAXBSIZE
);
2191 * Block if we are low on buffers. Certain processes are allowed
2192 * to completely exhaust the buffer cache.
2194 * If this check ever becomes a bottleneck it may be better to
2195 * move it into the else, when gbincore() fails. At the moment
2196 * it isn't a problem.
2198 * XXX remove, we cannot afford to block anywhere if holding a vnode
2199 * lock in low-memory situation, so take it to the max.
2201 if (numfreebuffers
== 0) {
2204 needsbuffer
|= VFS_BIO_NEED_ANY
;
2205 tsleep(&needsbuffer
, slpflag
, "newbuf", slptimeo
);
2208 if ((bp
= gbincore(vp
, blkno
))) {
2210 * Buffer is in-core. If the buffer is not busy, it must
2214 if (BUF_LOCK(bp
, LK_EXCLUSIVE
| LK_NOWAIT
)) {
2215 if (BUF_TIMELOCK(bp
, LK_EXCLUSIVE
| LK_SLEEPFAIL
,
2216 "getblk", slpflag
, slptimeo
) == ENOLCK
)
2219 return (struct buf
*) NULL
;
2223 * The buffer is locked. B_CACHE is cleared if the buffer is
2224 * invalid. Ohterwise, for a non-VMIO buffer, B_CACHE is set
2225 * and for a VMIO buffer B_CACHE is adjusted according to the
2228 if (bp
->b_flags
& B_INVAL
)
2229 bp
->b_flags
&= ~B_CACHE
;
2230 else if ((bp
->b_flags
& (B_VMIO
| B_INVAL
)) == 0)
2231 bp
->b_flags
|= B_CACHE
;
2235 * check for size inconsistancies for non-VMIO case.
2238 if (bp
->b_bcount
!= size
) {
2239 if ((bp
->b_flags
& B_VMIO
) == 0 ||
2240 (size
> bp
->b_kvasize
)) {
2241 if (bp
->b_flags
& B_DELWRI
) {
2242 bp
->b_flags
|= B_NOCACHE
;
2243 VOP_BWRITE(bp
->b_vp
, bp
);
2245 if ((bp
->b_flags
& B_VMIO
) &&
2246 (LIST_FIRST(&bp
->b_dep
) == NULL
)) {
2247 bp
->b_flags
|= B_RELBUF
;
2250 bp
->b_flags
|= B_NOCACHE
;
2251 VOP_BWRITE(bp
->b_vp
, bp
);
2259 * If the size is inconsistant in the VMIO case, we can resize
2260 * the buffer. This might lead to B_CACHE getting set or
2261 * cleared. If the size has not changed, B_CACHE remains
2262 * unchanged from its previous state.
2265 if (bp
->b_bcount
!= size
)
2268 KASSERT(bp
->b_offset
!= NOOFFSET
,
2269 ("getblk: no buffer offset"));
2272 * A buffer with B_DELWRI set and B_CACHE clear must
2273 * be committed before we can return the buffer in
2274 * order to prevent the caller from issuing a read
2275 * ( due to B_CACHE not being set ) and overwriting
2278 * Most callers, including NFS and FFS, need this to
2279 * operate properly either because they assume they
2280 * can issue a read if B_CACHE is not set, or because
2281 * ( for example ) an uncached B_DELWRI might loop due
2282 * to softupdates re-dirtying the buffer. In the latter
2283 * case, B_CACHE is set after the first write completes,
2284 * preventing further loops.
2286 * NOTE! b*write() sets B_CACHE. If we cleared B_CACHE
2287 * above while extending the buffer, we cannot allow the
2288 * buffer to remain with B_CACHE set after the write
2289 * completes or it will represent a corrupt state. To
2290 * deal with this we set B_NOCACHE to scrap the buffer
2293 * We might be able to do something fancy, like setting
2294 * B_CACHE in bwrite() except if B_DELWRI is already set,
2295 * so the below call doesn't set B_CACHE, but that gets real
2296 * confusing. This is much easier.
2299 if ((bp
->b_flags
& (B_CACHE
|B_DELWRI
)) == B_DELWRI
) {
2300 bp
->b_flags
|= B_NOCACHE
;
2301 VOP_BWRITE(bp
->b_vp
, bp
);
2306 bp
->b_flags
&= ~B_DONE
;
2309 * Buffer is not in-core, create new buffer. The buffer
2310 * returned by getnewbuf() is locked. Note that the returned
2311 * buffer is also considered valid (not marked B_INVAL).
2313 * Calculating the offset for the I/O requires figuring out
2314 * the block size. We use DEV_BSIZE for VBLK or VCHR and
2315 * the mount's f_iosize otherwise. If the vnode does not
2316 * have an associated mount we assume that the passed size is
2319 * Note that vn_isdisk() cannot be used here since it may
2320 * return a failure for numerous reasons. Note that the
2321 * buffer size may be larger then the block size (the caller
2322 * will use block numbers with the proper multiple). Beware
2323 * of using any v_* fields which are part of unions. In
2324 * particular, in DragonFly the mount point overloading
2325 * mechanism is such that the underlying directory (with a
2326 * non-NULL v_mountedhere) is not a special case.
2328 int bsize
, maxsize
, vmio
;
2331 if (vp
->v_type
== VBLK
|| vp
->v_type
== VCHR
)
2333 else if (vp
->v_mount
)
2334 bsize
= vp
->v_mount
->mnt_stat
.f_iosize
;
2338 offset
= (off_t
)blkno
* bsize
;
2339 vmio
= (VOP_GETVOBJECT(vp
, NULL
) == 0) && (vp
->v_flag
& VOBJBUF
);
2340 maxsize
= vmio
? size
+ (offset
& PAGE_MASK
) : size
;
2341 maxsize
= imax(maxsize
, bsize
);
2343 if ((bp
= getnewbuf(slpflag
, slptimeo
, size
, maxsize
)) == NULL
) {
2344 if (slpflag
|| slptimeo
) {
2352 * This code is used to make sure that a buffer is not
2353 * created while the getnewbuf routine is blocked.
2354 * This can be a problem whether the vnode is locked or not.
2355 * If the buffer is created out from under us, we have to
2356 * throw away the one we just created. There is now window
2357 * race because we are safely running at splbio() from the
2358 * point of the duplicate buffer creation through to here,
2359 * and we've locked the buffer.
2361 if (gbincore(vp
, blkno
)) {
2362 bp
->b_flags
|= B_INVAL
;
2368 * Insert the buffer into the hash, so that it can
2369 * be found by incore.
2371 bp
->b_blkno
= bp
->b_lblkno
= blkno
;
2372 bp
->b_offset
= offset
;
2375 LIST_REMOVE(bp
, b_hash
);
2376 bh
= bufhash(vp
, blkno
);
2377 LIST_INSERT_HEAD(bh
, bp
, b_hash
);
2380 * set B_VMIO bit. allocbuf() the buffer bigger. Since the
2381 * buffer size starts out as 0, B_CACHE will be set by
2382 * allocbuf() for the VMIO case prior to it testing the
2383 * backing store for validity.
2387 bp
->b_flags
|= B_VMIO
;
2388 #if defined(VFS_BIO_DEBUG)
2389 if (vn_canvmio(vp
) != TRUE
)
2390 printf("getblk: vmioing file type %d???\n", vp
->v_type
);
2393 bp
->b_flags
&= ~B_VMIO
;
2399 bp
->b_flags
&= ~B_DONE
;
2405 * Get an empty, disassociated buffer of given size. The buffer is initially
2408 * spl protection is not required for the allocbuf() call because races are
2418 maxsize
= (size
+ BKVAMASK
) & ~BKVAMASK
;
2421 while ((bp
= getnewbuf(0, 0, size
, maxsize
)) == 0);
2424 bp
->b_flags
|= B_INVAL
; /* b_dep cleared by getnewbuf() */
2430 * This code constitutes the buffer memory from either anonymous system
2431 * memory (in the case of non-VMIO operations) or from an associated
2432 * VM object (in the case of VMIO operations). This code is able to
2433 * resize a buffer up or down.
2435 * Note that this code is tricky, and has many complications to resolve
2436 * deadlock or inconsistant data situations. Tread lightly!!!
2437 * There are B_CACHE and B_DELWRI interactions that must be dealt with by
2438 * the caller. Calling this code willy nilly can result in the loss of data.
2440 * allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with
2441 * B_CACHE for the non-VMIO case.
2443 * This routine does not need to be called at splbio() but you must own the
2447 allocbuf(struct buf
*bp
, int size
)
2449 int newbsize
, mbsize
;
2452 if (BUF_REFCNT(bp
) == 0)
2453 panic("allocbuf: buffer not busy");
2455 if (bp
->b_kvasize
< size
)
2456 panic("allocbuf: buffer too small");
2458 if ((bp
->b_flags
& B_VMIO
) == 0) {
2462 * Just get anonymous memory from the kernel. Don't
2463 * mess with B_CACHE.
2465 mbsize
= (size
+ DEV_BSIZE
- 1) & ~(DEV_BSIZE
- 1);
2466 #if !defined(NO_B_MALLOC)
2467 if (bp
->b_flags
& B_MALLOC
)
2471 newbsize
= round_page(size
);
2473 if (newbsize
< bp
->b_bufsize
) {
2474 #if !defined(NO_B_MALLOC)
2476 * malloced buffers are not shrunk
2478 if (bp
->b_flags
& B_MALLOC
) {
2480 bp
->b_bcount
= size
;
2482 free(bp
->b_data
, M_BIOBUF
);
2483 if (bp
->b_bufsize
) {
2484 bufmallocspace
-= bp
->b_bufsize
;
2488 bp
->b_data
= bp
->b_kvabase
;
2490 bp
->b_flags
&= ~B_MALLOC
;
2497 (vm_offset_t
) bp
->b_data
+ newbsize
,
2498 (vm_offset_t
) bp
->b_data
+ bp
->b_bufsize
);
2499 } else if (newbsize
> bp
->b_bufsize
) {
2500 #if !defined(NO_B_MALLOC)
2502 * We only use malloced memory on the first allocation.
2503 * and revert to page-allocated memory when the buffer
2506 if ( (bufmallocspace
< maxbufmallocspace
) &&
2507 (bp
->b_bufsize
== 0) &&
2508 (mbsize
<= PAGE_SIZE
/2)) {
2510 bp
->b_data
= malloc(mbsize
, M_BIOBUF
, M_WAITOK
);
2511 bp
->b_bufsize
= mbsize
;
2512 bp
->b_bcount
= size
;
2513 bp
->b_flags
|= B_MALLOC
;
2514 bufmallocspace
+= mbsize
;
2520 #if !defined(NO_B_MALLOC)
2522 * If the buffer is growing on its other-than-first allocation,
2523 * then we revert to the page-allocation scheme.
2525 if (bp
->b_flags
& B_MALLOC
) {
2526 origbuf
= bp
->b_data
;
2527 origbufsize
= bp
->b_bufsize
;
2528 bp
->b_data
= bp
->b_kvabase
;
2529 if (bp
->b_bufsize
) {
2530 bufmallocspace
-= bp
->b_bufsize
;
2534 bp
->b_flags
&= ~B_MALLOC
;
2535 newbsize
= round_page(newbsize
);
2540 (vm_offset_t
) bp
->b_data
+ bp
->b_bufsize
,
2541 (vm_offset_t
) bp
->b_data
+ newbsize
);
2542 #if !defined(NO_B_MALLOC)
2544 bcopy(origbuf
, bp
->b_data
, origbufsize
);
2545 free(origbuf
, M_BIOBUF
);
2553 newbsize
= (size
+ DEV_BSIZE
- 1) & ~(DEV_BSIZE
- 1);
2554 desiredpages
= (size
== 0) ? 0 :
2555 num_pages((bp
->b_offset
& PAGE_MASK
) + newbsize
);
2557 #if !defined(NO_B_MALLOC)
2558 if (bp
->b_flags
& B_MALLOC
)
2559 panic("allocbuf: VMIO buffer can't be malloced");
2562 * Set B_CACHE initially if buffer is 0 length or will become
2565 if (size
== 0 || bp
->b_bufsize
== 0)
2566 bp
->b_flags
|= B_CACHE
;
2568 if (newbsize
< bp
->b_bufsize
) {
2570 * DEV_BSIZE aligned new buffer size is less then the
2571 * DEV_BSIZE aligned existing buffer size. Figure out
2572 * if we have to remove any pages.
2574 if (desiredpages
< bp
->b_xio
.xio_npages
) {
2575 for (i
= desiredpages
; i
< bp
->b_xio
.xio_npages
; i
++) {
2577 * the page is not freed here -- it
2578 * is the responsibility of
2579 * vnode_pager_setsize
2581 m
= bp
->b_xio
.xio_pages
[i
];
2582 KASSERT(m
!= bogus_page
,
2583 ("allocbuf: bogus page found"));
2584 while (vm_page_sleep_busy(m
, TRUE
, "biodep"))
2587 bp
->b_xio
.xio_pages
[i
] = NULL
;
2588 vm_page_unwire(m
, 0);
2590 pmap_qremove((vm_offset_t
) trunc_page((vm_offset_t
)bp
->b_data
) +
2591 (desiredpages
<< PAGE_SHIFT
), (bp
->b_xio
.xio_npages
- desiredpages
));
2592 bp
->b_xio
.xio_npages
= desiredpages
;
2594 } else if (size
> bp
->b_bcount
) {
2596 * We are growing the buffer, possibly in a
2597 * byte-granular fashion.
2605 * Step 1, bring in the VM pages from the object,
2606 * allocating them if necessary. We must clear
2607 * B_CACHE if these pages are not valid for the
2608 * range covered by the buffer.
2610 * spl protection is required to protect against
2611 * interrupts unbusying and freeing pages between
2612 * our vm_page_lookup() and our busycheck/wiring
2616 VOP_GETVOBJECT(vp
, &obj
);
2619 while (bp
->b_xio
.xio_npages
< desiredpages
) {
2623 pi
= OFF_TO_IDX(bp
->b_offset
) + bp
->b_xio
.xio_npages
;
2624 if ((m
= vm_page_lookup(obj
, pi
)) == NULL
) {
2626 * note: must allocate system pages
2627 * since blocking here could intefere
2628 * with paging I/O, no matter which
2631 m
= vm_page_alloc(obj
, pi
, VM_ALLOC_NORMAL
| VM_ALLOC_SYSTEM
);
2634 vm_pageout_deficit
+= desiredpages
-
2635 bp
->b_xio
.xio_npages
;
2639 bp
->b_flags
&= ~B_CACHE
;
2640 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
2641 ++bp
->b_xio
.xio_npages
;
2647 * We found a page. If we have to sleep on it,
2648 * retry because it might have gotten freed out
2651 * We can only test PG_BUSY here. Blocking on
2652 * m->busy might lead to a deadlock:
2654 * vm_fault->getpages->cluster_read->allocbuf
2658 if (vm_page_sleep_busy(m
, FALSE
, "pgtblk"))
2662 * We have a good page. Should we wakeup the
2665 if ((curthread
!= pagethread
) &&
2666 ((m
->queue
- m
->pc
) == PQ_CACHE
) &&
2667 ((vmstats
.v_free_count
+ vmstats
.v_cache_count
) <
2668 (vmstats
.v_free_min
+ vmstats
.v_cache_min
))) {
2669 pagedaemon_wakeup();
2671 vm_page_flag_clear(m
, PG_ZERO
);
2673 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
2674 ++bp
->b_xio
.xio_npages
;
2679 * Step 2. We've loaded the pages into the buffer,
2680 * we have to figure out if we can still have B_CACHE
2681 * set. Note that B_CACHE is set according to the
2682 * byte-granular range ( bcount and size ), new the
2683 * aligned range ( newbsize ).
2685 * The VM test is against m->valid, which is DEV_BSIZE
2686 * aligned. Needless to say, the validity of the data
2687 * needs to also be DEV_BSIZE aligned. Note that this
2688 * fails with NFS if the server or some other client
2689 * extends the file's EOF. If our buffer is resized,
2690 * B_CACHE may remain set! XXX
2693 toff
= bp
->b_bcount
;
2694 tinc
= PAGE_SIZE
- ((bp
->b_offset
+ toff
) & PAGE_MASK
);
2696 while ((bp
->b_flags
& B_CACHE
) && toff
< size
) {
2699 if (tinc
> (size
- toff
))
2702 pi
= ((bp
->b_offset
& PAGE_MASK
) + toff
) >>
2710 bp
->b_xio
.xio_pages
[pi
]
2717 * Step 3, fixup the KVM pmap. Remember that
2718 * bp->b_data is relative to bp->b_offset, but
2719 * bp->b_offset may be offset into the first page.
2722 bp
->b_data
= (caddr_t
)
2723 trunc_page((vm_offset_t
)bp
->b_data
);
2725 (vm_offset_t
)bp
->b_data
,
2726 bp
->b_xio
.xio_pages
,
2727 bp
->b_xio
.xio_npages
2729 bp
->b_data
= (caddr_t
)((vm_offset_t
)bp
->b_data
|
2730 (vm_offset_t
)(bp
->b_offset
& PAGE_MASK
));
2733 if (newbsize
< bp
->b_bufsize
)
2735 bp
->b_bufsize
= newbsize
; /* actual buffer allocation */
2736 bp
->b_bcount
= size
; /* requested buffer size */
2743 * Wait for buffer I/O completion, returning error status. The buffer
2744 * is left locked and B_DONE on return. B_EINTR is converted into a EINTR
2745 * error and cleared.
2748 biowait(struct buf
* bp
)
2753 while ((bp
->b_flags
& B_DONE
) == 0) {
2754 #if defined(NO_SCHEDULE_MODS)
2755 tsleep(bp
, 0, "biowait", 0);
2757 if (bp
->b_flags
& B_READ
)
2758 tsleep(bp
, 0, "biord", 0);
2760 tsleep(bp
, 0, "biowr", 0);
2764 if (bp
->b_flags
& B_EINTR
) {
2765 bp
->b_flags
&= ~B_EINTR
;
2768 if (bp
->b_flags
& B_ERROR
) {
2769 return (bp
->b_error
? bp
->b_error
: EIO
);
2778 * Finish I/O on a buffer, optionally calling a completion function.
2779 * This is usually called from an interrupt so process blocking is
2782 * biodone is also responsible for setting B_CACHE in a B_VMIO bp.
2783 * In a non-VMIO bp, B_CACHE will be set on the next getblk()
2784 * assuming B_INVAL is clear.
2786 * For the VMIO case, we set B_CACHE if the op was a read and no
2787 * read error occured, or if the op was a write. B_CACHE is never
2788 * set if the buffer is invalid or otherwise uncacheable.
2790 * biodone does not mess with B_INVAL, allowing the I/O routine or the
2791 * initiator to leave B_INVAL set to brelse the buffer out of existance
2792 * in the biodone routine.
2794 * b_dev is required to be reinitialized prior to the top level strategy
2795 * call in a device stack. To avoid improper reuse, biodone() sets
2799 biodone(struct buf
*bp
)
2805 KASSERT(BUF_REFCNTNB(bp
) > 0, ("biodone: bp %p not busy %d", bp
, BUF_REFCNTNB(bp
)));
2806 KASSERT(!(bp
->b_flags
& B_DONE
), ("biodone: bp %p already done", bp
));
2808 bp
->b_flags
|= B_DONE
;
2810 runningbufwakeup(bp
);
2812 if (bp
->b_flags
& B_FREEBUF
) {
2818 if ((bp
->b_flags
& B_READ
) == 0) {
2822 /* call optional completion function if requested */
2823 if (bp
->b_flags
& B_CALL
) {
2824 bp
->b_flags
&= ~B_CALL
;
2825 (*bp
->b_iodone
) (bp
);
2829 if (LIST_FIRST(&bp
->b_dep
) != NULL
&& bioops
.io_complete
)
2830 (*bioops
.io_complete
)(bp
);
2832 if (bp
->b_flags
& B_VMIO
) {
2838 struct vnode
*vp
= bp
->b_vp
;
2840 error
= VOP_GETVOBJECT(vp
, &obj
);
2842 #if defined(VFS_BIO_DEBUG)
2843 if (vp
->v_holdcnt
== 0) {
2844 panic("biodone: zero vnode hold count");
2848 panic("biodone: missing VM object");
2851 if ((vp
->v_flag
& VOBJBUF
) == 0) {
2852 panic("biodone: vnode is not setup for merged cache");
2856 foff
= bp
->b_offset
;
2857 KASSERT(bp
->b_offset
!= NOOFFSET
,
2858 ("biodone: no buffer offset"));
2861 panic("biodone: no object");
2863 #if defined(VFS_BIO_DEBUG)
2864 if (obj
->paging_in_progress
< bp
->b_xio
.xio_npages
) {
2865 printf("biodone: paging in progress(%d) < bp->b_xio.xio_npages(%d)\n",
2866 obj
->paging_in_progress
, bp
->b_xio
.xio_npages
);
2871 * Set B_CACHE if the op was a normal read and no error
2872 * occured. B_CACHE is set for writes in the b*write()
2875 iosize
= bp
->b_bcount
- bp
->b_resid
;
2876 if ((bp
->b_flags
& (B_READ
|B_FREEBUF
|B_INVAL
|B_NOCACHE
|B_ERROR
)) == B_READ
) {
2877 bp
->b_flags
|= B_CACHE
;
2880 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
2884 resid
= ((foff
+ PAGE_SIZE
) & ~(off_t
)PAGE_MASK
) - foff
;
2889 * cleanup bogus pages, restoring the originals. Since
2890 * the originals should still be wired, we don't have
2891 * to worry about interrupt/freeing races destroying
2892 * the VM object association.
2894 m
= bp
->b_xio
.xio_pages
[i
];
2895 if (m
== bogus_page
) {
2897 m
= vm_page_lookup(obj
, OFF_TO_IDX(foff
));
2899 panic("biodone: page disappeared");
2900 bp
->b_xio
.xio_pages
[i
] = m
;
2901 pmap_qenter(trunc_page((vm_offset_t
)bp
->b_data
),
2902 bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
2904 #if defined(VFS_BIO_DEBUG)
2905 if (OFF_TO_IDX(foff
) != m
->pindex
) {
2907 "biodone: foff(%lu)/m->pindex(%d) mismatch\n",
2908 (unsigned long)foff
, m
->pindex
);
2913 * In the write case, the valid and clean bits are
2914 * already changed correctly ( see bdwrite() ), so we
2915 * only need to do this here in the read case.
2917 if ((bp
->b_flags
& B_READ
) && !bogusflag
&& resid
> 0) {
2918 vfs_page_set_valid(bp
, foff
, i
, m
);
2920 vm_page_flag_clear(m
, PG_ZERO
);
2923 * when debugging new filesystems or buffer I/O methods, this
2924 * is the most common error that pops up. if you see this, you
2925 * have not set the page busy flag correctly!!!
2928 printf("biodone: page busy < 0, "
2929 "pindex: %d, foff: 0x(%x,%x), "
2930 "resid: %d, index: %d\n",
2931 (int) m
->pindex
, (int)(foff
>> 32),
2932 (int) foff
& 0xffffffff, resid
, i
);
2933 if (!vn_isdisk(vp
, NULL
))
2934 printf(" iosize: %ld, lblkno: %d, flags: 0x%lx, npages: %d\n",
2935 bp
->b_vp
->v_mount
->mnt_stat
.f_iosize
,
2937 bp
->b_flags
, bp
->b_xio
.xio_npages
);
2939 printf(" VDEV, lblkno: %d, flags: 0x%lx, npages: %d\n",
2941 bp
->b_flags
, bp
->b_xio
.xio_npages
);
2942 printf(" valid: 0x%x, dirty: 0x%x, wired: %d\n",
2943 m
->valid
, m
->dirty
, m
->wire_count
);
2944 panic("biodone: page busy < 0");
2946 vm_page_io_finish(m
);
2947 vm_object_pip_subtract(obj
, 1);
2948 foff
= (foff
+ PAGE_SIZE
) & ~(off_t
)PAGE_MASK
;
2952 vm_object_pip_wakeupn(obj
, 0);
2956 * For asynchronous completions, release the buffer now. The brelse
2957 * will do a wakeup there if necessary - so no need to do a wakeup
2958 * here in the async case. The sync case always needs to do a wakeup.
2961 if (bp
->b_flags
& B_ASYNC
) {
2962 if ((bp
->b_flags
& (B_NOCACHE
| B_INVAL
| B_ERROR
| B_RELBUF
)) != 0)
2973 * This routine is called in lieu of iodone in the case of
2974 * incomplete I/O. This keeps the busy status for pages
2978 vfs_unbusy_pages(struct buf
*bp
)
2982 runningbufwakeup(bp
);
2983 if (bp
->b_flags
& B_VMIO
) {
2984 struct vnode
*vp
= bp
->b_vp
;
2987 VOP_GETVOBJECT(vp
, &obj
);
2989 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
2990 vm_page_t m
= bp
->b_xio
.xio_pages
[i
];
2993 * When restoring bogus changes the original pages
2994 * should still be wired, so we are in no danger of
2995 * losing the object association and do not need
2996 * spl protection particularly.
2998 if (m
== bogus_page
) {
2999 m
= vm_page_lookup(obj
, OFF_TO_IDX(bp
->b_offset
) + i
);
3001 panic("vfs_unbusy_pages: page missing");
3003 bp
->b_xio
.xio_pages
[i
] = m
;
3004 pmap_qenter(trunc_page((vm_offset_t
)bp
->b_data
),
3005 bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
3007 vm_object_pip_subtract(obj
, 1);
3008 vm_page_flag_clear(m
, PG_ZERO
);
3009 vm_page_io_finish(m
);
3011 vm_object_pip_wakeupn(obj
, 0);
3016 * vfs_page_set_valid:
3018 * Set the valid bits in a page based on the supplied offset. The
3019 * range is restricted to the buffer's size.
3021 * This routine is typically called after a read completes.
3024 vfs_page_set_valid(struct buf
*bp
, vm_ooffset_t off
, int pageno
, vm_page_t m
)
3026 vm_ooffset_t soff
, eoff
;
3029 * Start and end offsets in buffer. eoff - soff may not cross a
3030 * page boundry or cross the end of the buffer. The end of the
3031 * buffer, in this case, is our file EOF, not the allocation size
3035 eoff
= (off
+ PAGE_SIZE
) & ~(off_t
)PAGE_MASK
;
3036 if (eoff
> bp
->b_offset
+ bp
->b_bcount
)
3037 eoff
= bp
->b_offset
+ bp
->b_bcount
;
3040 * Set valid range. This is typically the entire buffer and thus the
3044 vm_page_set_validclean(
3046 (vm_offset_t
) (soff
& PAGE_MASK
),
3047 (vm_offset_t
) (eoff
- soff
)
3053 * This routine is called before a device strategy routine.
3054 * It is used to tell the VM system that paging I/O is in
3055 * progress, and treat the pages associated with the buffer
3056 * almost as being PG_BUSY. Also the object paging_in_progress
3057 * flag is handled to make sure that the object doesn't become
3060 * Since I/O has not been initiated yet, certain buffer flags
3061 * such as B_ERROR or B_INVAL may be in an inconsistant state
3062 * and should be ignored.
3065 vfs_busy_pages(struct buf
*bp
, int clear_modify
)
3069 if (bp
->b_flags
& B_VMIO
) {
3070 struct vnode
*vp
= bp
->b_vp
;
3074 VOP_GETVOBJECT(vp
, &obj
);
3075 foff
= bp
->b_offset
;
3076 KASSERT(bp
->b_offset
!= NOOFFSET
,
3077 ("vfs_busy_pages: no buffer offset"));
3081 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
3082 vm_page_t m
= bp
->b_xio
.xio_pages
[i
];
3083 if (vm_page_sleep_busy(m
, FALSE
, "vbpage"))
3088 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
3089 vm_page_t m
= bp
->b_xio
.xio_pages
[i
];
3091 vm_page_flag_clear(m
, PG_ZERO
);
3092 if ((bp
->b_flags
& B_CLUSTER
) == 0) {
3093 vm_object_pip_add(obj
, 1);
3094 vm_page_io_start(m
);
3098 * When readying a buffer for a read ( i.e
3099 * clear_modify == 0 ), it is important to do
3100 * bogus_page replacement for valid pages in
3101 * partially instantiated buffers. Partially
3102 * instantiated buffers can, in turn, occur when
3103 * reconstituting a buffer from its VM backing store
3104 * base. We only have to do this if B_CACHE is
3105 * clear ( which causes the I/O to occur in the
3106 * first place ). The replacement prevents the read
3107 * I/O from overwriting potentially dirty VM-backed
3108 * pages. XXX bogus page replacement is, uh, bogus.
3109 * It may not work properly with small-block devices.
3110 * We need to find a better way.
3113 vm_page_protect(m
, VM_PROT_NONE
);
3115 vfs_page_set_valid(bp
, foff
, i
, m
);
3116 else if (m
->valid
== VM_PAGE_BITS_ALL
&&
3117 (bp
->b_flags
& B_CACHE
) == 0) {
3118 bp
->b_xio
.xio_pages
[i
] = bogus_page
;
3121 foff
= (foff
+ PAGE_SIZE
) & ~(off_t
)PAGE_MASK
;
3124 pmap_qenter(trunc_page((vm_offset_t
)bp
->b_data
),
3125 bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
3129 * This is the easiest place to put the process accounting for the I/O
3135 if ((p
= curthread
->td_proc
) != NULL
) {
3136 if (bp
->b_flags
& B_READ
)
3137 p
->p_stats
->p_ru
.ru_inblock
++;
3139 p
->p_stats
->p_ru
.ru_oublock
++;
3145 * Tell the VM system that the pages associated with this buffer
3146 * are clean. This is used for delayed writes where the data is
3147 * going to go to disk eventually without additional VM intevention.
3149 * Note that while we only really need to clean through to b_bcount, we
3150 * just go ahead and clean through to b_bufsize.
3153 vfs_clean_pages(struct buf
*bp
)
3157 if (bp
->b_flags
& B_VMIO
) {
3160 foff
= bp
->b_offset
;
3161 KASSERT(bp
->b_offset
!= NOOFFSET
,
3162 ("vfs_clean_pages: no buffer offset"));
3163 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
3164 vm_page_t m
= bp
->b_xio
.xio_pages
[i
];
3165 vm_ooffset_t noff
= (foff
+ PAGE_SIZE
) & ~(off_t
)PAGE_MASK
;
3166 vm_ooffset_t eoff
= noff
;
3168 if (eoff
> bp
->b_offset
+ bp
->b_bufsize
)
3169 eoff
= bp
->b_offset
+ bp
->b_bufsize
;
3170 vfs_page_set_valid(bp
, foff
, i
, m
);
3171 /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
3178 * vfs_bio_set_validclean:
3180 * Set the range within the buffer to valid and clean. The range is
3181 * relative to the beginning of the buffer, b_offset. Note that b_offset
3182 * itself may be offset from the beginning of the first page.
3186 vfs_bio_set_validclean(struct buf
*bp
, int base
, int size
)
3188 if (bp
->b_flags
& B_VMIO
) {
3193 * Fixup base to be relative to beginning of first page.
3194 * Set initial n to be the maximum number of bytes in the
3195 * first page that can be validated.
3198 base
+= (bp
->b_offset
& PAGE_MASK
);
3199 n
= PAGE_SIZE
- (base
& PAGE_MASK
);
3201 for (i
= base
/ PAGE_SIZE
; size
> 0 && i
< bp
->b_xio
.xio_npages
; ++i
) {
3202 vm_page_t m
= bp
->b_xio
.xio_pages
[i
];
3207 vm_page_set_validclean(m
, base
& PAGE_MASK
, n
);
3218 * clear a buffer. This routine essentially fakes an I/O, so we need
3219 * to clear B_ERROR and B_INVAL.
3221 * Note that while we only theoretically need to clear through b_bcount,
3222 * we go ahead and clear through b_bufsize.
3226 vfs_bio_clrbuf(struct buf
*bp
)
3230 if ((bp
->b_flags
& (B_VMIO
| B_MALLOC
)) == B_VMIO
) {
3231 bp
->b_flags
&= ~(B_INVAL
|B_ERROR
);
3232 if ((bp
->b_xio
.xio_npages
== 1) && (bp
->b_bufsize
< PAGE_SIZE
) &&
3233 (bp
->b_offset
& PAGE_MASK
) == 0) {
3234 mask
= (1 << (bp
->b_bufsize
/ DEV_BSIZE
)) - 1;
3235 if ((bp
->b_xio
.xio_pages
[0]->valid
& mask
) == mask
) {
3239 if (((bp
->b_xio
.xio_pages
[0]->flags
& PG_ZERO
) == 0) &&
3240 ((bp
->b_xio
.xio_pages
[0]->valid
& mask
) == 0)) {
3241 bzero(bp
->b_data
, bp
->b_bufsize
);
3242 bp
->b_xio
.xio_pages
[0]->valid
|= mask
;
3247 ea
= sa
= bp
->b_data
;
3248 for(i
=0;i
<bp
->b_xio
.xio_npages
;i
++,sa
=ea
) {
3249 int j
= ((vm_offset_t
)sa
& PAGE_MASK
) / DEV_BSIZE
;
3250 ea
= (caddr_t
)trunc_page((vm_offset_t
)sa
+ PAGE_SIZE
);
3251 ea
= (caddr_t
)(vm_offset_t
)ulmin(
3252 (u_long
)(vm_offset_t
)ea
,
3253 (u_long
)(vm_offset_t
)bp
->b_data
+ bp
->b_bufsize
);
3254 mask
= ((1 << ((ea
- sa
) / DEV_BSIZE
)) - 1) << j
;
3255 if ((bp
->b_xio
.xio_pages
[i
]->valid
& mask
) == mask
)
3257 if ((bp
->b_xio
.xio_pages
[i
]->valid
& mask
) == 0) {
3258 if ((bp
->b_xio
.xio_pages
[i
]->flags
& PG_ZERO
) == 0) {
3262 for (; sa
< ea
; sa
+= DEV_BSIZE
, j
++) {
3263 if (((bp
->b_xio
.xio_pages
[i
]->flags
& PG_ZERO
) == 0) &&
3264 (bp
->b_xio
.xio_pages
[i
]->valid
& (1<<j
)) == 0)
3265 bzero(sa
, DEV_BSIZE
);
3268 bp
->b_xio
.xio_pages
[i
]->valid
|= mask
;
3269 vm_page_flag_clear(bp
->b_xio
.xio_pages
[i
], PG_ZERO
);
3278 * vm_hold_load_pages and vm_hold_unload pages get pages into
3279 * a buffers address space. The pages are anonymous and are
3280 * not associated with a file object.
3283 vm_hold_load_pages(struct buf
*bp
, vm_offset_t from
, vm_offset_t to
)
3289 to
= round_page(to
);
3290 from
= round_page(from
);
3291 index
= (from
- trunc_page((vm_offset_t
)bp
->b_data
)) >> PAGE_SHIFT
;
3293 for (pg
= from
; pg
< to
; pg
+= PAGE_SIZE
, index
++) {
3298 * note: must allocate system pages since blocking here
3299 * could intefere with paging I/O, no matter which
3302 p
= vm_page_alloc(kernel_object
,
3303 ((pg
- VM_MIN_KERNEL_ADDRESS
) >> PAGE_SHIFT
),
3304 VM_ALLOC_NORMAL
| VM_ALLOC_SYSTEM
);
3306 vm_pageout_deficit
+= (to
- from
) >> PAGE_SHIFT
;
3311 p
->valid
= VM_PAGE_BITS_ALL
;
3312 vm_page_flag_clear(p
, PG_ZERO
);
3313 pmap_kenter(pg
, VM_PAGE_TO_PHYS(p
));
3314 bp
->b_xio
.xio_pages
[index
] = p
;
3317 bp
->b_xio
.xio_npages
= index
;
3321 vm_hold_free_pages(struct buf
*bp
, vm_offset_t from
, vm_offset_t to
)
3325 int index
, newnpages
;
3327 from
= round_page(from
);
3328 to
= round_page(to
);
3329 newnpages
= index
= (from
- trunc_page((vm_offset_t
)bp
->b_data
)) >> PAGE_SHIFT
;
3331 for (pg
= from
; pg
< to
; pg
+= PAGE_SIZE
, index
++) {
3332 p
= bp
->b_xio
.xio_pages
[index
];
3333 if (p
&& (index
< bp
->b_xio
.xio_npages
)) {
3335 printf("vm_hold_free_pages: blkno: %d, lblkno: %d\n",
3336 bp
->b_blkno
, bp
->b_lblkno
);
3338 bp
->b_xio
.xio_pages
[index
] = NULL
;
3341 vm_page_unwire(p
, 0);
3345 bp
->b_xio
.xio_npages
= newnpages
;
3349 * Map an IO request into kernel virtual address space.
3351 * All requests are (re)mapped into kernel VA space.
3352 * Notice that we use b_bufsize for the size of the buffer
3353 * to be mapped. b_bcount might be modified by the driver.
3356 vmapbuf(struct buf
*bp
)
3358 caddr_t addr
, v
, kva
;
3364 if ((bp
->b_flags
& B_PHYS
) == 0)
3366 if (bp
->b_bufsize
< 0)
3368 for (v
= bp
->b_saveaddr
,
3369 addr
= (caddr_t
)trunc_page((vm_offset_t
)bp
->b_data
),
3371 addr
< bp
->b_data
+ bp
->b_bufsize
;
3372 addr
+= PAGE_SIZE
, v
+= PAGE_SIZE
, pidx
++) {
3374 * Do the vm_fault if needed; do the copy-on-write thing
3375 * when reading stuff off device into memory.
3378 i
= vm_fault_quick((addr
>= bp
->b_data
) ? addr
: bp
->b_data
,
3379 (bp
->b_flags
&B_READ
)?(VM_PROT_READ
|VM_PROT_WRITE
):VM_PROT_READ
);
3381 for (i
= 0; i
< pidx
; ++i
) {
3382 vm_page_unhold(bp
->b_xio
.xio_pages
[i
]);
3383 bp
->b_xio
.xio_pages
[i
] = NULL
;
3389 * WARNING! If sparc support is MFCd in the future this will
3390 * have to be changed from pmap_kextract() to pmap_extract()
3394 #error "If MFCing sparc support use pmap_extract"
3396 pa
= pmap_kextract((vm_offset_t
)addr
);
3398 printf("vmapbuf: warning, race against user address during I/O");
3401 m
= PHYS_TO_VM_PAGE(pa
);
3403 bp
->b_xio
.xio_pages
[pidx
] = m
;
3405 if (pidx
> btoc(MAXPHYS
))
3406 panic("vmapbuf: mapped more than MAXPHYS");
3407 pmap_qenter((vm_offset_t
)bp
->b_saveaddr
, bp
->b_xio
.xio_pages
, pidx
);
3409 kva
= bp
->b_saveaddr
;
3410 bp
->b_xio
.xio_npages
= pidx
;
3411 bp
->b_saveaddr
= bp
->b_data
;
3412 bp
->b_data
= kva
+ (((vm_offset_t
) bp
->b_data
) & PAGE_MASK
);
3417 * Free the io map PTEs associated with this IO operation.
3418 * We also invalidate the TLB entries and restore the original b_addr.
3421 vunmapbuf(struct buf
*bp
)
3427 if ((bp
->b_flags
& B_PHYS
) == 0)
3430 npages
= bp
->b_xio
.xio_npages
;
3431 pmap_qremove(trunc_page((vm_offset_t
)bp
->b_data
),
3433 m
= bp
->b_xio
.xio_pages
;
3434 for (pidx
= 0; pidx
< npages
; pidx
++)
3435 vm_page_unhold(*m
++);
3437 bp
->b_data
= bp
->b_saveaddr
;
3440 #include "opt_ddb.h"
3442 #include <ddb/ddb.h>
3444 DB_SHOW_COMMAND(buffer
, db_show_buffer
)
3447 struct buf
*bp
= (struct buf
*)addr
;
3450 db_printf("usage: show buffer <addr>\n");
3454 db_printf("b_flags = 0x%b\n", (u_int
)bp
->b_flags
, PRINT_BUF_FLAGS
);
3455 db_printf("b_error = %d, b_bufsize = %ld, b_bcount = %ld, "
3456 "b_resid = %ld\nb_dev = (%d,%d), b_data = %p, "
3457 "b_blkno = %d, b_pblkno = %d\n",
3458 bp
->b_error
, bp
->b_bufsize
, bp
->b_bcount
, bp
->b_resid
,
3459 major(bp
->b_dev
), minor(bp
->b_dev
),
3460 bp
->b_data
, bp
->b_blkno
, bp
->b_pblkno
);
3461 if (bp
->b_xio
.xio_npages
) {
3463 db_printf("b_xio.xio_npages = %d, pages(OBJ, IDX, PA): ",
3464 bp
->b_xio
.xio_npages
);
3465 for (i
= 0; i
< bp
->b_xio
.xio_npages
; i
++) {
3467 m
= bp
->b_xio
.xio_pages
[i
];
3468 db_printf("(%p, 0x%lx, 0x%lx)", (void *)m
->object
,
3469 (u_long
)m
->pindex
, (u_long
)VM_PAGE_TO_PHYS(m
));
3470 if ((i
+ 1) < bp
->b_xio
.xio_npages
)