3 * The Regents of the University of California. All rights reserved.
4 * Modifications/enhancements:
5 * Copyright (c) 1995 John S. Dyson. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by the University of
18 * California, Berkeley and its contributors.
19 * 4. Neither the name of the University nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * @(#)vfs_cluster.c 8.7 (Berkeley) 2/13/94
36 * $FreeBSD: src/sys/kern/vfs_cluster.c,v 1.92.2.9 2001/11/18 07:10:59 dillon Exp $
37 * $DragonFly: src/sys/kern/vfs_cluster.c,v 1.39 2008/06/20 05:40:04 dillon Exp $
40 #include "opt_debug_cluster.h"
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
47 #include <sys/vnode.h>
48 #include <sys/malloc.h>
49 #include <sys/mount.h>
50 #include <sys/resourcevar.h>
51 #include <sys/vmmeter.h>
53 #include <vm/vm_object.h>
54 #include <vm/vm_page.h>
55 #include <sys/sysctl.h>
57 #include <vm/vm_page2.h>
59 #if defined(CLUSTERDEBUG)
60 #include <sys/sysctl.h>
61 static int rcluster
= 0;
62 SYSCTL_INT(_debug
, OID_AUTO
, rcluster
, CTLFLAG_RW
, &rcluster
, 0, "");
65 static MALLOC_DEFINE(M_SEGMENT
, "cluster_save", "cluster_save buffer");
67 static struct cluster_save
*
68 cluster_collectbufs (struct vnode
*vp
, struct buf
*last_bp
,
71 cluster_rbuild (struct vnode
*vp
, off_t filesize
, off_t loffset
,
72 off_t doffset
, int blksize
, int run
,
73 struct buf
*fbp
, int doasync
);
74 static void cluster_callback (struct bio
*);
77 static int write_behind
= 1;
78 SYSCTL_INT(_vfs
, OID_AUTO
, write_behind
, CTLFLAG_RW
, &write_behind
, 0, "");
80 extern vm_page_t bogus_page
;
82 extern int cluster_pbuf_freecnt
;
85 * Maximum number of blocks for read-ahead.
90 * This replaces bread.
93 cluster_read(struct vnode
*vp
, off_t filesize
, off_t loffset
,
94 int blksize
, int totread
, int seqcount
, struct buf
**bpp
)
96 struct buf
*bp
, *rbp
, *reqbp
;
101 int maxra
, racluster
;
106 * Try to limit the amount of read-ahead by a few
107 * ad-hoc parameters. This needs work!!!
109 racluster
= vmaxiosize(vp
) / blksize
;
110 maxra
= 2 * racluster
+ (totread
/ blksize
);
117 * get the requested block
119 *bpp
= reqbp
= bp
= getblk(vp
, loffset
, blksize
, 0, 0);
120 origoffset
= loffset
;
123 * if it is in the cache, then check to see if the reads have been
124 * sequential. If they have, then try some read-ahead, otherwise
125 * back-off on prospective read-aheads.
127 if (bp
->b_flags
& B_CACHE
) {
130 } else if ((bp
->b_flags
& B_RAM
) == 0) {
134 bp
->b_flags
&= ~B_RAM
;
136 * We do the crit here so that there is no window
137 * between the findblk and the b_usecount increment
138 * below. We opt to keep the crit out of the loop
142 for (i
= 1; i
< maxra
; i
++) {
143 if (!(tbp
= findblk(vp
, loffset
+ i
* blksize
))) {
148 * Set another read-ahead mark so we know
151 if (((i
% racluster
) == (racluster
- 1)) ||
153 tbp
->b_flags
|= B_RAM
;
159 loffset
+= i
* blksize
;
163 off_t firstread
= bp
->b_loffset
;
166 KASSERT(firstread
!= NOOFFSET
,
167 ("cluster_read: no buffer offset"));
168 if (firstread
+ totread
> filesize
)
169 totread
= (int)(filesize
- firstread
);
170 nblks
= totread
/ blksize
;
174 if (nblks
> racluster
)
177 error
= VOP_BMAP(vp
, loffset
, &doffset
,
178 &burstbytes
, NULL
, BUF_CMD_READ
);
180 goto single_block_read
;
181 if (doffset
== NOOFFSET
)
182 goto single_block_read
;
183 if (burstbytes
< blksize
* 2)
184 goto single_block_read
;
185 if (nblks
> burstbytes
/ blksize
)
186 nblks
= burstbytes
/ blksize
;
188 bp
= cluster_rbuild(vp
, filesize
, loffset
,
189 doffset
, blksize
, nblks
, bp
, 0);
190 loffset
+= bp
->b_bufsize
;
194 * if it isn't in the cache, then get a chunk from
195 * disk if sequential, otherwise just get the block.
197 bp
->b_flags
|= B_RAM
;
203 * Handle the synchronous read. This only occurs if B_CACHE was
204 * not set. bp (and rbp) could be either a cluster bp or a normal
205 * bp depending on the what cluster_rbuild() decided to do. If
206 * it is a cluster bp, vfs_busy_pages() has already been called.
209 #if defined(CLUSTERDEBUG)
211 kprintf("S(%lld,%d,%d) ",
212 bp
->b_loffset
, bp
->b_bcount
, seqcount
);
214 bp
->b_cmd
= BUF_CMD_READ
;
215 if ((bp
->b_flags
& B_CLUSTER
) == 0)
216 vfs_busy_pages(vp
, bp
);
217 bp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
218 if ((bp
->b_flags
& B_ASYNC
) || bp
->b_bio1
.bio_done
!= NULL
)
220 vn_strategy(vp
, &bp
->b_bio1
);
221 if (bp
->b_flags
& B_ERROR
) {
222 if ((error
= bp
->b_error
) == 0)
230 * If we have been doing sequential I/O, then do some read-ahead.
232 * Only mess with buffers which we can immediately lock. HAMMER
233 * will do device-readahead irrespective of what the blocks
239 loffset
< origoffset
+ seqcount
* blksize
&&
240 loffset
+ blksize
<= filesize
247 if ((rbp
= findblk(vp
, loffset
)) != NULL
) {
248 if (BUF_LOCK(rbp
, LK_EXCLUSIVE
| LK_NOWAIT
)) {
253 rbp
= getblk(vp
, loffset
, blksize
, 0, 0);
254 if ((rbp
->b_flags
& B_CACHE
)) {
260 * An error from the read-ahead bmap has nothing to do
261 * with the caller's original request.
263 tmp_error
= VOP_BMAP(vp
, loffset
, &doffset
,
264 &burstbytes
, NULL
, BUF_CMD_READ
);
265 if (tmp_error
|| doffset
== NOOFFSET
) {
266 rbp
->b_flags
|= B_INVAL
;
271 ntoread
= burstbytes
/ blksize
;
272 nblksread
= (totread
+ blksize
- 1) / blksize
;
273 if (seqcount
< nblksread
)
274 seqcount
= nblksread
;
275 if (ntoread
> seqcount
)
278 rbp
->b_flags
|= B_RAM
/* | B_AGE*/;
280 rbp
= cluster_rbuild(vp
, filesize
, loffset
,
284 rbp
->b_bio2
.bio_offset
= doffset
;
286 #if defined(CLUSTERDEBUG)
289 kprintf("A+(%lld,%d,%lld,%d) ",
290 rbp
->b_loffset
, rbp
->b_bcount
,
291 rbp
->b_loffset
- origoffset
,
294 kprintf("A(%lld,%d,%lld,%d) ",
295 rbp
->b_loffset
, rbp
->b_bcount
,
296 rbp
->b_loffset
- origoffset
,
300 rbp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
301 rbp
->b_flags
|= B_ASYNC
;
302 rbp
->b_cmd
= BUF_CMD_READ
;
304 if ((rbp
->b_flags
& B_CLUSTER
) == 0)
305 vfs_busy_pages(vp
, rbp
);
306 BUF_KERNPROC(rbp
); /* B_ASYNC */
307 vn_strategy(vp
, &rbp
->b_bio1
);
312 return (biowait(reqbp
));
318 * If blocks are contiguous on disk, use this to provide clustered
319 * read ahead. We will read as many blocks as possible sequentially
320 * and then parcel them up into logical blocks in the buffer hash table.
323 cluster_rbuild(struct vnode
*vp
, off_t filesize
, off_t loffset
,
324 off_t doffset
, int blksize
, int run
, struct buf
*fbp
, int doasync
)
326 struct buf
*bp
, *tbp
;
329 int maxiosize
= vmaxiosize(vp
);
334 while (loffset
+ run
* blksize
> filesize
) {
339 tbp
->b_bio2
.bio_offset
= doffset
;
340 if((tbp
->b_flags
& B_MALLOC
) ||
341 ((tbp
->b_flags
& B_VMIO
) == 0) || (run
<= 1)) {
345 bp
= trypbuf(&cluster_pbuf_freecnt
);
350 * We are synthesizing a buffer out of vm_page_t's, but
351 * if the block size is not page aligned then the starting
352 * address may not be either. Inherit the b_data offset
353 * from the original buffer.
355 bp
->b_data
= (char *)((vm_offset_t
)bp
->b_data
|
356 ((vm_offset_t
)tbp
->b_data
& PAGE_MASK
));
357 bp
->b_flags
|= B_ASYNC
| B_CLUSTER
| B_VMIO
;
358 bp
->b_cmd
= BUF_CMD_READ
;
359 bp
->b_bio1
.bio_done
= cluster_callback
;
360 bp
->b_bio1
.bio_caller_info1
.cluster_head
= NULL
;
361 bp
->b_bio1
.bio_caller_info2
.cluster_tail
= NULL
;
362 bp
->b_loffset
= loffset
;
363 bp
->b_bio2
.bio_offset
= doffset
;
364 KASSERT(bp
->b_loffset
!= NOOFFSET
,
365 ("cluster_rbuild: no buffer offset"));
369 bp
->b_xio
.xio_npages
= 0;
371 for (boffset
= doffset
, i
= 0; i
< run
; ++i
, boffset
+= blksize
) {
373 if ((bp
->b_xio
.xio_npages
* PAGE_SIZE
) +
374 round_page(blksize
) > maxiosize
) {
379 * Shortcut some checks and try to avoid buffers that
380 * would block in the lock. The same checks have to
381 * be made again after we officially get the buffer.
383 if ((tbp
= findblk(vp
, loffset
+ i
* blksize
)) != NULL
) {
384 if (BUF_LOCK(tbp
, LK_EXCLUSIVE
| LK_NOWAIT
))
388 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; j
++) {
389 if (tbp
->b_xio
.xio_pages
[j
]->valid
)
393 if (j
!= tbp
->b_xio
.xio_npages
)
396 if (tbp
->b_bcount
!= blksize
)
400 tbp
= getblk(vp
, loffset
+ i
* blksize
, blksize
, 0, 0);
403 * Stop scanning if the buffer is fuly valid
404 * (marked B_CACHE), or locked (may be doing a
405 * background write), or if the buffer is not
406 * VMIO backed. The clustering code can only deal
407 * with VMIO-backed buffers.
409 if ((tbp
->b_flags
& (B_CACHE
|B_LOCKED
)) ||
410 (tbp
->b_flags
& B_VMIO
) == 0 ||
411 (LIST_FIRST(&tbp
->b_dep
) != NULL
&&
419 * The buffer must be completely invalid in order to
420 * take part in the cluster. If it is partially valid
423 for (j
= 0;j
< tbp
->b_xio
.xio_npages
; j
++) {
424 if (tbp
->b_xio
.xio_pages
[j
]->valid
)
427 if (j
!= tbp
->b_xio
.xio_npages
) {
433 * Set a read-ahead mark as appropriate
435 if (i
== 1 || i
== (run
- 1))
436 tbp
->b_flags
|= B_RAM
;
439 * Depress the priority of buffers not explicitly
442 /* tbp->b_flags |= B_AGE; */
445 * Set the block number if it isn't set, otherwise
446 * if it is make sure it matches the block number we
449 if (tbp
->b_bio2
.bio_offset
== NOOFFSET
) {
450 tbp
->b_bio2
.bio_offset
= boffset
;
451 } else if (tbp
->b_bio2
.bio_offset
!= boffset
) {
457 * The first buffer is setup async if doasync is specified.
458 * All other buffers in the cluster are setup async. This
459 * way the caller can decide how to deal with the requested
463 tbp
->b_flags
|= B_ASYNC
;
464 tbp
->b_cmd
= BUF_CMD_READ
;
466 cluster_append(&bp
->b_bio1
, tbp
);
467 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
469 m
= tbp
->b_xio
.xio_pages
[j
];
471 vm_object_pip_add(m
->object
, 1);
472 if ((bp
->b_xio
.xio_npages
== 0) ||
473 (bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
-1] != m
)) {
474 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
475 bp
->b_xio
.xio_npages
++;
477 if ((m
->valid
& VM_PAGE_BITS_ALL
) == VM_PAGE_BITS_ALL
)
478 tbp
->b_xio
.xio_pages
[j
] = bogus_page
;
481 * XXX shouldn't this be += size for both, like in
484 * Don't inherit tbp->b_bufsize as it may be larger due to
485 * a non-page-aligned size. Instead just aggregate using
488 if (tbp
->b_bcount
!= blksize
)
489 kprintf("warning: tbp->b_bcount wrong %d vs %d\n", tbp
->b_bcount
, blksize
);
490 if (tbp
->b_bufsize
!= blksize
)
491 kprintf("warning: tbp->b_bufsize wrong %d vs %d\n", tbp
->b_bufsize
, blksize
);
492 bp
->b_bcount
+= blksize
;
493 bp
->b_bufsize
+= blksize
;
497 * Fully valid pages in the cluster are already good and do not need
498 * to be re-read from disk. Replace the page with bogus_page
500 for (j
= 0; j
< bp
->b_xio
.xio_npages
; j
++) {
501 if ((bp
->b_xio
.xio_pages
[j
]->valid
& VM_PAGE_BITS_ALL
) ==
503 bp
->b_xio
.xio_pages
[j
] = bogus_page
;
506 if (bp
->b_bufsize
> bp
->b_kvasize
) {
507 panic("cluster_rbuild: b_bufsize(%d) > b_kvasize(%d)",
508 bp
->b_bufsize
, bp
->b_kvasize
);
511 pmap_qenter(trunc_page((vm_offset_t
) bp
->b_data
),
512 (vm_page_t
*)bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
517 * Cleanup after a clustered read or write.
518 * This is complicated by the fact that any of the buffers might have
519 * extra memory (if there were no empty buffer headers at allocbuf time)
520 * that we will need to shift around.
522 * The returned bio is &bp->b_bio1
525 cluster_callback(struct bio
*bio
)
527 struct buf
*bp
= bio
->bio_buf
;
532 * Must propogate errors to all the components. A short read (EOF)
533 * is a critical error.
535 if (bp
->b_flags
& B_ERROR
) {
537 } else if (bp
->b_bcount
!= bp
->b_bufsize
) {
538 panic("cluster_callback: unexpected EOF on cluster %p!", bio
);
541 pmap_qremove(trunc_page((vm_offset_t
) bp
->b_data
), bp
->b_xio
.xio_npages
);
543 * Move memory from the large cluster buffer into the component
544 * buffers and mark IO as done on these. Since the memory map
545 * is the same, no actual copying is required.
547 while ((tbp
= bio
->bio_caller_info1
.cluster_head
) != NULL
) {
548 bio
->bio_caller_info1
.cluster_head
= tbp
->b_cluster_next
;
550 tbp
->b_flags
|= B_ERROR
;
551 tbp
->b_error
= error
;
553 tbp
->b_dirtyoff
= tbp
->b_dirtyend
= 0;
554 tbp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
556 * XXX the bdwrite()/bqrelse() issued during
557 * cluster building clears B_RELBUF (see bqrelse()
558 * comment). If direct I/O was specified, we have
559 * to restore it here to allow the buffer and VM
562 if (tbp
->b_flags
& B_DIRECT
)
563 tbp
->b_flags
|= B_RELBUF
;
565 biodone(&tbp
->b_bio1
);
567 relpbuf(bp
, &cluster_pbuf_freecnt
);
573 * Implement modified write build for cluster.
575 * write_behind = 0 write behind disabled
576 * write_behind = 1 write behind normal (default)
577 * write_behind = 2 write behind backed-off
581 cluster_wbuild_wb(struct vnode
*vp
, int blksize
, off_t start_loffset
, int len
)
585 switch(write_behind
) {
587 if (start_loffset
< len
)
589 start_loffset
-= len
;
592 r
= cluster_wbuild(vp
, blksize
, start_loffset
, len
);
602 * Do clustered write for FFS.
605 * 1. Write is not sequential (write asynchronously)
606 * Write is sequential:
607 * 2. beginning of cluster - begin cluster
608 * 3. middle of a cluster - add to cluster
609 * 4. end of a cluster - asynchronously write cluster
612 cluster_write(struct buf
*bp
, off_t filesize
, int blksize
, int seqcount
)
616 int maxclen
, cursize
;
620 if (vp
->v_type
== VREG
)
621 async
= vp
->v_mount
->mnt_flag
& MNT_ASYNC
;
624 loffset
= bp
->b_loffset
;
625 KASSERT(bp
->b_loffset
!= NOOFFSET
,
626 ("cluster_write: no buffer offset"));
628 /* Initialize vnode to beginning of file. */
630 vp
->v_lasta
= vp
->v_clen
= vp
->v_cstart
= vp
->v_lastw
= 0;
632 if (vp
->v_clen
== 0 || loffset
!= vp
->v_lastw
+ blksize
||
633 bp
->b_bio2
.bio_offset
== NOOFFSET
||
634 (bp
->b_bio2
.bio_offset
!= vp
->v_lasta
+ blksize
)) {
635 maxclen
= vmaxiosize(vp
);
636 if (vp
->v_clen
!= 0) {
638 * Next block is not sequential.
640 * If we are not writing at end of file, the process
641 * seeked to another point in the file since its last
642 * write, or we have reached our maximum cluster size,
643 * then push the previous cluster. Otherwise try
644 * reallocating to make it sequential.
646 * Change to algorithm: only push previous cluster if
647 * it was sequential from the point of view of the
648 * seqcount heuristic, otherwise leave the buffer
649 * intact so we can potentially optimize the I/O
650 * later on in the buf_daemon or update daemon
653 cursize
= vp
->v_lastw
- vp
->v_cstart
+ blksize
;
654 if (bp
->b_loffset
+ blksize
!= filesize
||
655 loffset
!= vp
->v_lastw
+ blksize
|| vp
->v_clen
<= cursize
) {
656 if (!async
&& seqcount
> 0) {
657 cluster_wbuild_wb(vp
, blksize
,
658 vp
->v_cstart
, cursize
);
661 struct buf
**bpp
, **endbp
;
662 struct cluster_save
*buflist
;
664 buflist
= cluster_collectbufs(vp
, bp
, blksize
);
665 endbp
= &buflist
->bs_children
666 [buflist
->bs_nchildren
- 1];
667 if (VOP_REALLOCBLKS(vp
, buflist
)) {
669 * Failed, push the previous cluster
670 * if *really* writing sequentially
671 * in the logical file (seqcount > 1),
672 * otherwise delay it in the hopes that
673 * the low level disk driver can
674 * optimize the write ordering.
676 for (bpp
= buflist
->bs_children
;
679 kfree(buflist
, M_SEGMENT
);
681 cluster_wbuild_wb(vp
,
682 blksize
, vp
->v_cstart
,
687 * Succeeded, keep building cluster.
689 for (bpp
= buflist
->bs_children
;
692 kfree(buflist
, M_SEGMENT
);
693 vp
->v_lastw
= loffset
;
694 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
700 * Consider beginning a cluster. If at end of file, make
701 * cluster as large as possible, otherwise find size of
704 if ((vp
->v_type
== VREG
) &&
705 bp
->b_loffset
+ blksize
!= filesize
&&
706 (bp
->b_bio2
.bio_offset
== NOOFFSET
) &&
707 (VOP_BMAP(vp
, loffset
, &bp
->b_bio2
.bio_offset
, &maxclen
, NULL
, BUF_CMD_WRITE
) ||
708 bp
->b_bio2
.bio_offset
== NOOFFSET
)) {
711 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
712 vp
->v_cstart
= loffset
+ blksize
;
713 vp
->v_lastw
= loffset
;
716 if (maxclen
> blksize
)
717 vp
->v_clen
= maxclen
- blksize
;
720 if (!async
&& vp
->v_clen
== 0) { /* I/O not contiguous */
721 vp
->v_cstart
= loffset
+ blksize
;
723 } else { /* Wait for rest of cluster */
724 vp
->v_cstart
= loffset
;
727 } else if (loffset
== vp
->v_cstart
+ vp
->v_clen
) {
729 * At end of cluster, write it out if seqcount tells us we
730 * are operating sequentially, otherwise let the buf or
731 * update daemon handle it.
735 cluster_wbuild_wb(vp
, blksize
, vp
->v_cstart
,
736 vp
->v_clen
+ blksize
);
738 vp
->v_cstart
= loffset
+ blksize
;
739 } else if (vm_page_count_severe()) {
741 * We are low on memory, get it going NOW
746 * In the middle of a cluster, so just delay the I/O for now.
750 vp
->v_lastw
= loffset
;
751 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
756 * This is an awful lot like cluster_rbuild...wish they could be combined.
757 * The last lbn argument is the current block on which I/O is being
758 * performed. Check to see that it doesn't fall in the middle of
759 * the current block (if last_bp == NULL).
762 cluster_wbuild(struct vnode
*vp
, int blksize
, off_t start_loffset
, int bytes
)
764 struct buf
*bp
, *tbp
;
766 int totalwritten
= 0;
767 int maxiosize
= vmaxiosize(vp
);
772 * If the buffer is not delayed-write (i.e. dirty), or it
773 * is delayed-write but either locked or inval, it cannot
774 * partake in the clustered write.
776 if (((tbp
= findblk(vp
, start_loffset
)) == NULL
) ||
777 ((tbp
->b_flags
& (B_LOCKED
| B_INVAL
| B_DELWRI
)) != B_DELWRI
) ||
778 (LIST_FIRST(&tbp
->b_dep
) != NULL
&& buf_checkwrite(tbp
)) ||
779 BUF_LOCK(tbp
, LK_EXCLUSIVE
| LK_NOWAIT
)) {
780 start_loffset
+= blksize
;
786 KKASSERT(tbp
->b_cmd
== BUF_CMD_DONE
);
790 * Extra memory in the buffer, punt on this buffer.
791 * XXX we could handle this in most cases, but we would
792 * have to push the extra memory down to after our max
793 * possible cluster size and then potentially pull it back
794 * up if the cluster was terminated prematurely--too much
797 if (((tbp
->b_flags
& (B_CLUSTEROK
|B_MALLOC
)) != B_CLUSTEROK
) ||
798 (tbp
->b_bcount
!= tbp
->b_bufsize
) ||
799 (tbp
->b_bcount
!= blksize
) ||
800 (bytes
== blksize
) ||
801 ((bp
= getpbuf(&cluster_pbuf_freecnt
)) == NULL
)) {
802 totalwritten
+= tbp
->b_bufsize
;
804 start_loffset
+= blksize
;
810 * Set up the pbuf. Track our append point with b_bcount
811 * and b_bufsize. b_bufsize is not used by the device but
812 * our caller uses it to loop clusters and we use it to
813 * detect a premature EOF on the block device.
817 bp
->b_xio
.xio_npages
= 0;
818 bp
->b_loffset
= tbp
->b_loffset
;
819 bp
->b_bio2
.bio_offset
= tbp
->b_bio2
.bio_offset
;
822 * We are synthesizing a buffer out of vm_page_t's, but
823 * if the block size is not page aligned then the starting
824 * address may not be either. Inherit the b_data offset
825 * from the original buffer.
827 bp
->b_data
= (char *)((vm_offset_t
)bp
->b_data
|
828 ((vm_offset_t
)tbp
->b_data
& PAGE_MASK
));
829 bp
->b_flags
&= ~B_ERROR
;
830 bp
->b_flags
|= B_CLUSTER
| B_BNOCLIP
|
831 (tbp
->b_flags
& (B_VMIO
| B_NEEDCOMMIT
));
832 bp
->b_bio1
.bio_done
= cluster_callback
;
833 bp
->b_bio1
.bio_caller_info1
.cluster_head
= NULL
;
834 bp
->b_bio1
.bio_caller_info2
.cluster_tail
= NULL
;
836 * From this location in the file, scan forward to see
837 * if there are buffers with adjacent data that need to
838 * be written as well.
840 for (i
= 0; i
< bytes
; (i
+= blksize
), (start_loffset
+= blksize
)) {
841 if (i
!= 0) { /* If not the first buffer */
844 * If the adjacent data is not even in core it
845 * can't need to be written.
847 if ((tbp
= findblk(vp
, start_loffset
)) == NULL
) {
853 * If it IS in core, but has different
854 * characteristics, or is locked (which
855 * means it could be undergoing a background
856 * I/O or be in a weird state), then don't
859 if ((tbp
->b_flags
& (B_VMIO
| B_CLUSTEROK
|
860 B_INVAL
| B_DELWRI
| B_NEEDCOMMIT
))
861 != (B_DELWRI
| B_CLUSTEROK
|
862 (bp
->b_flags
& (B_VMIO
| B_NEEDCOMMIT
))) ||
863 (tbp
->b_flags
& B_LOCKED
) ||
864 (LIST_FIRST(&tbp
->b_dep
) != NULL
&& buf_checkwrite(tbp
)) ||
865 BUF_LOCK(tbp
, LK_EXCLUSIVE
| LK_NOWAIT
)) {
871 * Check that the combined cluster
872 * would make sense with regard to pages
873 * and would not be too large
875 if ((tbp
->b_bcount
!= blksize
) ||
876 ((bp
->b_bio2
.bio_offset
+ i
) !=
877 tbp
->b_bio2
.bio_offset
) ||
878 ((tbp
->b_xio
.xio_npages
+ bp
->b_xio
.xio_npages
) >
879 (maxiosize
/ PAGE_SIZE
))) {
885 * Ok, it's passed all the tests,
886 * so remove it from the free list
887 * and mark it busy. We will use it.
890 KKASSERT(tbp
->b_cmd
== BUF_CMD_DONE
);
892 } /* end of code for non-first buffers only */
895 * If the IO is via the VM then we do some
896 * special VM hackery (yuck). Since the buffer's
897 * block size may not be page-aligned it is possible
898 * for a page to be shared between two buffers. We
899 * have to get rid of the duplication when building
902 if (tbp
->b_flags
& B_VMIO
) {
905 if (i
!= 0) { /* if not first buffer */
906 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
907 m
= tbp
->b_xio
.xio_pages
[j
];
908 if (m
->flags
& PG_BUSY
) {
915 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
916 m
= tbp
->b_xio
.xio_pages
[j
];
918 vm_object_pip_add(m
->object
, 1);
919 if ((bp
->b_xio
.xio_npages
== 0) ||
920 (bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
- 1] != m
)) {
921 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
922 bp
->b_xio
.xio_npages
++;
926 bp
->b_bcount
+= blksize
;
927 bp
->b_bufsize
+= blksize
;
931 tbp
->b_flags
&= ~B_ERROR
;
932 tbp
->b_flags
|= B_ASYNC
;
933 tbp
->b_cmd
= BUF_CMD_WRITE
;
936 cluster_append(&bp
->b_bio1
, tbp
);
939 * check for latent dependencies to be handled
941 if (LIST_FIRST(&tbp
->b_dep
) != NULL
)
945 pmap_qenter(trunc_page((vm_offset_t
) bp
->b_data
),
946 (vm_page_t
*) bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
947 if (bp
->b_bufsize
> bp
->b_kvasize
) {
949 "cluster_wbuild: b_bufsize(%d) > b_kvasize(%d)\n",
950 bp
->b_bufsize
, bp
->b_kvasize
);
952 totalwritten
+= bp
->b_bufsize
;
954 bp
->b_dirtyend
= bp
->b_bufsize
;
955 bp
->b_flags
|= B_ASYNC
;
956 bp
->b_cmd
= BUF_CMD_WRITE
;
957 vfs_busy_pages(vp
, bp
);
958 bp
->b_runningbufspace
= bp
->b_bufsize
;
959 if (bp
->b_runningbufspace
) {
960 runningbufspace
+= bp
->b_runningbufspace
;
963 BUF_KERNPROC(bp
); /* B_ASYNC */
964 vn_strategy(vp
, &bp
->b_bio1
);
972 * Collect together all the buffers in a cluster.
973 * Plus add one additional buffer.
975 static struct cluster_save
*
976 cluster_collectbufs(struct vnode
*vp
, struct buf
*last_bp
, int blksize
)
978 struct cluster_save
*buflist
;
983 len
= (int)(vp
->v_lastw
- vp
->v_cstart
+ blksize
) / blksize
;
984 buflist
= kmalloc(sizeof(struct buf
*) * (len
+ 1) + sizeof(*buflist
),
985 M_SEGMENT
, M_WAITOK
);
986 buflist
->bs_nchildren
= 0;
987 buflist
->bs_children
= (struct buf
**) (buflist
+ 1);
988 for (loffset
= vp
->v_cstart
, i
= 0; i
< len
; (loffset
+= blksize
), i
++) {
989 (void) bread(vp
, loffset
, last_bp
->b_bcount
, &bp
);
990 buflist
->bs_children
[i
] = bp
;
991 if (bp
->b_bio2
.bio_offset
== NOOFFSET
) {
992 VOP_BMAP(bp
->b_vp
, bp
->b_loffset
,
993 &bp
->b_bio2
.bio_offset
,
994 NULL
, NULL
, BUF_CMD_WRITE
);
997 buflist
->bs_children
[i
] = bp
= last_bp
;
998 if (bp
->b_bio2
.bio_offset
== NOOFFSET
) {
999 VOP_BMAP(bp
->b_vp
, bp
->b_loffset
, &bp
->b_bio2
.bio_offset
,
1000 NULL
, NULL
, BUF_CMD_WRITE
);
1002 buflist
->bs_nchildren
= i
+ 1;
1007 cluster_append(struct bio
*bio
, struct buf
*tbp
)
1009 tbp
->b_cluster_next
= NULL
;
1010 if (bio
->bio_caller_info1
.cluster_head
== NULL
) {
1011 bio
->bio_caller_info1
.cluster_head
= tbp
;
1012 bio
->bio_caller_info2
.cluster_tail
= tbp
;
1014 bio
->bio_caller_info2
.cluster_tail
->b_cluster_next
= tbp
;
1015 bio
->bio_caller_info2
.cluster_tail
= tbp
;