3 * The Regents of the University of California. All rights reserved.
4 * Modifications/enhancements:
5 * Copyright (c) 1995 John S. Dyson. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by the University of
18 * California, Berkeley and its contributors.
19 * 4. Neither the name of the University nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * @(#)vfs_cluster.c 8.7 (Berkeley) 2/13/94
36 * $FreeBSD: src/sys/kern/vfs_cluster.c,v 1.92.2.9 2001/11/18 07:10:59 dillon Exp $
37 * $DragonFly: src/sys/kern/vfs_cluster.c,v 1.40 2008/07/14 03:09:00 dillon Exp $
40 #include "opt_debug_cluster.h"
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
47 #include <sys/vnode.h>
48 #include <sys/malloc.h>
49 #include <sys/mount.h>
50 #include <sys/resourcevar.h>
51 #include <sys/vmmeter.h>
53 #include <vm/vm_object.h>
54 #include <vm/vm_page.h>
55 #include <sys/sysctl.h>
57 #include <vm/vm_page2.h>
59 #include <machine/limits.h>
61 #if defined(CLUSTERDEBUG)
62 #include <sys/sysctl.h>
63 static int rcluster
= 0;
64 SYSCTL_INT(_debug
, OID_AUTO
, rcluster
, CTLFLAG_RW
, &rcluster
, 0, "");
67 static MALLOC_DEFINE(M_SEGMENT
, "cluster_save", "cluster_save buffer");
69 static struct cluster_save
*
70 cluster_collectbufs (struct vnode
*vp
, struct buf
*last_bp
,
73 cluster_rbuild (struct vnode
*vp
, off_t filesize
, off_t loffset
,
74 off_t doffset
, int blksize
, int run
,
76 static void cluster_callback (struct bio
*);
79 static int write_behind
= 1;
80 SYSCTL_INT(_vfs
, OID_AUTO
, write_behind
, CTLFLAG_RW
, &write_behind
, 0, "");
82 extern vm_page_t bogus_page
;
84 extern int cluster_pbuf_freecnt
;
87 * Maximum number of blocks for read-ahead.
92 * This replaces bread.
95 cluster_read(struct vnode
*vp
, off_t filesize
, off_t loffset
,
96 int blksize
, size_t resid
, int seqcount
, struct buf
**bpp
)
98 struct buf
*bp
, *rbp
, *reqbp
;
103 int maxra
, racluster
;
107 totread
= (resid
> INT_MAX
) ? INT_MAX
: (int)resid
;
110 * Try to limit the amount of read-ahead by a few
111 * ad-hoc parameters. This needs work!!!
113 racluster
= vmaxiosize(vp
) / blksize
;
114 maxra
= 2 * racluster
+ (totread
/ blksize
);
121 * Get the requested block.
123 *bpp
= reqbp
= bp
= getblk(vp
, loffset
, blksize
, 0, 0);
124 origoffset
= loffset
;
127 * if it is in the cache, then check to see if the reads have been
128 * sequential. If they have, then try some read-ahead, otherwise
129 * back-off on prospective read-aheads.
131 if (bp
->b_flags
& B_CACHE
) {
134 } else if ((bp
->b_flags
& B_RAM
) == 0) {
138 bp
->b_flags
&= ~B_RAM
;
141 * Set read-ahead-mark only if we can passively lock
142 * the buffer. Note that with these flags the bp
143 * could very exist even though NULL is returned.
145 for (i
= 1; i
< maxra
; i
++) {
146 tbp
= findblk(vp
, loffset
+ i
* blksize
,
150 if (((i
% racluster
) == (racluster
- 1)) ||
151 (i
== (maxra
- 1))) {
152 tbp
->b_flags
|= B_RAM
;
158 loffset
+= i
* blksize
;
162 off_t firstread
= bp
->b_loffset
;
166 * Set-up synchronous read for bp.
168 bp
->b_cmd
= BUF_CMD_READ
;
169 bp
->b_bio1
.bio_done
= biodone_sync
;
170 bp
->b_bio1
.bio_flags
|= BIO_SYNC
;
172 KASSERT(firstread
!= NOOFFSET
,
173 ("cluster_read: no buffer offset"));
174 if (firstread
+ totread
> filesize
)
175 totread
= (int)(filesize
- firstread
);
176 nblks
= totread
/ blksize
;
180 if (nblks
> racluster
)
183 error
= VOP_BMAP(vp
, loffset
, &doffset
,
184 &burstbytes
, NULL
, BUF_CMD_READ
);
186 goto single_block_read
;
187 if (doffset
== NOOFFSET
)
188 goto single_block_read
;
189 if (burstbytes
< blksize
* 2)
190 goto single_block_read
;
191 if (nblks
> burstbytes
/ blksize
)
192 nblks
= burstbytes
/ blksize
;
194 bp
= cluster_rbuild(vp
, filesize
, loffset
,
195 doffset
, blksize
, nblks
, bp
);
196 loffset
+= bp
->b_bufsize
;
200 * if it isn't in the cache, then get a chunk from
201 * disk if sequential, otherwise just get the block.
203 bp
->b_flags
|= B_RAM
;
209 * If B_CACHE was not set issue bp. bp will either be an
210 * asynchronous cluster buf or a synchronous single-buf.
211 * If it is a single buf it will be the same as reqbp.
213 * NOTE: Once an async cluster buf is issued bp becomes invalid.
216 #if defined(CLUSTERDEBUG)
218 kprintf("S(%lld,%d,%d) ",
219 bp
->b_loffset
, bp
->b_bcount
, seqcount
);
221 if ((bp
->b_flags
& B_CLUSTER
) == 0)
222 vfs_busy_pages(vp
, bp
);
223 bp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
224 vn_strategy(vp
, &bp
->b_bio1
);
230 * If we have been doing sequential I/O, then do some read-ahead.
232 * Only mess with buffers which we can immediately lock. HAMMER
233 * will do device-readahead irrespective of what the blocks
239 loffset
< origoffset
+ seqcount
* blksize
&&
240 loffset
+ blksize
<= filesize
247 rbp
= getblk(vp
, loffset
, blksize
,
248 GETBLK_SZMATCH
|GETBLK_NOWAIT
, 0);
251 if ((rbp
->b_flags
& B_CACHE
)) {
257 * An error from the read-ahead bmap has nothing to do
258 * with the caller's original request.
260 tmp_error
= VOP_BMAP(vp
, loffset
, &doffset
,
261 &burstbytes
, NULL
, BUF_CMD_READ
);
262 if (tmp_error
|| doffset
== NOOFFSET
) {
263 rbp
->b_flags
|= B_INVAL
;
268 ntoread
= burstbytes
/ blksize
;
269 nblksread
= (totread
+ blksize
- 1) / blksize
;
270 if (seqcount
< nblksread
)
271 seqcount
= nblksread
;
272 if (ntoread
> seqcount
)
278 rbp
->b_cmd
= BUF_CMD_READ
;
279 rbp
->b_flags
|= B_RAM
/* | B_AGE*/;
282 rbp
= cluster_rbuild(vp
, filesize
, loffset
,
286 rbp
->b_bio2
.bio_offset
= doffset
;
288 #if defined(CLUSTERDEBUG)
291 kprintf("A+(%lld,%d,%lld,%d) ",
292 rbp
->b_loffset
, rbp
->b_bcount
,
293 rbp
->b_loffset
- origoffset
,
296 kprintf("A(%lld,%d,%lld,%d) ",
297 rbp
->b_loffset
, rbp
->b_bcount
,
298 rbp
->b_loffset
- origoffset
,
302 rbp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
304 if ((rbp
->b_flags
& B_CLUSTER
) == 0)
305 vfs_busy_pages(vp
, rbp
);
307 vn_strategy(vp
, &rbp
->b_bio1
);
308 /* rbp invalid now */
312 * Wait for our original buffer to complete its I/O. reqbp will
313 * be NULL if the original buffer was B_CACHE. We are returning
314 * (*bpp) which is the same as reqbp when reqbp != NULL.
318 KKASSERT(reqbp
->b_bio1
.bio_flags
& BIO_SYNC
);
319 error
= biowait(&reqbp
->b_bio1
, "clurd");
325 * If blocks are contiguous on disk, use this to provide clustered
326 * read ahead. We will read as many blocks as possible sequentially
327 * and then parcel them up into logical blocks in the buffer hash table.
329 * This function either returns a cluster buf or it returns fbp. fbp is
330 * already expected to be set up as a synchronous or asynchronous request.
332 * If a cluster buf is returned it will always be async.
335 cluster_rbuild(struct vnode
*vp
, off_t filesize
, off_t loffset
, off_t doffset
,
336 int blksize
, int run
, struct buf
*fbp
)
338 struct buf
*bp
, *tbp
;
341 int maxiosize
= vmaxiosize(vp
);
346 while (loffset
+ run
* blksize
> filesize
) {
351 tbp
->b_bio2
.bio_offset
= doffset
;
352 if((tbp
->b_flags
& B_MALLOC
) ||
353 ((tbp
->b_flags
& B_VMIO
) == 0) || (run
<= 1)) {
357 bp
= trypbuf(&cluster_pbuf_freecnt
);
363 * We are synthesizing a buffer out of vm_page_t's, but
364 * if the block size is not page aligned then the starting
365 * address may not be either. Inherit the b_data offset
366 * from the original buffer.
368 bp
->b_data
= (char *)((vm_offset_t
)bp
->b_data
|
369 ((vm_offset_t
)tbp
->b_data
& PAGE_MASK
));
370 bp
->b_flags
|= B_CLUSTER
| B_VMIO
;
371 bp
->b_cmd
= BUF_CMD_READ
;
372 bp
->b_bio1
.bio_done
= cluster_callback
; /* default to async */
373 bp
->b_bio1
.bio_caller_info1
.cluster_head
= NULL
;
374 bp
->b_bio1
.bio_caller_info2
.cluster_tail
= NULL
;
375 bp
->b_loffset
= loffset
;
376 bp
->b_bio2
.bio_offset
= doffset
;
377 KASSERT(bp
->b_loffset
!= NOOFFSET
,
378 ("cluster_rbuild: no buffer offset"));
382 bp
->b_xio
.xio_npages
= 0;
384 for (boffset
= doffset
, i
= 0; i
< run
; ++i
, boffset
+= blksize
) {
386 if ((bp
->b_xio
.xio_npages
* PAGE_SIZE
) +
387 round_page(blksize
) > maxiosize
) {
392 * Shortcut some checks and try to avoid buffers that
393 * would block in the lock. The same checks have to
394 * be made again after we officially get the buffer.
396 tbp
= getblk(vp
, loffset
+ i
* blksize
, blksize
,
397 GETBLK_SZMATCH
|GETBLK_NOWAIT
, 0);
400 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; j
++) {
401 if (tbp
->b_xio
.xio_pages
[j
]->valid
)
404 if (j
!= tbp
->b_xio
.xio_npages
) {
410 * Stop scanning if the buffer is fuly valid
411 * (marked B_CACHE), or locked (may be doing a
412 * background write), or if the buffer is not
413 * VMIO backed. The clustering code can only deal
414 * with VMIO-backed buffers.
416 if ((tbp
->b_flags
& (B_CACHE
|B_LOCKED
)) ||
417 (tbp
->b_flags
& B_VMIO
) == 0 ||
418 (LIST_FIRST(&tbp
->b_dep
) != NULL
&&
426 * The buffer must be completely invalid in order to
427 * take part in the cluster. If it is partially valid
430 for (j
= 0;j
< tbp
->b_xio
.xio_npages
; j
++) {
431 if (tbp
->b_xio
.xio_pages
[j
]->valid
)
434 if (j
!= tbp
->b_xio
.xio_npages
) {
440 * Set a read-ahead mark as appropriate
442 if (i
== 1 || i
== (run
- 1))
443 tbp
->b_flags
|= B_RAM
;
446 * Depress the priority of buffers not explicitly
449 /* tbp->b_flags |= B_AGE; */
452 * Set the block number if it isn't set, otherwise
453 * if it is make sure it matches the block number we
456 if (tbp
->b_bio2
.bio_offset
== NOOFFSET
) {
457 tbp
->b_bio2
.bio_offset
= boffset
;
458 } else if (tbp
->b_bio2
.bio_offset
!= boffset
) {
465 * The passed-in tbp (i == 0) will already be set up for
466 * async or sync operation. All other tbp's acquire in
467 * our loop are set up for async operation.
469 tbp
->b_cmd
= BUF_CMD_READ
;
471 cluster_append(&bp
->b_bio1
, tbp
);
472 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
474 m
= tbp
->b_xio
.xio_pages
[j
];
476 vm_object_pip_add(m
->object
, 1);
477 if ((bp
->b_xio
.xio_npages
== 0) ||
478 (bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
-1] != m
)) {
479 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
480 bp
->b_xio
.xio_npages
++;
482 if ((m
->valid
& VM_PAGE_BITS_ALL
) == VM_PAGE_BITS_ALL
)
483 tbp
->b_xio
.xio_pages
[j
] = bogus_page
;
486 * XXX shouldn't this be += size for both, like in
489 * Don't inherit tbp->b_bufsize as it may be larger due to
490 * a non-page-aligned size. Instead just aggregate using
493 if (tbp
->b_bcount
!= blksize
)
494 kprintf("warning: tbp->b_bcount wrong %d vs %d\n", tbp
->b_bcount
, blksize
);
495 if (tbp
->b_bufsize
!= blksize
)
496 kprintf("warning: tbp->b_bufsize wrong %d vs %d\n", tbp
->b_bufsize
, blksize
);
497 bp
->b_bcount
+= blksize
;
498 bp
->b_bufsize
+= blksize
;
502 * Fully valid pages in the cluster are already good and do not need
503 * to be re-read from disk. Replace the page with bogus_page
505 for (j
= 0; j
< bp
->b_xio
.xio_npages
; j
++) {
506 if ((bp
->b_xio
.xio_pages
[j
]->valid
& VM_PAGE_BITS_ALL
) ==
508 bp
->b_xio
.xio_pages
[j
] = bogus_page
;
511 if (bp
->b_bufsize
> bp
->b_kvasize
) {
512 panic("cluster_rbuild: b_bufsize(%d) > b_kvasize(%d)",
513 bp
->b_bufsize
, bp
->b_kvasize
);
515 pmap_qenter(trunc_page((vm_offset_t
) bp
->b_data
),
516 (vm_page_t
*)bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
522 * Cleanup after a clustered read or write.
523 * This is complicated by the fact that any of the buffers might have
524 * extra memory (if there were no empty buffer headers at allocbuf time)
525 * that we will need to shift around.
527 * The returned bio is &bp->b_bio1
530 cluster_callback(struct bio
*bio
)
532 struct buf
*bp
= bio
->bio_buf
;
537 * Must propogate errors to all the components. A short read (EOF)
538 * is a critical error.
540 if (bp
->b_flags
& B_ERROR
) {
542 } else if (bp
->b_bcount
!= bp
->b_bufsize
) {
543 panic("cluster_callback: unexpected EOF on cluster %p!", bio
);
546 pmap_qremove(trunc_page((vm_offset_t
) bp
->b_data
), bp
->b_xio
.xio_npages
);
548 * Move memory from the large cluster buffer into the component
549 * buffers and mark IO as done on these. Since the memory map
550 * is the same, no actual copying is required.
552 while ((tbp
= bio
->bio_caller_info1
.cluster_head
) != NULL
) {
553 bio
->bio_caller_info1
.cluster_head
= tbp
->b_cluster_next
;
555 tbp
->b_flags
|= B_ERROR
;
556 tbp
->b_error
= error
;
558 tbp
->b_dirtyoff
= tbp
->b_dirtyend
= 0;
559 tbp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
561 * XXX the bdwrite()/bqrelse() issued during
562 * cluster building clears B_RELBUF (see bqrelse()
563 * comment). If direct I/O was specified, we have
564 * to restore it here to allow the buffer and VM
567 if (tbp
->b_flags
& B_DIRECT
)
568 tbp
->b_flags
|= B_RELBUF
;
570 biodone(&tbp
->b_bio1
);
572 relpbuf(bp
, &cluster_pbuf_freecnt
);
578 * Implement modified write build for cluster.
580 * write_behind = 0 write behind disabled
581 * write_behind = 1 write behind normal (default)
582 * write_behind = 2 write behind backed-off
586 cluster_wbuild_wb(struct vnode
*vp
, int blksize
, off_t start_loffset
, int len
)
590 switch(write_behind
) {
592 if (start_loffset
< len
)
594 start_loffset
-= len
;
597 r
= cluster_wbuild(vp
, blksize
, start_loffset
, len
);
607 * Do clustered write for FFS.
610 * 1. Write is not sequential (write asynchronously)
611 * Write is sequential:
612 * 2. beginning of cluster - begin cluster
613 * 3. middle of a cluster - add to cluster
614 * 4. end of a cluster - asynchronously write cluster
617 cluster_write(struct buf
*bp
, off_t filesize
, int blksize
, int seqcount
)
621 int maxclen
, cursize
;
625 if (vp
->v_type
== VREG
)
626 async
= vp
->v_mount
->mnt_flag
& MNT_ASYNC
;
629 loffset
= bp
->b_loffset
;
630 KASSERT(bp
->b_loffset
!= NOOFFSET
,
631 ("cluster_write: no buffer offset"));
633 /* Initialize vnode to beginning of file. */
635 vp
->v_lasta
= vp
->v_clen
= vp
->v_cstart
= vp
->v_lastw
= 0;
637 if (vp
->v_clen
== 0 || loffset
!= vp
->v_lastw
+ blksize
||
638 bp
->b_bio2
.bio_offset
== NOOFFSET
||
639 (bp
->b_bio2
.bio_offset
!= vp
->v_lasta
+ blksize
)) {
640 maxclen
= vmaxiosize(vp
);
641 if (vp
->v_clen
!= 0) {
643 * Next block is not sequential.
645 * If we are not writing at end of file, the process
646 * seeked to another point in the file since its last
647 * write, or we have reached our maximum cluster size,
648 * then push the previous cluster. Otherwise try
649 * reallocating to make it sequential.
651 * Change to algorithm: only push previous cluster if
652 * it was sequential from the point of view of the
653 * seqcount heuristic, otherwise leave the buffer
654 * intact so we can potentially optimize the I/O
655 * later on in the buf_daemon or update daemon
658 cursize
= vp
->v_lastw
- vp
->v_cstart
+ blksize
;
659 if (bp
->b_loffset
+ blksize
!= filesize
||
660 loffset
!= vp
->v_lastw
+ blksize
|| vp
->v_clen
<= cursize
) {
661 if (!async
&& seqcount
> 0) {
662 cluster_wbuild_wb(vp
, blksize
,
663 vp
->v_cstart
, cursize
);
666 struct buf
**bpp
, **endbp
;
667 struct cluster_save
*buflist
;
669 buflist
= cluster_collectbufs(vp
, bp
, blksize
);
670 endbp
= &buflist
->bs_children
671 [buflist
->bs_nchildren
- 1];
672 if (VOP_REALLOCBLKS(vp
, buflist
)) {
674 * Failed, push the previous cluster
675 * if *really* writing sequentially
676 * in the logical file (seqcount > 1),
677 * otherwise delay it in the hopes that
678 * the low level disk driver can
679 * optimize the write ordering.
681 for (bpp
= buflist
->bs_children
;
684 kfree(buflist
, M_SEGMENT
);
686 cluster_wbuild_wb(vp
,
687 blksize
, vp
->v_cstart
,
692 * Succeeded, keep building cluster.
694 for (bpp
= buflist
->bs_children
;
697 kfree(buflist
, M_SEGMENT
);
698 vp
->v_lastw
= loffset
;
699 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
705 * Consider beginning a cluster. If at end of file, make
706 * cluster as large as possible, otherwise find size of
709 if ((vp
->v_type
== VREG
) &&
710 bp
->b_loffset
+ blksize
!= filesize
&&
711 (bp
->b_bio2
.bio_offset
== NOOFFSET
) &&
712 (VOP_BMAP(vp
, loffset
, &bp
->b_bio2
.bio_offset
, &maxclen
, NULL
, BUF_CMD_WRITE
) ||
713 bp
->b_bio2
.bio_offset
== NOOFFSET
)) {
716 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
717 vp
->v_cstart
= loffset
+ blksize
;
718 vp
->v_lastw
= loffset
;
721 if (maxclen
> blksize
)
722 vp
->v_clen
= maxclen
- blksize
;
725 if (!async
&& vp
->v_clen
== 0) { /* I/O not contiguous */
726 vp
->v_cstart
= loffset
+ blksize
;
728 } else { /* Wait for rest of cluster */
729 vp
->v_cstart
= loffset
;
732 } else if (loffset
== vp
->v_cstart
+ vp
->v_clen
) {
734 * At end of cluster, write it out if seqcount tells us we
735 * are operating sequentially, otherwise let the buf or
736 * update daemon handle it.
740 cluster_wbuild_wb(vp
, blksize
, vp
->v_cstart
,
741 vp
->v_clen
+ blksize
);
743 vp
->v_cstart
= loffset
+ blksize
;
744 } else if (vm_page_count_severe()) {
746 * We are low on memory, get it going NOW
751 * In the middle of a cluster, so just delay the I/O for now.
755 vp
->v_lastw
= loffset
;
756 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
761 * This is an awful lot like cluster_rbuild...wish they could be combined.
762 * The last lbn argument is the current block on which I/O is being
763 * performed. Check to see that it doesn't fall in the middle of
764 * the current block (if last_bp == NULL).
767 cluster_wbuild(struct vnode
*vp
, int blksize
, off_t start_loffset
, int bytes
)
769 struct buf
*bp
, *tbp
;
771 int totalwritten
= 0;
772 int maxiosize
= vmaxiosize(vp
);
776 * If the buffer is not delayed-write (i.e. dirty), or it
777 * is delayed-write but either locked or inval, it cannot
778 * partake in the clustered write.
780 tbp
= findblk(vp
, start_loffset
, FINDBLK_NBLOCK
);
782 (tbp
->b_flags
& (B_LOCKED
| B_INVAL
| B_DELWRI
)) != B_DELWRI
||
783 (LIST_FIRST(&tbp
->b_dep
) && buf_checkwrite(tbp
))) {
786 start_loffset
+= blksize
;
791 KKASSERT(tbp
->b_cmd
== BUF_CMD_DONE
);
794 * Extra memory in the buffer, punt on this buffer.
795 * XXX we could handle this in most cases, but we would
796 * have to push the extra memory down to after our max
797 * possible cluster size and then potentially pull it back
798 * up if the cluster was terminated prematurely--too much
801 if (((tbp
->b_flags
& (B_CLUSTEROK
|B_MALLOC
)) != B_CLUSTEROK
) ||
802 (tbp
->b_bcount
!= tbp
->b_bufsize
) ||
803 (tbp
->b_bcount
!= blksize
) ||
804 (bytes
== blksize
) ||
805 ((bp
= getpbuf(&cluster_pbuf_freecnt
)) == NULL
)) {
806 totalwritten
+= tbp
->b_bufsize
;
808 start_loffset
+= blksize
;
814 * Set up the pbuf. Track our append point with b_bcount
815 * and b_bufsize. b_bufsize is not used by the device but
816 * our caller uses it to loop clusters and we use it to
817 * detect a premature EOF on the block device.
821 bp
->b_xio
.xio_npages
= 0;
822 bp
->b_loffset
= tbp
->b_loffset
;
823 bp
->b_bio2
.bio_offset
= tbp
->b_bio2
.bio_offset
;
826 * We are synthesizing a buffer out of vm_page_t's, but
827 * if the block size is not page aligned then the starting
828 * address may not be either. Inherit the b_data offset
829 * from the original buffer.
831 bp
->b_data
= (char *)((vm_offset_t
)bp
->b_data
|
832 ((vm_offset_t
)tbp
->b_data
& PAGE_MASK
));
833 bp
->b_flags
&= ~B_ERROR
;
834 bp
->b_flags
|= B_CLUSTER
| B_BNOCLIP
|
835 (tbp
->b_flags
& (B_VMIO
| B_NEEDCOMMIT
));
836 bp
->b_bio1
.bio_caller_info1
.cluster_head
= NULL
;
837 bp
->b_bio1
.bio_caller_info2
.cluster_tail
= NULL
;
840 * From this location in the file, scan forward to see
841 * if there are buffers with adjacent data that need to
842 * be written as well.
844 for (i
= 0; i
< bytes
; (i
+= blksize
), (start_loffset
+= blksize
)) {
845 if (i
!= 0) { /* If not the first buffer */
846 tbp
= findblk(vp
, start_loffset
,
849 * Buffer not found or could not be locked
856 * If it IS in core, but has different
857 * characteristics, then don't cluster
860 if ((tbp
->b_flags
& (B_VMIO
| B_CLUSTEROK
|
861 B_INVAL
| B_DELWRI
| B_NEEDCOMMIT
))
862 != (B_DELWRI
| B_CLUSTEROK
|
863 (bp
->b_flags
& (B_VMIO
| B_NEEDCOMMIT
))) ||
864 (tbp
->b_flags
& B_LOCKED
) ||
865 (LIST_FIRST(&tbp
->b_dep
) &&
873 * Check that the combined cluster
874 * would make sense with regard to pages
875 * and would not be too large
877 if ((tbp
->b_bcount
!= blksize
) ||
878 ((bp
->b_bio2
.bio_offset
+ i
) !=
879 tbp
->b_bio2
.bio_offset
) ||
880 ((tbp
->b_xio
.xio_npages
+ bp
->b_xio
.xio_npages
) >
881 (maxiosize
/ PAGE_SIZE
))) {
886 * Ok, it's passed all the tests,
887 * so remove it from the free list
888 * and mark it busy. We will use it.
891 KKASSERT(tbp
->b_cmd
== BUF_CMD_DONE
);
892 } /* end of code for non-first buffers only */
895 * If the IO is via the VM then we do some
896 * special VM hackery (yuck). Since the buffer's
897 * block size may not be page-aligned it is possible
898 * for a page to be shared between two buffers. We
899 * have to get rid of the duplication when building
902 if (tbp
->b_flags
& B_VMIO
) {
905 if (i
!= 0) { /* if not first buffer */
906 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
907 m
= tbp
->b_xio
.xio_pages
[j
];
908 if (m
->flags
& PG_BUSY
) {
915 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
916 m
= tbp
->b_xio
.xio_pages
[j
];
918 vm_object_pip_add(m
->object
, 1);
919 if ((bp
->b_xio
.xio_npages
== 0) ||
920 (bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
- 1] != m
)) {
921 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
922 bp
->b_xio
.xio_npages
++;
926 bp
->b_bcount
+= blksize
;
927 bp
->b_bufsize
+= blksize
;
930 tbp
->b_flags
&= ~B_ERROR
;
931 tbp
->b_cmd
= BUF_CMD_WRITE
;
933 cluster_append(&bp
->b_bio1
, tbp
);
936 * check for latent dependencies to be handled
938 if (LIST_FIRST(&tbp
->b_dep
) != NULL
)
942 pmap_qenter(trunc_page((vm_offset_t
) bp
->b_data
),
943 (vm_page_t
*) bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
944 if (bp
->b_bufsize
> bp
->b_kvasize
) {
946 "cluster_wbuild: b_bufsize(%d) > b_kvasize(%d)\n",
947 bp
->b_bufsize
, bp
->b_kvasize
);
949 totalwritten
+= bp
->b_bufsize
;
951 bp
->b_dirtyend
= bp
->b_bufsize
;
952 bp
->b_bio1
.bio_done
= cluster_callback
;
953 bp
->b_cmd
= BUF_CMD_WRITE
;
955 vfs_busy_pages(vp
, bp
);
956 bp
->b_runningbufspace
= bp
->b_bufsize
;
957 if (bp
->b_runningbufspace
) {
958 runningbufspace
+= bp
->b_runningbufspace
;
962 vn_strategy(vp
, &bp
->b_bio1
);
970 * Collect together all the buffers in a cluster.
971 * Plus add one additional buffer.
973 static struct cluster_save
*
974 cluster_collectbufs(struct vnode
*vp
, struct buf
*last_bp
, int blksize
)
976 struct cluster_save
*buflist
;
981 len
= (int)(vp
->v_lastw
- vp
->v_cstart
+ blksize
) / blksize
;
982 buflist
= kmalloc(sizeof(struct buf
*) * (len
+ 1) + sizeof(*buflist
),
983 M_SEGMENT
, M_WAITOK
);
984 buflist
->bs_nchildren
= 0;
985 buflist
->bs_children
= (struct buf
**) (buflist
+ 1);
986 for (loffset
= vp
->v_cstart
, i
= 0; i
< len
; (loffset
+= blksize
), i
++) {
987 (void) bread(vp
, loffset
, last_bp
->b_bcount
, &bp
);
988 buflist
->bs_children
[i
] = bp
;
989 if (bp
->b_bio2
.bio_offset
== NOOFFSET
) {
990 VOP_BMAP(bp
->b_vp
, bp
->b_loffset
,
991 &bp
->b_bio2
.bio_offset
,
992 NULL
, NULL
, BUF_CMD_WRITE
);
995 buflist
->bs_children
[i
] = bp
= last_bp
;
996 if (bp
->b_bio2
.bio_offset
== NOOFFSET
) {
997 VOP_BMAP(bp
->b_vp
, bp
->b_loffset
, &bp
->b_bio2
.bio_offset
,
998 NULL
, NULL
, BUF_CMD_WRITE
);
1000 buflist
->bs_nchildren
= i
+ 1;
1005 cluster_append(struct bio
*bio
, struct buf
*tbp
)
1007 tbp
->b_cluster_next
= NULL
;
1008 if (bio
->bio_caller_info1
.cluster_head
== NULL
) {
1009 bio
->bio_caller_info1
.cluster_head
= tbp
;
1010 bio
->bio_caller_info2
.cluster_tail
= tbp
;
1012 bio
->bio_caller_info2
.cluster_tail
->b_cluster_next
= tbp
;
1013 bio
->bio_caller_info2
.cluster_tail
= tbp
;