3 * The Regents of the University of California. All rights reserved.
4 * Modifications/enhancements:
5 * Copyright (c) 1995 John S. Dyson. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 4. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * @(#)vfs_cluster.c 8.7 (Berkeley) 2/13/94
32 * $FreeBSD: src/sys/kern/vfs_cluster.c,v 1.92.2.9 2001/11/18 07:10:59 dillon Exp $
33 * $DragonFly: src/sys/kern/vfs_cluster.c,v 1.40 2008/07/14 03:09:00 dillon Exp $
36 #include "opt_debug_cluster.h"
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/kernel.h>
43 #include <sys/vnode.h>
44 #include <sys/malloc.h>
45 #include <sys/mount.h>
46 #include <sys/resourcevar.h>
47 #include <sys/vmmeter.h>
49 #include <vm/vm_object.h>
50 #include <vm/vm_page.h>
51 #include <sys/sysctl.h>
54 #include <vm/vm_page2.h>
56 #include <machine/limits.h>
58 #if defined(CLUSTERDEBUG)
59 #include <sys/sysctl.h>
60 static int rcluster
= 0;
61 SYSCTL_INT(_debug
, OID_AUTO
, rcluster
, CTLFLAG_RW
, &rcluster
, 0, "");
64 static MALLOC_DEFINE(M_SEGMENT
, "cluster_save", "cluster_save buffer");
66 static struct cluster_save
*
67 cluster_collectbufs (struct vnode
*vp
, struct buf
*last_bp
,
70 cluster_rbuild (struct vnode
*vp
, off_t filesize
, off_t loffset
,
71 off_t doffset
, int blksize
, int run
,
73 static void cluster_callback (struct bio
*);
74 static void cluster_setram (struct buf
*);
75 static int cluster_wbuild(struct vnode
*vp
, struct buf
**bpp
, int blksize
,
76 off_t start_loffset
, int bytes
);
78 static int write_behind
= 1;
79 SYSCTL_INT(_vfs
, OID_AUTO
, write_behind
, CTLFLAG_RW
, &write_behind
, 0,
80 "Cluster write-behind setting");
81 static quad_t write_behind_minfilesize
= 10 * 1024 * 1024;
82 SYSCTL_QUAD(_vfs
, OID_AUTO
, write_behind_minfilesize
, CTLFLAG_RW
,
83 &write_behind_minfilesize
, 0, "Cluster write-behind setting");
84 static int max_readahead
= 2 * 1024 * 1024;
85 SYSCTL_INT(_vfs
, OID_AUTO
, max_readahead
, CTLFLAG_RW
, &max_readahead
, 0,
86 "Limit in bytes for desired cluster read-ahead");
88 extern vm_page_t bogus_page
;
90 extern int cluster_pbuf_freecnt
;
93 * This replaces bread.
95 * filesize - read-ahead @ blksize will not cross this boundary
96 * loffset - loffset for returned *bpp
97 * blksize - blocksize for returned *bpp and read-ahead bps
98 * minreq - minimum (not a hard minimum) in bytes, typically reflects
99 * a higher level uio resid.
100 * maxreq - maximum (sequential heuristic) in bytes (highet typ ~2MB)
101 * bpp - return buffer (*bpp) for (loffset,blksize)
104 cluster_readx(struct vnode
*vp
, off_t filesize
, off_t loffset
,
105 int blksize
, size_t minreq
, size_t maxreq
, struct buf
**bpp
)
107 struct buf
*bp
, *rbp
, *reqbp
;
118 * Calculate the desired read-ahead in blksize'd blocks (maxra).
119 * To do this we calculate maxreq.
121 * maxreq typically starts out as a sequential heuristic. If the
122 * high level uio/resid is bigger (minreq), we pop maxreq up to
123 * minreq. This represents the case where random I/O is being
124 * performed by the userland is issuing big read()'s.
126 * Then we limit maxreq to max_readahead to ensure it is a reasonable
129 * Finally we must ensure that (loffset + maxreq) does not cross the
130 * boundary (filesize) for the current blocksize. If we allowed it
131 * to cross we could end up with buffers past the boundary with the
132 * wrong block size (HAMMER large-data areas use mixed block sizes).
133 * minreq is also absolutely limited to filesize.
137 /* minreq not used beyond this point */
139 if (maxreq
> max_readahead
) {
140 maxreq
= max_readahead
;
141 if (maxreq
> 16 * 1024 * 1024)
142 maxreq
= 16 * 1024 * 1024;
144 if (maxreq
< blksize
)
146 if (loffset
+ maxreq
> filesize
) {
147 if (loffset
> filesize
)
150 maxreq
= filesize
- loffset
;
153 maxra
= (int)(maxreq
/ blksize
);
156 * Get the requested block.
161 *bpp
= reqbp
= bp
= getblk(vp
, loffset
, blksize
, 0, 0);
162 origoffset
= loffset
;
165 * Calculate the maximum cluster size for a single I/O, used
166 * by cluster_rbuild().
168 maxrbuild
= vmaxiosize(vp
) / blksize
;
171 * if it is in the cache, then check to see if the reads have been
172 * sequential. If they have, then try some read-ahead, otherwise
173 * back-off on prospective read-aheads.
175 if (bp
->b_flags
& B_CACHE
) {
177 * Not sequential, do not do any read-ahead
183 * No read-ahead mark, do not do any read-ahead
186 if ((bp
->b_flags
& B_RAM
) == 0)
190 * We hit a read-ahead-mark, figure out how much read-ahead
191 * to do (maxra) and where to start (loffset).
193 * Shortcut the scan. Typically the way this works is that
194 * we've built up all the blocks inbetween except for the
195 * last in previous iterations, so if the second-to-last
196 * block is present we just skip ahead to it.
198 * This algorithm has O(1) cpu in the steady state no
199 * matter how large maxra is.
201 bp
->b_flags
&= ~B_RAM
;
203 if (findblk(vp
, loffset
+ (maxra
- 2) * blksize
, FINDBLK_TEST
))
208 if (findblk(vp
, loffset
+ i
* blksize
,
209 FINDBLK_TEST
) == NULL
) {
216 * We got everything or everything is in the cache, no
223 * Calculate where to start the read-ahead and how much
224 * to do. Generally speaking we want to read-ahead by
225 * (maxra) when we've found a read-ahead mark. We do
226 * not want to reduce maxra here as it will cause
227 * successive read-ahead I/O's to be smaller and smaller.
229 * However, we have to make sure we don't break the
230 * filesize limitation for the clustered operation.
232 loffset
+= i
* blksize
;
235 if (loffset
>= filesize
)
237 if (loffset
+ maxra
* blksize
> filesize
) {
238 maxreq
= filesize
- loffset
;
239 maxra
= (int)(maxreq
/ blksize
);
242 __debugvar off_t firstread
= bp
->b_loffset
;
246 * Set-up synchronous read for bp.
248 bp
->b_cmd
= BUF_CMD_READ
;
249 bp
->b_bio1
.bio_done
= biodone_sync
;
250 bp
->b_bio1
.bio_flags
|= BIO_SYNC
;
252 KASSERT(firstread
!= NOOFFSET
,
253 ("cluster_read: no buffer offset"));
256 * nblks is our cluster_rbuild request size, limited
257 * primarily by the device.
259 if ((nblks
= maxra
) > maxrbuild
)
265 error
= VOP_BMAP(vp
, loffset
, &doffset
,
266 &burstbytes
, NULL
, BUF_CMD_READ
);
268 goto single_block_read
;
269 if (nblks
> burstbytes
/ blksize
)
270 nblks
= burstbytes
/ blksize
;
271 if (doffset
== NOOFFSET
)
272 goto single_block_read
;
274 goto single_block_read
;
276 bp
= cluster_rbuild(vp
, filesize
, loffset
,
277 doffset
, blksize
, nblks
, bp
);
278 loffset
+= bp
->b_bufsize
;
279 maxra
-= bp
->b_bufsize
/ blksize
;
283 * If it isn't in the cache, then get a chunk from
284 * disk if sequential, otherwise just get the block.
293 * If B_CACHE was not set issue bp. bp will either be an
294 * asynchronous cluster buf or a synchronous single-buf.
295 * If it is a single buf it will be the same as reqbp.
297 * NOTE: Once an async cluster buf is issued bp becomes invalid.
300 #if defined(CLUSTERDEBUG)
302 kprintf("S(%012jx,%d,%d)\n",
303 (intmax_t)bp
->b_loffset
, bp
->b_bcount
, maxra
);
305 if ((bp
->b_flags
& B_CLUSTER
) == 0)
306 vfs_busy_pages(vp
, bp
);
307 bp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
308 vn_strategy(vp
, &bp
->b_bio1
);
314 * If we have been doing sequential I/O, then do some read-ahead.
315 * The code above us should have positioned us at the next likely
318 * Only mess with buffers which we can immediately lock. HAMMER
319 * will do device-readahead irrespective of what the blocks
322 while (error
== 0 && maxra
> 0) {
327 rbp
= getblk(vp
, loffset
, blksize
,
328 GETBLK_SZMATCH
|GETBLK_NOWAIT
, 0);
331 if ((rbp
->b_flags
& B_CACHE
)) {
337 * An error from the read-ahead bmap has nothing to do
338 * with the caller's original request.
340 tmp_error
= VOP_BMAP(vp
, loffset
, &doffset
,
341 &burstbytes
, NULL
, BUF_CMD_READ
);
342 if (tmp_error
|| doffset
== NOOFFSET
) {
343 rbp
->b_flags
|= B_INVAL
;
348 if ((nblks
= maxra
) > maxrbuild
)
350 if (nblks
> burstbytes
/ blksize
)
351 nblks
= burstbytes
/ blksize
;
356 rbp
->b_cmd
= BUF_CMD_READ
;
357 /*rbp->b_flags |= B_AGE*/;
361 rbp
= cluster_rbuild(vp
, filesize
, loffset
,
365 rbp
->b_bio2
.bio_offset
= doffset
;
368 #if defined(CLUSTERDEBUG)
371 kprintf("A+(%012jx,%d,%jd) "
372 "doff=%012jx minr=%zd ra=%d\n",
373 (intmax_t)loffset
, rbp
->b_bcount
,
374 (intmax_t)(loffset
- origoffset
),
375 (intmax_t)doffset
, minreq
, maxra
);
377 kprintf("A-(%012jx,%d,%jd) "
378 "doff=%012jx minr=%zd ra=%d\n",
379 (intmax_t)rbp
->b_loffset
, rbp
->b_bcount
,
380 (intmax_t)(loffset
- origoffset
),
381 (intmax_t)doffset
, minreq
, maxra
);
385 rbp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
387 if ((rbp
->b_flags
& B_CLUSTER
) == 0)
388 vfs_busy_pages(vp
, rbp
);
390 loffset
+= rbp
->b_bufsize
;
391 maxra
-= rbp
->b_bufsize
/ blksize
;
392 vn_strategy(vp
, &rbp
->b_bio1
);
393 /* rbp invalid now */
397 * Wait for our original buffer to complete its I/O. reqbp will
398 * be NULL if the original buffer was B_CACHE. We are returning
399 * (*bpp) which is the same as reqbp when reqbp != NULL.
403 KKASSERT(reqbp
->b_bio1
.bio_flags
& BIO_SYNC
);
404 error
= biowait(&reqbp
->b_bio1
, "clurd");
410 * If blocks are contiguous on disk, use this to provide clustered
411 * read ahead. We will read as many blocks as possible sequentially
412 * and then parcel them up into logical blocks in the buffer hash table.
414 * This function either returns a cluster buf or it returns fbp. fbp is
415 * already expected to be set up as a synchronous or asynchronous request.
417 * If a cluster buf is returned it will always be async.
420 cluster_rbuild(struct vnode
*vp
, off_t filesize
, off_t loffset
, off_t doffset
,
421 int blksize
, int run
, struct buf
*fbp
)
423 struct buf
*bp
, *tbp
;
426 int maxiosize
= vmaxiosize(vp
);
431 while (loffset
+ run
* blksize
> filesize
) {
436 tbp
->b_bio2
.bio_offset
= doffset
;
437 if((tbp
->b_flags
& B_MALLOC
) ||
438 ((tbp
->b_flags
& B_VMIO
) == 0) || (run
<= 1)) {
442 bp
= trypbuf_kva(&cluster_pbuf_freecnt
);
448 * We are synthesizing a buffer out of vm_page_t's, but
449 * if the block size is not page aligned then the starting
450 * address may not be either. Inherit the b_data offset
451 * from the original buffer.
453 bp
->b_data
= (char *)((vm_offset_t
)bp
->b_data
|
454 ((vm_offset_t
)tbp
->b_data
& PAGE_MASK
));
455 bp
->b_flags
|= B_CLUSTER
| B_VMIO
;
456 bp
->b_cmd
= BUF_CMD_READ
;
457 bp
->b_bio1
.bio_done
= cluster_callback
; /* default to async */
458 bp
->b_bio1
.bio_caller_info1
.cluster_head
= NULL
;
459 bp
->b_bio1
.bio_caller_info2
.cluster_tail
= NULL
;
460 bp
->b_loffset
= loffset
;
461 bp
->b_bio2
.bio_offset
= doffset
;
462 KASSERT(bp
->b_loffset
!= NOOFFSET
,
463 ("cluster_rbuild: no buffer offset"));
467 bp
->b_xio
.xio_npages
= 0;
469 for (boffset
= doffset
, i
= 0; i
< run
; ++i
, boffset
+= blksize
) {
471 if ((bp
->b_xio
.xio_npages
* PAGE_SIZE
) +
472 round_page(blksize
) > maxiosize
) {
477 * Shortcut some checks and try to avoid buffers that
478 * would block in the lock. The same checks have to
479 * be made again after we officially get the buffer.
481 tbp
= getblk(vp
, loffset
+ i
* blksize
, blksize
,
482 GETBLK_SZMATCH
|GETBLK_NOWAIT
, 0);
485 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; j
++) {
486 if (tbp
->b_xio
.xio_pages
[j
]->valid
)
489 if (j
!= tbp
->b_xio
.xio_npages
) {
495 * Stop scanning if the buffer is fuly valid
496 * (marked B_CACHE), or locked (may be doing a
497 * background write), or if the buffer is not
498 * VMIO backed. The clustering code can only deal
499 * with VMIO-backed buffers.
501 if ((tbp
->b_flags
& (B_CACHE
|B_LOCKED
)) ||
502 (tbp
->b_flags
& B_VMIO
) == 0 ||
503 (LIST_FIRST(&tbp
->b_dep
) != NULL
&&
511 * The buffer must be completely invalid in order to
512 * take part in the cluster. If it is partially valid
515 for (j
= 0;j
< tbp
->b_xio
.xio_npages
; j
++) {
516 if (tbp
->b_xio
.xio_pages
[j
]->valid
)
519 if (j
!= tbp
->b_xio
.xio_npages
) {
525 * Set a read-ahead mark as appropriate. Always
526 * set the read-ahead mark at (run - 1). It is
527 * unclear why we were also setting it at i == 1.
529 if (/*i == 1 ||*/ i
== (run
- 1))
533 * Depress the priority of buffers not explicitly
536 /* tbp->b_flags |= B_AGE; */
539 * Set the block number if it isn't set, otherwise
540 * if it is make sure it matches the block number we
543 if (tbp
->b_bio2
.bio_offset
== NOOFFSET
) {
544 tbp
->b_bio2
.bio_offset
= boffset
;
545 } else if (tbp
->b_bio2
.bio_offset
!= boffset
) {
552 * The passed-in tbp (i == 0) will already be set up for
553 * async or sync operation. All other tbp's acquire in
554 * our loop are set up for async operation.
556 tbp
->b_cmd
= BUF_CMD_READ
;
558 cluster_append(&bp
->b_bio1
, tbp
);
559 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
562 m
= tbp
->b_xio
.xio_pages
[j
];
563 vm_page_busy_wait(m
, FALSE
, "clurpg");
566 vm_object_pip_add(m
->object
, 1);
567 if ((bp
->b_xio
.xio_npages
== 0) ||
568 (bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
-1] != m
)) {
569 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
570 bp
->b_xio
.xio_npages
++;
572 if ((m
->valid
& VM_PAGE_BITS_ALL
) == VM_PAGE_BITS_ALL
)
573 tbp
->b_xio
.xio_pages
[j
] = bogus_page
;
576 * XXX shouldn't this be += size for both, like in
579 * Don't inherit tbp->b_bufsize as it may be larger due to
580 * a non-page-aligned size. Instead just aggregate using
583 if (tbp
->b_bcount
!= blksize
)
584 kprintf("warning: tbp->b_bcount wrong %d vs %d\n", tbp
->b_bcount
, blksize
);
585 if (tbp
->b_bufsize
!= blksize
)
586 kprintf("warning: tbp->b_bufsize wrong %d vs %d\n", tbp
->b_bufsize
, blksize
);
587 bp
->b_bcount
+= blksize
;
588 bp
->b_bufsize
+= blksize
;
592 * Fully valid pages in the cluster are already good and do not need
593 * to be re-read from disk. Replace the page with bogus_page
595 for (j
= 0; j
< bp
->b_xio
.xio_npages
; j
++) {
596 if ((bp
->b_xio
.xio_pages
[j
]->valid
& VM_PAGE_BITS_ALL
) ==
598 bp
->b_xio
.xio_pages
[j
] = bogus_page
;
601 if (bp
->b_bufsize
> bp
->b_kvasize
) {
602 panic("cluster_rbuild: b_bufsize(%d) > b_kvasize(%d)",
603 bp
->b_bufsize
, bp
->b_kvasize
);
605 pmap_qenter(trunc_page((vm_offset_t
) bp
->b_data
),
606 (vm_page_t
*)bp
->b_xio
.xio_pages
, bp
->b_xio
.xio_npages
);
612 * Cleanup after a clustered read or write.
613 * This is complicated by the fact that any of the buffers might have
614 * extra memory (if there were no empty buffer headers at allocbuf time)
615 * that we will need to shift around.
617 * The returned bio is &bp->b_bio1
620 cluster_callback(struct bio
*bio
)
622 struct buf
*bp
= bio
->bio_buf
;
627 * Must propogate errors to all the components. A short read (EOF)
628 * is a critical error.
630 if (bp
->b_flags
& B_ERROR
) {
632 } else if (bp
->b_bcount
!= bp
->b_bufsize
) {
633 panic("cluster_callback: unexpected EOF on cluster %p!", bio
);
636 pmap_qremove(trunc_page((vm_offset_t
) bp
->b_data
), bp
->b_xio
.xio_npages
);
638 * Move memory from the large cluster buffer into the component
639 * buffers and mark IO as done on these. Since the memory map
640 * is the same, no actual copying is required.
642 while ((tbp
= bio
->bio_caller_info1
.cluster_head
) != NULL
) {
643 bio
->bio_caller_info1
.cluster_head
= tbp
->b_cluster_next
;
645 tbp
->b_flags
|= B_ERROR
| B_IODEBUG
;
646 tbp
->b_error
= error
;
648 tbp
->b_dirtyoff
= tbp
->b_dirtyend
= 0;
649 tbp
->b_flags
&= ~(B_ERROR
|B_INVAL
);
650 tbp
->b_flags
|= B_IODEBUG
;
652 * XXX the bdwrite()/bqrelse() issued during
653 * cluster building clears B_RELBUF (see bqrelse()
654 * comment). If direct I/O was specified, we have
655 * to restore it here to allow the buffer and VM
658 if (tbp
->b_flags
& B_DIRECT
)
659 tbp
->b_flags
|= B_RELBUF
;
661 biodone(&tbp
->b_bio1
);
663 relpbuf(bp
, &cluster_pbuf_freecnt
);
667 * Implement modified write build for cluster.
669 * write_behind = 0 write behind disabled
670 * write_behind = 1 write behind normal (default)
671 * write_behind = 2 write behind backed-off
673 * In addition, write_behind is only activated for files that have
674 * grown past a certain size (default 10MB). Otherwise temporary files
675 * wind up generating a lot of unnecessary disk I/O.
678 cluster_wbuild_wb(struct vnode
*vp
, int blksize
, off_t start_loffset
, int len
)
682 switch(write_behind
) {
684 if (start_loffset
< len
)
686 start_loffset
-= len
;
689 if (vp
->v_filesize
>= write_behind_minfilesize
) {
690 r
= cluster_wbuild(vp
, NULL
, blksize
,
702 * Do clustered write for FFS.
705 * 1. Write is not sequential (write asynchronously)
706 * Write is sequential:
707 * 2. beginning of cluster - begin cluster
708 * 3. middle of a cluster - add to cluster
709 * 4. end of a cluster - asynchronously write cluster
712 cluster_write(struct buf
*bp
, off_t filesize
, int blksize
, int seqcount
)
716 int maxclen
, cursize
;
720 if (vp
->v_type
== VREG
)
721 async
= vp
->v_mount
->mnt_flag
& MNT_ASYNC
;
724 loffset
= bp
->b_loffset
;
725 KASSERT(bp
->b_loffset
!= NOOFFSET
,
726 ("cluster_write: no buffer offset"));
728 /* Initialize vnode to beginning of file. */
730 vp
->v_lasta
= vp
->v_clen
= vp
->v_cstart
= vp
->v_lastw
= 0;
732 if (vp
->v_clen
== 0 || loffset
!= vp
->v_lastw
+ blksize
||
733 bp
->b_bio2
.bio_offset
== NOOFFSET
||
734 (bp
->b_bio2
.bio_offset
!= vp
->v_lasta
+ blksize
)) {
735 maxclen
= vmaxiosize(vp
);
736 if (vp
->v_clen
!= 0) {
738 * Next block is not sequential.
740 * If we are not writing at end of file, the process
741 * seeked to another point in the file since its last
742 * write, or we have reached our maximum cluster size,
743 * then push the previous cluster. Otherwise try
744 * reallocating to make it sequential.
746 * Change to algorithm: only push previous cluster if
747 * it was sequential from the point of view of the
748 * seqcount heuristic, otherwise leave the buffer
749 * intact so we can potentially optimize the I/O
750 * later on in the buf_daemon or update daemon
753 cursize
= vp
->v_lastw
- vp
->v_cstart
+ blksize
;
754 if (bp
->b_loffset
+ blksize
< filesize
||
755 loffset
!= vp
->v_lastw
+ blksize
|| vp
->v_clen
<= cursize
) {
756 if (!async
&& seqcount
> 0) {
757 cluster_wbuild_wb(vp
, blksize
,
758 vp
->v_cstart
, cursize
);
761 struct buf
**bpp
, **endbp
;
762 struct cluster_save
*buflist
;
764 buflist
= cluster_collectbufs(vp
, bp
, blksize
);
765 endbp
= &buflist
->bs_children
766 [buflist
->bs_nchildren
- 1];
767 if (VOP_REALLOCBLKS(vp
, buflist
)) {
769 * Failed, push the previous cluster
770 * if *really* writing sequentially
771 * in the logical file (seqcount > 1),
772 * otherwise delay it in the hopes that
773 * the low level disk driver can
774 * optimize the write ordering.
776 for (bpp
= buflist
->bs_children
;
779 kfree(buflist
, M_SEGMENT
);
781 cluster_wbuild_wb(vp
,
782 blksize
, vp
->v_cstart
,
787 * Succeeded, keep building cluster.
789 for (bpp
= buflist
->bs_children
;
792 kfree(buflist
, M_SEGMENT
);
793 vp
->v_lastw
= loffset
;
794 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
800 * Consider beginning a cluster. If at end of file, make
801 * cluster as large as possible, otherwise find size of
804 if ((vp
->v_type
== VREG
) &&
805 bp
->b_loffset
+ blksize
< filesize
&&
806 (bp
->b_bio2
.bio_offset
== NOOFFSET
) &&
807 (VOP_BMAP(vp
, loffset
, &bp
->b_bio2
.bio_offset
, &maxclen
, NULL
, BUF_CMD_WRITE
) ||
808 bp
->b_bio2
.bio_offset
== NOOFFSET
)) {
811 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
812 vp
->v_cstart
= loffset
+ blksize
;
813 vp
->v_lastw
= loffset
;
816 if (maxclen
> blksize
)
817 vp
->v_clen
= maxclen
- blksize
;
820 if (!async
&& vp
->v_clen
== 0) { /* I/O not contiguous */
821 vp
->v_cstart
= loffset
+ blksize
;
823 } else { /* Wait for rest of cluster */
824 vp
->v_cstart
= loffset
;
827 } else if (loffset
== vp
->v_cstart
+ vp
->v_clen
) {
829 * At end of cluster, write it out if seqcount tells us we
830 * are operating sequentially, otherwise let the buf or
831 * update daemon handle it.
835 cluster_wbuild_wb(vp
, blksize
, vp
->v_cstart
,
836 vp
->v_clen
+ blksize
);
838 vp
->v_cstart
= loffset
+ blksize
;
839 } else if (vm_page_count_severe() &&
840 bp
->b_loffset
+ blksize
< filesize
) {
842 * We are low on memory, get it going NOW. However, do not
843 * try to push out a partial block at the end of the file
844 * as this could lead to extremely non-optimal write activity.
849 * In the middle of a cluster, so just delay the I/O for now.
853 vp
->v_lastw
= loffset
;
854 vp
->v_lasta
= bp
->b_bio2
.bio_offset
;
858 * This is the clustered version of bawrite(). It works similarly to
859 * cluster_write() except I/O on the buffer is guaranteed to occur.
862 cluster_awrite(struct buf
*bp
)
867 * Don't bother if it isn't clusterable.
869 if ((bp
->b_flags
& B_CLUSTEROK
) == 0 ||
871 (bp
->b_vp
->v_flag
& VOBJBUF
) == 0) {
872 total
= bp
->b_bufsize
;
877 total
= cluster_wbuild(bp
->b_vp
, &bp
, bp
->b_bufsize
,
878 bp
->b_loffset
, vmaxiosize(bp
->b_vp
));
886 * This is an awful lot like cluster_rbuild...wish they could be combined.
887 * The last lbn argument is the current block on which I/O is being
888 * performed. Check to see that it doesn't fall in the middle of
889 * the current block (if last_bp == NULL).
891 * cluster_wbuild() normally does not guarantee anything. If bpp is
892 * non-NULL and cluster_wbuild() is able to incorporate it into the
893 * I/O it will set *bpp to NULL, otherwise it will leave it alone and
894 * the caller must dispose of *bpp.
897 cluster_wbuild(struct vnode
*vp
, struct buf
**bpp
,
898 int blksize
, off_t start_loffset
, int bytes
)
900 struct buf
*bp
, *tbp
;
902 int totalwritten
= 0;
904 int maxiosize
= vmaxiosize(vp
);
908 * If the buffer matches the passed locked & removed buffer
909 * we used the passed buffer (which might not be B_DELWRI).
911 * Otherwise locate the buffer and determine if it is
914 if (bpp
&& (*bpp
)->b_loffset
== start_loffset
) {
919 tbp
= findblk(vp
, start_loffset
, FINDBLK_NBLOCK
);
921 (tbp
->b_flags
& (B_LOCKED
| B_INVAL
| B_DELWRI
)) !=
923 (LIST_FIRST(&tbp
->b_dep
) && buf_checkwrite(tbp
))) {
926 start_loffset
+= blksize
;
932 KKASSERT(tbp
->b_cmd
== BUF_CMD_DONE
);
935 * Extra memory in the buffer, punt on this buffer.
936 * XXX we could handle this in most cases, but we would
937 * have to push the extra memory down to after our max
938 * possible cluster size and then potentially pull it back
939 * up if the cluster was terminated prematurely--too much
942 if (((tbp
->b_flags
& (B_CLUSTEROK
|B_MALLOC
)) != B_CLUSTEROK
) ||
943 (tbp
->b_bcount
!= tbp
->b_bufsize
) ||
944 (tbp
->b_bcount
!= blksize
) ||
945 (bytes
== blksize
) ||
946 ((bp
= getpbuf_kva(&cluster_pbuf_freecnt
)) == NULL
)) {
947 totalwritten
+= tbp
->b_bufsize
;
949 start_loffset
+= blksize
;
955 * Set up the pbuf. Track our append point with b_bcount
956 * and b_bufsize. b_bufsize is not used by the device but
957 * our caller uses it to loop clusters and we use it to
958 * detect a premature EOF on the block device.
962 bp
->b_xio
.xio_npages
= 0;
963 bp
->b_loffset
= tbp
->b_loffset
;
964 bp
->b_bio2
.bio_offset
= tbp
->b_bio2
.bio_offset
;
967 * We are synthesizing a buffer out of vm_page_t's, but
968 * if the block size is not page aligned then the starting
969 * address may not be either. Inherit the b_data offset
970 * from the original buffer.
972 bp
->b_data
= (char *)((vm_offset_t
)bp
->b_data
|
973 ((vm_offset_t
)tbp
->b_data
& PAGE_MASK
));
974 bp
->b_flags
&= ~B_ERROR
;
975 bp
->b_flags
|= B_CLUSTER
| B_BNOCLIP
|
976 (tbp
->b_flags
& (B_VMIO
| B_NEEDCOMMIT
));
977 bp
->b_bio1
.bio_caller_info1
.cluster_head
= NULL
;
978 bp
->b_bio1
.bio_caller_info2
.cluster_tail
= NULL
;
981 * From this location in the file, scan forward to see
982 * if there are buffers with adjacent data that need to
983 * be written as well.
985 * IO *must* be initiated on index 0 at this point
986 * (particularly when called from cluster_awrite()).
988 for (i
= 0; i
< bytes
; (i
+= blksize
), (start_loffset
+= blksize
)) {
996 tbp
= findblk(vp
, start_loffset
,
999 * Buffer not found or could not be locked
1006 * If it IS in core, but has different
1007 * characteristics, then don't cluster
1010 if ((tbp
->b_flags
& (B_VMIO
| B_CLUSTEROK
|
1011 B_INVAL
| B_DELWRI
| B_NEEDCOMMIT
))
1012 != (B_DELWRI
| B_CLUSTEROK
|
1013 (bp
->b_flags
& (B_VMIO
| B_NEEDCOMMIT
))) ||
1014 (tbp
->b_flags
& B_LOCKED
)
1021 * Check that the combined cluster
1022 * would make sense with regard to pages
1023 * and would not be too large
1025 * WARNING! buf_checkwrite() must be the last
1026 * check made. If it returns 0 then
1027 * we must initiate the I/O.
1029 if ((tbp
->b_bcount
!= blksize
) ||
1030 ((bp
->b_bio2
.bio_offset
+ i
) !=
1031 tbp
->b_bio2
.bio_offset
) ||
1032 ((tbp
->b_xio
.xio_npages
+ bp
->b_xio
.xio_npages
) >
1033 (maxiosize
/ PAGE_SIZE
)) ||
1034 (LIST_FIRST(&tbp
->b_dep
) &&
1035 buf_checkwrite(tbp
))
1040 if (LIST_FIRST(&tbp
->b_dep
))
1043 * Ok, it's passed all the tests,
1044 * so remove it from the free list
1045 * and mark it busy. We will use it.
1048 KKASSERT(tbp
->b_cmd
== BUF_CMD_DONE
);
1052 * If the IO is via the VM then we do some
1053 * special VM hackery (yuck). Since the buffer's
1054 * block size may not be page-aligned it is possible
1055 * for a page to be shared between two buffers. We
1056 * have to get rid of the duplication when building
1059 if (tbp
->b_flags
& B_VMIO
) {
1063 * Try to avoid deadlocks with the VM system.
1064 * However, we cannot abort the I/O if
1065 * must_initiate is non-zero.
1067 if (must_initiate
== 0) {
1069 j
< tbp
->b_xio
.xio_npages
;
1071 m
= tbp
->b_xio
.xio_pages
[j
];
1072 if (m
->flags
& PG_BUSY
) {
1079 for (j
= 0; j
< tbp
->b_xio
.xio_npages
; ++j
) {
1080 m
= tbp
->b_xio
.xio_pages
[j
];
1081 vm_page_busy_wait(m
, FALSE
, "clurpg");
1082 vm_page_io_start(m
);
1084 vm_object_pip_add(m
->object
, 1);
1085 if ((bp
->b_xio
.xio_npages
== 0) ||
1086 (bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
- 1] != m
)) {
1087 bp
->b_xio
.xio_pages
[bp
->b_xio
.xio_npages
] = m
;
1088 bp
->b_xio
.xio_npages
++;
1092 bp
->b_bcount
+= blksize
;
1093 bp
->b_bufsize
+= blksize
;
1096 tbp
->b_flags
&= ~B_ERROR
;
1097 tbp
->b_cmd
= BUF_CMD_WRITE
;
1099 cluster_append(&bp
->b_bio1
, tbp
);
1102 * check for latent dependencies to be handled
1104 if (LIST_FIRST(&tbp
->b_dep
) != NULL
)
1108 pmap_qenter(trunc_page((vm_offset_t
)bp
->b_data
),
1109 (vm_page_t
*)bp
->b_xio
.xio_pages
,
1110 bp
->b_xio
.xio_npages
);
1111 if (bp
->b_bufsize
> bp
->b_kvasize
) {
1112 panic("cluster_wbuild: b_bufsize(%d) "
1113 "> b_kvasize(%d)\n",
1114 bp
->b_bufsize
, bp
->b_kvasize
);
1116 totalwritten
+= bp
->b_bufsize
;
1118 bp
->b_dirtyend
= bp
->b_bufsize
;
1119 bp
->b_bio1
.bio_done
= cluster_callback
;
1120 bp
->b_cmd
= BUF_CMD_WRITE
;
1122 vfs_busy_pages(vp
, bp
);
1123 bsetrunningbufspace(bp
, bp
->b_bufsize
);
1125 vn_strategy(vp
, &bp
->b_bio1
);
1129 return totalwritten
;
1133 * Collect together all the buffers in a cluster, plus add one
1134 * additional buffer passed-in.
1136 * Only pre-existing buffers whos block size matches blksize are collected.
1137 * (this is primarily because HAMMER1 uses varying block sizes and we don't
1138 * want to override its choices).
1140 static struct cluster_save
*
1141 cluster_collectbufs(struct vnode
*vp
, struct buf
*last_bp
, int blksize
)
1143 struct cluster_save
*buflist
;
1150 len
= (int)(vp
->v_lastw
- vp
->v_cstart
+ blksize
) / blksize
;
1151 buflist
= kmalloc(sizeof(struct buf
*) * (len
+ 1) + sizeof(*buflist
),
1152 M_SEGMENT
, M_WAITOK
);
1153 buflist
->bs_nchildren
= 0;
1154 buflist
->bs_children
= (struct buf
**) (buflist
+ 1);
1155 for (loffset
= vp
->v_cstart
, i
= 0, j
= 0;
1157 (loffset
+= blksize
), i
++) {
1158 bp
= getcacheblk(vp
, loffset
,
1159 last_bp
->b_bcount
, GETBLK_SZMATCH
);
1160 buflist
->bs_children
[i
] = bp
;
1163 } else if (bp
->b_bio2
.bio_offset
== NOOFFSET
) {
1164 VOP_BMAP(bp
->b_vp
, bp
->b_loffset
,
1165 &bp
->b_bio2
.bio_offset
,
1166 NULL
, NULL
, BUF_CMD_WRITE
);
1173 for (k
= 0; k
< j
; ++k
) {
1174 if (buflist
->bs_children
[k
]) {
1175 bqrelse(buflist
->bs_children
[k
]);
1176 buflist
->bs_children
[k
] = NULL
;
1181 bcopy(buflist
->bs_children
+ j
,
1182 buflist
->bs_children
+ 0,
1183 sizeof(buflist
->bs_children
[0]) * (i
- j
));
1187 buflist
->bs_children
[i
] = bp
= last_bp
;
1188 if (bp
->b_bio2
.bio_offset
== NOOFFSET
) {
1189 VOP_BMAP(bp
->b_vp
, bp
->b_loffset
, &bp
->b_bio2
.bio_offset
,
1190 NULL
, NULL
, BUF_CMD_WRITE
);
1192 buflist
->bs_nchildren
= i
+ 1;
1197 cluster_append(struct bio
*bio
, struct buf
*tbp
)
1199 tbp
->b_cluster_next
= NULL
;
1200 if (bio
->bio_caller_info1
.cluster_head
== NULL
) {
1201 bio
->bio_caller_info1
.cluster_head
= tbp
;
1202 bio
->bio_caller_info2
.cluster_tail
= tbp
;
1204 bio
->bio_caller_info2
.cluster_tail
->b_cluster_next
= tbp
;
1205 bio
->bio_caller_info2
.cluster_tail
= tbp
;
1211 cluster_setram (struct buf
*bp
)
1213 bp
->b_flags
|= B_RAM
;
1214 if (bp
->b_xio
.xio_npages
)
1215 vm_page_flag_set(bp
->b_xio
.xio_pages
[0], PG_RAM
);