1 // SPDX-License-Identifier: GPL-2.0+
3 * Copyright (C) 2016 Oracle. All Rights Reserved.
4 * Author: Darrick J. Wong <darrick.wong@oracle.com>
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_defer.h"
14 #include "xfs_inode.h"
15 #include "xfs_trans.h"
17 #include "xfs_bmap_util.h"
18 #include "xfs_trace.h"
19 #include "xfs_icache.h"
20 #include "xfs_btree.h"
21 #include "xfs_refcount_btree.h"
22 #include "xfs_refcount.h"
23 #include "xfs_bmap_btree.h"
24 #include "xfs_trans_space.h"
26 #include "xfs_alloc.h"
27 #include "xfs_quota.h"
28 #include "xfs_reflink.h"
29 #include "xfs_iomap.h"
31 #include "xfs_ag_resv.h"
32 #include "xfs_health.h"
35 * Copy on Write of Shared Blocks
37 * XFS must preserve "the usual" file semantics even when two files share
38 * the same physical blocks. This means that a write to one file must not
39 * alter the blocks in a different file; the way that we'll do that is
40 * through the use of a copy-on-write mechanism. At a high level, that
41 * means that when we want to write to a shared block, we allocate a new
42 * block, write the data to the new block, and if that succeeds we map the
43 * new block into the file.
45 * XFS provides a "delayed allocation" mechanism that defers the allocation
46 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as
47 * possible. This reduces fragmentation by enabling the filesystem to ask
48 * for bigger chunks less often, which is exactly what we want for CoW.
50 * The delalloc mechanism begins when the kernel wants to make a block
51 * writable (write_begin or page_mkwrite). If the offset is not mapped, we
52 * create a delalloc mapping, which is a regular in-core extent, but without
53 * a real startblock. (For delalloc mappings, the startblock encodes both
54 * a flag that this is a delalloc mapping, and a worst-case estimate of how
55 * many blocks might be required to put the mapping into the BMBT.) delalloc
56 * mappings are a reservation against the free space in the filesystem;
57 * adjacent mappings can also be combined into fewer larger mappings.
59 * As an optimization, the CoW extent size hint (cowextsz) creates
60 * outsized aligned delalloc reservations in the hope of landing out of
61 * order nearby CoW writes in a single extent on disk, thereby reducing
62 * fragmentation and improving future performance.
64 * D: --RRRRRRSSSRRRRRRRR--- (data fork)
65 * C: ------DDDDDDD--------- (CoW fork)
67 * When dirty pages are being written out (typically in writepage), the
68 * delalloc reservations are converted into unwritten mappings by
69 * allocating blocks and replacing the delalloc mapping with real ones.
70 * A delalloc mapping can be replaced by several unwritten ones if the
71 * free space is fragmented.
73 * D: --RRRRRRSSSRRRRRRRR---
74 * C: ------UUUUUUU---------
76 * We want to adapt the delalloc mechanism for copy-on-write, since the
77 * write paths are similar. The first two steps (creating the reservation
78 * and allocating the blocks) are exactly the same as delalloc except that
79 * the mappings must be stored in a separate CoW fork because we do not want
80 * to disturb the mapping in the data fork until we're sure that the write
81 * succeeded. IO completion in this case is the process of removing the old
82 * mapping from the data fork and moving the new mapping from the CoW fork to
83 * the data fork. This will be discussed shortly.
85 * For now, unaligned directio writes will be bounced back to the page cache.
86 * Block-aligned directio writes will use the same mechanism as buffered
89 * Just prior to submitting the actual disk write requests, we convert
90 * the extents representing the range of the file actually being written
91 * (as opposed to extra pieces created for the cowextsize hint) to real
92 * extents. This will become important in the next step:
94 * D: --RRRRRRSSSRRRRRRRR---
95 * C: ------UUrrUUU---------
97 * CoW remapping must be done after the data block write completes,
98 * because we don't want to destroy the old data fork map until we're sure
99 * the new block has been written. Since the new mappings are kept in a
100 * separate fork, we can simply iterate these mappings to find the ones
101 * that cover the file blocks that we just CoW'd. For each extent, simply
102 * unmap the corresponding range in the data fork, map the new range into
103 * the data fork, and remove the extent from the CoW fork. Because of
104 * the presence of the cowextsize hint, however, we must be careful
105 * only to remap the blocks that we've actually written out -- we must
106 * never remap delalloc reservations nor CoW staging blocks that have
107 * yet to be written. This corresponds exactly to the real extents in
110 * D: --RRRRRRrrSRRRRRRRR---
111 * C: ------UU--UUU---------
113 * Since the remapping operation can be applied to an arbitrary file
114 * range, we record the need for the remap step as a flag in the ioend
115 * instead of declaring a new IO type. This is required for direct io
116 * because we only have ioend for the whole dio, and we have to be able to
117 * remember the presence of unwritten blocks and CoW blocks with a single
118 * ioend structure. Better yet, the more ground we can cover with one
123 * Given an AG extent, find the lowest-numbered run of shared blocks
124 * within that range and return the range in fbno/flen. If
125 * find_end_of_shared is true, return the longest contiguous extent of
126 * shared blocks. If there are no shared extents, fbno and flen will
127 * be set to NULLAGBLOCK and 0, respectively.
130 xfs_reflink_find_shared(
131 struct xfs_perag
*pag
,
132 struct xfs_trans
*tp
,
137 bool find_end_of_shared
)
139 struct xfs_buf
*agbp
;
140 struct xfs_btree_cur
*cur
;
143 error
= xfs_alloc_read_agf(pag
, tp
, 0, &agbp
);
147 cur
= xfs_refcountbt_init_cursor(pag
->pag_mount
, tp
, agbp
, pag
);
149 error
= xfs_refcount_find_shared(cur
, agbno
, aglen
, fbno
, flen
,
152 xfs_btree_del_cursor(cur
, error
);
154 xfs_trans_brelse(tp
, agbp
);
159 * Trim the mapping to the next block where there's a change in the
160 * shared/unshared status. More specifically, this means that we
161 * find the lowest-numbered extent of shared blocks that coincides with
162 * the given block mapping. If the shared extent overlaps the start of
163 * the mapping, trim the mapping to the end of the shared extent. If
164 * the shared region intersects the mapping, trim the mapping to the
165 * start of the shared extent. If there are no shared regions that
166 * overlap, just return the original extent.
169 xfs_reflink_trim_around_shared(
170 struct xfs_inode
*ip
,
171 struct xfs_bmbt_irec
*irec
,
174 struct xfs_mount
*mp
= ip
->i_mount
;
175 struct xfs_perag
*pag
;
182 /* Holes, unwritten, and delalloc extents cannot be shared */
183 if (!xfs_is_cow_inode(ip
) || !xfs_bmap_is_written_extent(irec
)) {
188 trace_xfs_reflink_trim_around_shared(ip
, irec
);
190 pag
= xfs_perag_get(mp
, XFS_FSB_TO_AGNO(mp
, irec
->br_startblock
));
191 agbno
= XFS_FSB_TO_AGBNO(mp
, irec
->br_startblock
);
192 aglen
= irec
->br_blockcount
;
194 error
= xfs_reflink_find_shared(pag
, NULL
, agbno
, aglen
, &fbno
, &flen
,
201 if (fbno
== NULLAGBLOCK
) {
202 /* No shared blocks at all. */
208 * The start of this extent is shared. Truncate the
209 * mapping at the end of the shared region so that a
210 * subsequent iteration starts at the start of the
213 irec
->br_blockcount
= flen
;
219 * There's a shared extent midway through this extent.
220 * Truncate the mapping at the start of the shared
221 * extent so that a subsequent iteration starts at the
222 * start of the shared region.
224 irec
->br_blockcount
= fbno
- agbno
;
230 struct xfs_inode
*ip
,
231 struct xfs_bmbt_irec
*imap
,
234 /* We can't update any real extents in always COW mode. */
235 if (xfs_is_always_cow_inode(ip
) &&
236 !isnullstartblock(imap
->br_startblock
)) {
241 /* Trim the mapping to the nearest shared extent boundary. */
242 return xfs_reflink_trim_around_shared(ip
, imap
, shared
);
246 xfs_reflink_convert_cow_locked(
247 struct xfs_inode
*ip
,
248 xfs_fileoff_t offset_fsb
,
249 xfs_filblks_t count_fsb
)
251 struct xfs_iext_cursor icur
;
252 struct xfs_bmbt_irec got
;
253 struct xfs_btree_cur
*dummy_cur
= NULL
;
257 if (!xfs_iext_lookup_extent(ip
, ip
->i_cowfp
, offset_fsb
, &icur
, &got
))
261 if (got
.br_startoff
>= offset_fsb
+ count_fsb
)
263 if (got
.br_state
== XFS_EXT_NORM
)
265 if (WARN_ON_ONCE(isnullstartblock(got
.br_startblock
)))
268 xfs_trim_extent(&got
, offset_fsb
, count_fsb
);
269 if (!got
.br_blockcount
)
272 got
.br_state
= XFS_EXT_NORM
;
273 error
= xfs_bmap_add_extent_unwritten_real(NULL
, ip
,
274 XFS_COW_FORK
, &icur
, &dummy_cur
, &got
,
278 } while (xfs_iext_next_extent(ip
->i_cowfp
, &icur
, &got
));
283 /* Convert all of the unwritten CoW extents in a file's range to real ones. */
285 xfs_reflink_convert_cow(
286 struct xfs_inode
*ip
,
290 struct xfs_mount
*mp
= ip
->i_mount
;
291 xfs_fileoff_t offset_fsb
= XFS_B_TO_FSBT(mp
, offset
);
292 xfs_fileoff_t end_fsb
= XFS_B_TO_FSB(mp
, offset
+ count
);
293 xfs_filblks_t count_fsb
= end_fsb
- offset_fsb
;
298 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
299 error
= xfs_reflink_convert_cow_locked(ip
, offset_fsb
, count_fsb
);
300 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
305 * Find the extent that maps the given range in the COW fork. Even if the extent
306 * is not shared we might have a preallocation for it in the COW fork. If so we
307 * use it that rather than trigger a new allocation.
310 xfs_find_trim_cow_extent(
311 struct xfs_inode
*ip
,
312 struct xfs_bmbt_irec
*imap
,
313 struct xfs_bmbt_irec
*cmap
,
317 xfs_fileoff_t offset_fsb
= imap
->br_startoff
;
318 xfs_filblks_t count_fsb
= imap
->br_blockcount
;
319 struct xfs_iext_cursor icur
;
324 * If we don't find an overlapping extent, trim the range we need to
325 * allocate to fit the hole we found.
327 if (!xfs_iext_lookup_extent(ip
, ip
->i_cowfp
, offset_fsb
, &icur
, cmap
))
328 cmap
->br_startoff
= offset_fsb
+ count_fsb
;
329 if (cmap
->br_startoff
> offset_fsb
) {
330 xfs_trim_extent(imap
, imap
->br_startoff
,
331 cmap
->br_startoff
- imap
->br_startoff
);
332 return xfs_bmap_trim_cow(ip
, imap
, shared
);
336 if (isnullstartblock(cmap
->br_startblock
)) {
337 xfs_trim_extent(imap
, cmap
->br_startoff
, cmap
->br_blockcount
);
341 /* real extent found - no need to allocate */
342 xfs_trim_extent(cmap
, offset_fsb
, count_fsb
);
348 xfs_reflink_convert_unwritten(
349 struct xfs_inode
*ip
,
350 struct xfs_bmbt_irec
*imap
,
351 struct xfs_bmbt_irec
*cmap
,
354 xfs_fileoff_t offset_fsb
= imap
->br_startoff
;
355 xfs_filblks_t count_fsb
= imap
->br_blockcount
;
359 * cmap might larger than imap due to cowextsize hint.
361 xfs_trim_extent(cmap
, offset_fsb
, count_fsb
);
364 * COW fork extents are supposed to remain unwritten until we're ready
365 * to initiate a disk write. For direct I/O we are going to write the
366 * data and need the conversion, but for buffered writes we're done.
368 if (!convert_now
|| cmap
->br_state
== XFS_EXT_NORM
)
371 trace_xfs_reflink_convert_cow(ip
, cmap
);
373 error
= xfs_reflink_convert_cow_locked(ip
, offset_fsb
, count_fsb
);
375 cmap
->br_state
= XFS_EXT_NORM
;
381 xfs_reflink_fill_cow_hole(
382 struct xfs_inode
*ip
,
383 struct xfs_bmbt_irec
*imap
,
384 struct xfs_bmbt_irec
*cmap
,
389 struct xfs_mount
*mp
= ip
->i_mount
;
390 struct xfs_trans
*tp
;
391 xfs_filblks_t resaligned
;
392 xfs_extlen_t resblks
;
397 resaligned
= xfs_aligned_fsb_count(imap
->br_startoff
,
398 imap
->br_blockcount
, xfs_get_cowextsz_hint(ip
));
399 resblks
= XFS_DIOSTRAT_SPACE_RES(mp
, resaligned
);
401 xfs_iunlock(ip
, *lockmode
);
404 error
= xfs_trans_alloc_inode(ip
, &M_RES(mp
)->tr_write
, resblks
, 0,
409 *lockmode
= XFS_ILOCK_EXCL
;
411 error
= xfs_find_trim_cow_extent(ip
, imap
, cmap
, shared
, &found
);
412 if (error
|| !*shared
)
413 goto out_trans_cancel
;
416 xfs_trans_cancel(tp
);
420 /* Allocate the entire reservation as unwritten blocks. */
422 error
= xfs_bmapi_write(tp
, ip
, imap
->br_startoff
, imap
->br_blockcount
,
423 XFS_BMAPI_COWFORK
| XFS_BMAPI_PREALLOC
, 0, cmap
,
426 goto out_trans_cancel
;
428 xfs_inode_set_cowblocks_tag(ip
);
429 error
= xfs_trans_commit(tp
);
434 return xfs_reflink_convert_unwritten(ip
, imap
, cmap
, convert_now
);
437 xfs_trans_cancel(tp
);
442 xfs_reflink_fill_delalloc(
443 struct xfs_inode
*ip
,
444 struct xfs_bmbt_irec
*imap
,
445 struct xfs_bmbt_irec
*cmap
,
450 struct xfs_mount
*mp
= ip
->i_mount
;
451 struct xfs_trans
*tp
;
457 xfs_iunlock(ip
, *lockmode
);
460 error
= xfs_trans_alloc_inode(ip
, &M_RES(mp
)->tr_write
, 0, 0,
465 *lockmode
= XFS_ILOCK_EXCL
;
467 error
= xfs_find_trim_cow_extent(ip
, imap
, cmap
, shared
,
469 if (error
|| !*shared
)
470 goto out_trans_cancel
;
473 xfs_trans_cancel(tp
);
477 ASSERT(isnullstartblock(cmap
->br_startblock
) ||
478 cmap
->br_startblock
== DELAYSTARTBLOCK
);
481 * Replace delalloc reservation with an unwritten extent.
484 error
= xfs_bmapi_write(tp
, ip
, cmap
->br_startoff
,
486 XFS_BMAPI_COWFORK
| XFS_BMAPI_PREALLOC
, 0,
489 goto out_trans_cancel
;
491 xfs_inode_set_cowblocks_tag(ip
);
492 error
= xfs_trans_commit(tp
);
495 } while (cmap
->br_startoff
+ cmap
->br_blockcount
<= imap
->br_startoff
);
497 return xfs_reflink_convert_unwritten(ip
, imap
, cmap
, convert_now
);
500 xfs_trans_cancel(tp
);
504 /* Allocate all CoW reservations covering a range of blocks in a file. */
506 xfs_reflink_allocate_cow(
507 struct xfs_inode
*ip
,
508 struct xfs_bmbt_irec
*imap
,
509 struct xfs_bmbt_irec
*cmap
,
517 xfs_assert_ilocked(ip
, XFS_ILOCK_EXCL
);
519 ASSERT(!xfs_is_reflink_inode(ip
));
520 xfs_ifork_init_cow(ip
);
523 error
= xfs_find_trim_cow_extent(ip
, imap
, cmap
, shared
, &found
);
524 if (error
|| !*shared
)
527 /* CoW fork has a real extent */
529 return xfs_reflink_convert_unwritten(ip
, imap
, cmap
,
533 * CoW fork does not have an extent and data extent is shared.
534 * Allocate a real extent in the CoW fork.
536 if (cmap
->br_startoff
> imap
->br_startoff
)
537 return xfs_reflink_fill_cow_hole(ip
, imap
, cmap
, shared
,
538 lockmode
, convert_now
);
541 * CoW fork has a delalloc reservation. Replace it with a real extent.
542 * There may or may not be a data fork mapping.
544 if (isnullstartblock(cmap
->br_startblock
) ||
545 cmap
->br_startblock
== DELAYSTARTBLOCK
)
546 return xfs_reflink_fill_delalloc(ip
, imap
, cmap
, shared
,
547 lockmode
, convert_now
);
549 /* Shouldn't get here. */
551 return -EFSCORRUPTED
;
555 * Cancel CoW reservations for some block range of an inode.
557 * If cancel_real is true this function cancels all COW fork extents for the
558 * inode; if cancel_real is false, real extents are not cleared.
560 * Caller must have already joined the inode to the current transaction. The
561 * inode will be joined to the transaction returned to the caller.
564 xfs_reflink_cancel_cow_blocks(
565 struct xfs_inode
*ip
,
566 struct xfs_trans
**tpp
,
567 xfs_fileoff_t offset_fsb
,
568 xfs_fileoff_t end_fsb
,
571 struct xfs_ifork
*ifp
= xfs_ifork_ptr(ip
, XFS_COW_FORK
);
572 struct xfs_bmbt_irec got
, del
;
573 struct xfs_iext_cursor icur
;
576 if (!xfs_inode_has_cow_data(ip
))
578 if (!xfs_iext_lookup_extent_before(ip
, ifp
, &end_fsb
, &icur
, &got
))
581 /* Walk backwards until we're out of the I/O range... */
582 while (got
.br_startoff
+ got
.br_blockcount
> offset_fsb
) {
584 xfs_trim_extent(&del
, offset_fsb
, end_fsb
- offset_fsb
);
586 /* Extent delete may have bumped ext forward */
587 if (!del
.br_blockcount
) {
588 xfs_iext_prev(ifp
, &icur
);
592 trace_xfs_reflink_cancel_cow(ip
, &del
);
594 if (isnullstartblock(del
.br_startblock
)) {
595 xfs_bmap_del_extent_delay(ip
, XFS_COW_FORK
, &icur
, &got
,
597 } else if (del
.br_state
== XFS_EXT_UNWRITTEN
|| cancel_real
) {
598 ASSERT((*tpp
)->t_highest_agno
== NULLAGNUMBER
);
600 /* Free the CoW orphan record. */
601 xfs_refcount_free_cow_extent(*tpp
, del
.br_startblock
,
604 error
= xfs_free_extent_later(*tpp
, del
.br_startblock
,
605 del
.br_blockcount
, NULL
,
606 XFS_AG_RESV_NONE
, 0);
610 /* Roll the transaction */
611 error
= xfs_defer_finish(tpp
);
615 /* Remove the mapping from the CoW fork. */
616 xfs_bmap_del_extent_cow(ip
, &icur
, &got
, &del
);
618 /* Remove the quota reservation */
619 xfs_quota_unreserve_blkres(ip
, del
.br_blockcount
);
621 /* Didn't do anything, push cursor back. */
622 xfs_iext_prev(ifp
, &icur
);
625 if (!xfs_iext_get_extent(ifp
, &icur
, &got
))
629 /* clear tag if cow fork is emptied */
631 xfs_inode_clear_cowblocks_tag(ip
);
636 * Cancel CoW reservations for some byte range of an inode.
638 * If cancel_real is true this function cancels all COW fork extents for the
639 * inode; if cancel_real is false, real extents are not cleared.
642 xfs_reflink_cancel_cow_range(
643 struct xfs_inode
*ip
,
648 struct xfs_trans
*tp
;
649 xfs_fileoff_t offset_fsb
;
650 xfs_fileoff_t end_fsb
;
653 trace_xfs_reflink_cancel_cow_range(ip
, offset
, count
);
656 offset_fsb
= XFS_B_TO_FSBT(ip
->i_mount
, offset
);
657 if (count
== NULLFILEOFF
)
658 end_fsb
= NULLFILEOFF
;
660 end_fsb
= XFS_B_TO_FSB(ip
->i_mount
, offset
+ count
);
662 /* Start a rolling transaction to remove the mappings */
663 error
= xfs_trans_alloc(ip
->i_mount
, &M_RES(ip
->i_mount
)->tr_write
,
668 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
669 xfs_trans_ijoin(tp
, ip
, 0);
671 /* Scrape out the old CoW reservations */
672 error
= xfs_reflink_cancel_cow_blocks(ip
, &tp
, offset_fsb
, end_fsb
,
677 error
= xfs_trans_commit(tp
);
679 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
683 xfs_trans_cancel(tp
);
684 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
686 trace_xfs_reflink_cancel_cow_range_error(ip
, error
, _RET_IP_
);
691 * Remap part of the CoW fork into the data fork.
693 * We aim to remap the range starting at @offset_fsb and ending at @end_fsb
694 * into the data fork; this function will remap what it can (at the end of the
695 * range) and update @end_fsb appropriately. Each remap gets its own
696 * transaction because we can end up merging and splitting bmbt blocks for
697 * every remap operation and we'd like to keep the block reservation
698 * requirements as low as possible.
701 xfs_reflink_end_cow_extent(
702 struct xfs_inode
*ip
,
703 xfs_fileoff_t
*offset_fsb
,
704 xfs_fileoff_t end_fsb
)
706 struct xfs_iext_cursor icur
;
707 struct xfs_bmbt_irec got
, del
, data
;
708 struct xfs_mount
*mp
= ip
->i_mount
;
709 struct xfs_trans
*tp
;
710 struct xfs_ifork
*ifp
= xfs_ifork_ptr(ip
, XFS_COW_FORK
);
711 unsigned int resblks
;
715 resblks
= XFS_EXTENTADD_SPACE_RES(mp
, XFS_DATA_FORK
);
716 error
= xfs_trans_alloc(mp
, &M_RES(mp
)->tr_write
, resblks
, 0,
717 XFS_TRANS_RESERVE
, &tp
);
722 * Lock the inode. We have to ijoin without automatic unlock because
723 * the lead transaction is the refcountbt record deletion; the data
724 * fork update follows as a deferred log item.
726 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
727 xfs_trans_ijoin(tp
, ip
, 0);
730 * In case of racing, overlapping AIO writes no COW extents might be
731 * left by the time I/O completes for the loser of the race. In that
734 if (!xfs_iext_lookup_extent(ip
, ifp
, *offset_fsb
, &icur
, &got
) ||
735 got
.br_startoff
>= end_fsb
) {
736 *offset_fsb
= end_fsb
;
741 * Only remap real extents that contain data. With AIO, speculative
742 * preallocations can leak into the range we are called upon, and we
743 * need to skip them. Preserve @got for the eventual CoW fork
744 * deletion; from now on @del represents the mapping that we're
745 * actually remapping.
747 while (!xfs_bmap_is_written_extent(&got
)) {
748 if (!xfs_iext_next_extent(ifp
, &icur
, &got
) ||
749 got
.br_startoff
>= end_fsb
) {
750 *offset_fsb
= end_fsb
;
755 xfs_trim_extent(&del
, *offset_fsb
, end_fsb
- *offset_fsb
);
757 error
= xfs_iext_count_extend(tp
, ip
, XFS_DATA_FORK
,
758 XFS_IEXT_REFLINK_END_COW_CNT
);
762 /* Grab the corresponding mapping in the data fork. */
764 error
= xfs_bmapi_read(ip
, del
.br_startoff
, del
.br_blockcount
, &data
,
769 /* We can only remap the smaller of the two extent sizes. */
770 data
.br_blockcount
= min(data
.br_blockcount
, del
.br_blockcount
);
771 del
.br_blockcount
= data
.br_blockcount
;
773 trace_xfs_reflink_cow_remap_from(ip
, &del
);
774 trace_xfs_reflink_cow_remap_to(ip
, &data
);
776 if (xfs_bmap_is_real_extent(&data
)) {
778 * If the extent we're remapping is backed by storage (written
779 * or not), unmap the extent and drop its refcount.
781 xfs_bmap_unmap_extent(tp
, ip
, XFS_DATA_FORK
, &data
);
782 xfs_refcount_decrease_extent(tp
, &data
);
783 xfs_trans_mod_dquot_byino(tp
, ip
, XFS_TRANS_DQ_BCOUNT
,
784 -data
.br_blockcount
);
785 } else if (data
.br_startblock
== DELAYSTARTBLOCK
) {
789 * If the extent we're remapping is a delalloc reservation,
790 * we can use the regular bunmapi function to release the
791 * incore state. Dropping the delalloc reservation takes care
792 * of the quota reservation for us.
794 error
= xfs_bunmapi(NULL
, ip
, data
.br_startoff
,
795 data
.br_blockcount
, 0, 1, &done
);
801 /* Free the CoW orphan record. */
802 xfs_refcount_free_cow_extent(tp
, del
.br_startblock
, del
.br_blockcount
);
804 /* Map the new blocks into the data fork. */
805 xfs_bmap_map_extent(tp
, ip
, XFS_DATA_FORK
, &del
);
807 /* Charge this new data fork mapping to the on-disk quota. */
808 xfs_trans_mod_dquot_byino(tp
, ip
, XFS_TRANS_DQ_DELBCOUNT
,
809 (long)del
.br_blockcount
);
811 /* Remove the mapping from the CoW fork. */
812 xfs_bmap_del_extent_cow(ip
, &icur
, &got
, &del
);
814 error
= xfs_trans_commit(tp
);
815 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
819 /* Update the caller about how much progress we made. */
820 *offset_fsb
= del
.br_startoff
+ del
.br_blockcount
;
824 xfs_trans_cancel(tp
);
825 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
830 * Remap parts of a file's data fork after a successful CoW.
834 struct xfs_inode
*ip
,
838 xfs_fileoff_t offset_fsb
;
839 xfs_fileoff_t end_fsb
;
842 trace_xfs_reflink_end_cow(ip
, offset
, count
);
844 offset_fsb
= XFS_B_TO_FSBT(ip
->i_mount
, offset
);
845 end_fsb
= XFS_B_TO_FSB(ip
->i_mount
, offset
+ count
);
848 * Walk forwards until we've remapped the I/O range. The loop function
849 * repeatedly cycles the ILOCK to allocate one transaction per remapped
852 * If we're being called by writeback then the pages will still
853 * have PageWriteback set, which prevents races with reflink remapping
854 * and truncate. Reflink remapping prevents races with writeback by
855 * taking the iolock and mmaplock before flushing the pages and
856 * remapping, which means there won't be any further writeback or page
857 * cache dirtying until the reflink completes.
859 * We should never have two threads issuing writeback for the same file
860 * region. There are also have post-eof checks in the writeback
861 * preparation code so that we don't bother writing out pages that are
862 * about to be truncated.
864 * If we're being called as part of directio write completion, the dio
865 * count is still elevated, which reflink and truncate will wait for.
866 * Reflink remapping takes the iolock and mmaplock and waits for
867 * pending dio to finish, which should prevent any directio until the
868 * remap completes. Multiple concurrent directio writes to the same
869 * region are handled by end_cow processing only occurring for the
870 * threads which succeed; the outcome of multiple overlapping direct
871 * writes is not well defined anyway.
873 * It's possible that a buffered write and a direct write could collide
874 * here (the buffered write stumbles in after the dio flushes and
875 * invalidates the page cache and immediately queues writeback), but we
876 * have never supported this 100%. If either disk write succeeds the
877 * blocks will be remapped.
879 while (end_fsb
> offset_fsb
&& !error
)
880 error
= xfs_reflink_end_cow_extent(ip
, &offset_fsb
, end_fsb
);
883 trace_xfs_reflink_end_cow_error(ip
, error
, _RET_IP_
);
888 * Free all CoW staging blocks that are still referenced by the ondisk refcount
889 * metadata. The ondisk metadata does not track which inode created the
890 * staging extent, so callers must ensure that there are no cached inodes with
891 * live CoW staging extents.
894 xfs_reflink_recover_cow(
895 struct xfs_mount
*mp
)
897 struct xfs_perag
*pag
;
901 if (!xfs_has_reflink(mp
))
904 for_each_perag(mp
, agno
, pag
) {
905 error
= xfs_refcount_recover_cow_leftovers(mp
, pag
);
916 * Reflinking (Block) Ranges of Two Files Together
918 * First, ensure that the reflink flag is set on both inodes. The flag is an
919 * optimization to avoid unnecessary refcount btree lookups in the write path.
921 * Now we can iteratively remap the range of extents (and holes) in src to the
922 * corresponding ranges in dest. Let drange and srange denote the ranges of
923 * logical blocks in dest and src touched by the reflink operation.
925 * While the length of drange is greater than zero,
926 * - Read src's bmbt at the start of srange ("imap")
927 * - If imap doesn't exist, make imap appear to start at the end of srange
929 * - If imap starts before srange, advance imap to start at srange.
930 * - If imap goes beyond srange, truncate imap to end at the end of srange.
931 * - Punch (imap start - srange start + imap len) blocks from dest at
932 * offset (drange start).
933 * - If imap points to a real range of pblks,
934 * > Increase the refcount of the imap's pblks
935 * > Map imap's pblks into dest at the offset
936 * (drange start + imap start - srange start)
937 * - Advance drange and srange by (imap start - srange start + imap len)
939 * Finally, if the reflink made dest longer, update both the in-core and
940 * on-disk file sizes.
942 * ASCII Art Demonstration:
944 * Let's say we want to reflink this source file:
946 * ----SSSSSSS-SSSSS----SSSSSS (src file)
947 * <-------------------->
949 * into this destination file:
951 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file)
952 * <-------------------->
953 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest.
954 * Observe that the range has different logical offsets in either file.
956 * Consider that the first extent in the source file doesn't line up with our
957 * reflink range. Unmapping and remapping are separate operations, so we can
958 * unmap more blocks from the destination file than we remap.
960 * ----SSSSSSS-SSSSS----SSSSSS
962 * --DDDDD---------DDDDD--DDD
965 * Now remap the source extent into the destination file:
967 * ----SSSSSSS-SSSSS----SSSSSS
969 * --DDDDD--SSSSSSSDDDDD--DDD
972 * Do likewise with the second hole and extent in our range. Holes in the
973 * unmap range don't affect our operation.
975 * ----SSSSSSS-SSSSS----SSSSSS
977 * --DDDDD--SSSSSSS-SSSSS-DDD
980 * Finally, unmap and remap part of the third extent. This will increase the
981 * size of the destination file.
983 * ----SSSSSSS-SSSSS----SSSSSS
985 * --DDDDD--SSSSSSS-SSSSS----SSS
988 * Once we update the destination file's i_size, we're done.
992 * Ensure the reflink bit is set in both inodes.
995 xfs_reflink_set_inode_flag(
996 struct xfs_inode
*src
,
997 struct xfs_inode
*dest
)
999 struct xfs_mount
*mp
= src
->i_mount
;
1001 struct xfs_trans
*tp
;
1003 if (xfs_is_reflink_inode(src
) && xfs_is_reflink_inode(dest
))
1006 error
= xfs_trans_alloc(mp
, &M_RES(mp
)->tr_ichange
, 0, 0, 0, &tp
);
1010 /* Lock both files against IO */
1011 if (src
->i_ino
== dest
->i_ino
)
1012 xfs_ilock(src
, XFS_ILOCK_EXCL
);
1014 xfs_lock_two_inodes(src
, XFS_ILOCK_EXCL
, dest
, XFS_ILOCK_EXCL
);
1016 if (!xfs_is_reflink_inode(src
)) {
1017 trace_xfs_reflink_set_inode_flag(src
);
1018 xfs_trans_ijoin(tp
, src
, XFS_ILOCK_EXCL
);
1019 src
->i_diflags2
|= XFS_DIFLAG2_REFLINK
;
1020 xfs_trans_log_inode(tp
, src
, XFS_ILOG_CORE
);
1021 xfs_ifork_init_cow(src
);
1023 xfs_iunlock(src
, XFS_ILOCK_EXCL
);
1025 if (src
->i_ino
== dest
->i_ino
)
1028 if (!xfs_is_reflink_inode(dest
)) {
1029 trace_xfs_reflink_set_inode_flag(dest
);
1030 xfs_trans_ijoin(tp
, dest
, XFS_ILOCK_EXCL
);
1031 dest
->i_diflags2
|= XFS_DIFLAG2_REFLINK
;
1032 xfs_trans_log_inode(tp
, dest
, XFS_ILOG_CORE
);
1033 xfs_ifork_init_cow(dest
);
1035 xfs_iunlock(dest
, XFS_ILOCK_EXCL
);
1038 error
= xfs_trans_commit(tp
);
1044 trace_xfs_reflink_set_inode_flag_error(dest
, error
, _RET_IP_
);
1049 * Update destination inode size & cowextsize hint, if necessary.
1052 xfs_reflink_update_dest(
1053 struct xfs_inode
*dest
,
1055 xfs_extlen_t cowextsize
,
1056 unsigned int remap_flags
)
1058 struct xfs_mount
*mp
= dest
->i_mount
;
1059 struct xfs_trans
*tp
;
1062 if (newlen
<= i_size_read(VFS_I(dest
)) && cowextsize
== 0)
1065 error
= xfs_trans_alloc(mp
, &M_RES(mp
)->tr_ichange
, 0, 0, 0, &tp
);
1069 xfs_ilock(dest
, XFS_ILOCK_EXCL
);
1070 xfs_trans_ijoin(tp
, dest
, XFS_ILOCK_EXCL
);
1072 if (newlen
> i_size_read(VFS_I(dest
))) {
1073 trace_xfs_reflink_update_inode_size(dest
, newlen
);
1074 i_size_write(VFS_I(dest
), newlen
);
1075 dest
->i_disk_size
= newlen
;
1079 dest
->i_cowextsize
= cowextsize
;
1080 dest
->i_diflags2
|= XFS_DIFLAG2_COWEXTSIZE
;
1083 xfs_trans_log_inode(tp
, dest
, XFS_ILOG_CORE
);
1085 error
= xfs_trans_commit(tp
);
1091 trace_xfs_reflink_update_inode_size_error(dest
, error
, _RET_IP_
);
1096 * Do we have enough reserve in this AG to handle a reflink? The refcount
1097 * btree already reserved all the space it needs, but the rmap btree can grow
1098 * infinitely, so we won't allow more reflinks when the AG is down to the
1102 xfs_reflink_ag_has_free_space(
1103 struct xfs_mount
*mp
,
1104 xfs_agnumber_t agno
)
1106 struct xfs_perag
*pag
;
1109 if (!xfs_has_rmapbt(mp
))
1112 pag
= xfs_perag_get(mp
, agno
);
1113 if (xfs_ag_resv_critical(pag
, XFS_AG_RESV_RMAPBT
) ||
1114 xfs_ag_resv_critical(pag
, XFS_AG_RESV_METADATA
))
1121 * Remap the given extent into the file. The dmap blockcount will be set to
1122 * the number of blocks that were actually remapped.
1125 xfs_reflink_remap_extent(
1126 struct xfs_inode
*ip
,
1127 struct xfs_bmbt_irec
*dmap
,
1128 xfs_off_t new_isize
)
1130 struct xfs_bmbt_irec smap
;
1131 struct xfs_mount
*mp
= ip
->i_mount
;
1132 struct xfs_trans
*tp
;
1135 unsigned int resblks
;
1136 bool quota_reserved
= true;
1138 bool dmap_written
= xfs_bmap_is_written_extent(dmap
);
1144 * Start a rolling transaction to switch the mappings.
1146 * Adding a written extent to the extent map can cause a bmbt split,
1147 * and removing a mapped extent from the extent can cause a bmbt split.
1148 * The two operations cannot both cause a split since they operate on
1149 * the same index in the bmap btree, so we only need a reservation for
1150 * one bmbt split if either thing is happening. However, we haven't
1151 * locked the inode yet, so we reserve assuming this is the case.
1153 * The first allocation call tries to reserve enough space to handle
1154 * mapping dmap into a sparse part of the file plus the bmbt split. We
1155 * haven't locked the inode or read the existing mapping yet, so we do
1156 * not know for sure that we need the space. This should succeed most
1159 * If the first attempt fails, try again but reserving only enough
1160 * space to handle a bmbt split. This is the hard minimum requirement,
1161 * and we revisit quota reservations later when we know more about what
1164 resblks
= XFS_EXTENTADD_SPACE_RES(mp
, XFS_DATA_FORK
);
1165 error
= xfs_trans_alloc_inode(ip
, &M_RES(mp
)->tr_write
,
1166 resblks
+ dmap
->br_blockcount
, 0, false, &tp
);
1167 if (error
== -EDQUOT
|| error
== -ENOSPC
) {
1168 quota_reserved
= false;
1169 error
= xfs_trans_alloc_inode(ip
, &M_RES(mp
)->tr_write
,
1170 resblks
, 0, false, &tp
);
1176 * Read what's currently mapped in the destination file into smap.
1177 * If smap isn't a hole, we will have to remove it before we can add
1178 * dmap to the destination file.
1181 error
= xfs_bmapi_read(ip
, dmap
->br_startoff
, dmap
->br_blockcount
,
1185 ASSERT(nimaps
== 1 && smap
.br_startoff
== dmap
->br_startoff
);
1186 smap_real
= xfs_bmap_is_real_extent(&smap
);
1189 * We can only remap as many blocks as the smaller of the two extent
1190 * maps, because we can only remap one extent at a time.
1192 dmap
->br_blockcount
= min(dmap
->br_blockcount
, smap
.br_blockcount
);
1193 ASSERT(dmap
->br_blockcount
== smap
.br_blockcount
);
1195 trace_xfs_reflink_remap_extent_dest(ip
, &smap
);
1198 * Two extents mapped to the same physical block must not have
1199 * different states; that's filesystem corruption. Move on to the next
1200 * extent if they're both holes or both the same physical extent.
1202 if (dmap
->br_startblock
== smap
.br_startblock
) {
1203 if (dmap
->br_state
!= smap
.br_state
) {
1204 xfs_bmap_mark_sick(ip
, XFS_DATA_FORK
);
1205 error
= -EFSCORRUPTED
;
1210 /* If both extents are unwritten, leave them alone. */
1211 if (dmap
->br_state
== XFS_EXT_UNWRITTEN
&&
1212 smap
.br_state
== XFS_EXT_UNWRITTEN
)
1215 /* No reflinking if the AG of the dest mapping is low on space. */
1217 error
= xfs_reflink_ag_has_free_space(mp
,
1218 XFS_FSB_TO_AGNO(mp
, dmap
->br_startblock
));
1224 * Increase quota reservation if we think the quota block counter for
1225 * this file could increase.
1227 * If we are mapping a written extent into the file, we need to have
1228 * enough quota block count reservation to handle the blocks in that
1229 * extent. We log only the delta to the quota block counts, so if the
1230 * extent we're unmapping also has blocks allocated to it, we don't
1231 * need a quota reservation for the extent itself.
1233 * Note that if we're replacing a delalloc reservation with a written
1234 * extent, we have to take the full quota reservation because removing
1235 * the delalloc reservation gives the block count back to the quota
1236 * count. This is suboptimal, but the VFS flushed the dest range
1237 * before we started. That should have removed all the delalloc
1238 * reservations, but we code defensively.
1240 * xfs_trans_alloc_inode above already tried to grab an even larger
1241 * quota reservation, and kicked off a blockgc scan if it couldn't.
1242 * If we can't get a potentially smaller quota reservation now, we're
1245 if (!quota_reserved
&& !smap_real
&& dmap_written
) {
1246 error
= xfs_trans_reserve_quota_nblks(tp
, ip
,
1247 dmap
->br_blockcount
, 0, false);
1258 error
= xfs_iext_count_extend(tp
, ip
, XFS_DATA_FORK
, iext_delta
);
1264 * If the extent we're unmapping is backed by storage (written
1265 * or not), unmap the extent and drop its refcount.
1267 xfs_bmap_unmap_extent(tp
, ip
, XFS_DATA_FORK
, &smap
);
1268 xfs_refcount_decrease_extent(tp
, &smap
);
1269 qdelta
-= smap
.br_blockcount
;
1270 } else if (smap
.br_startblock
== DELAYSTARTBLOCK
) {
1274 * If the extent we're unmapping is a delalloc reservation,
1275 * we can use the regular bunmapi function to release the
1276 * incore state. Dropping the delalloc reservation takes care
1277 * of the quota reservation for us.
1279 error
= xfs_bunmapi(NULL
, ip
, smap
.br_startoff
,
1280 smap
.br_blockcount
, 0, 1, &done
);
1287 * If the extent we're sharing is backed by written storage, increase
1288 * its refcount and map it into the file.
1291 xfs_refcount_increase_extent(tp
, dmap
);
1292 xfs_bmap_map_extent(tp
, ip
, XFS_DATA_FORK
, dmap
);
1293 qdelta
+= dmap
->br_blockcount
;
1296 xfs_trans_mod_dquot_byino(tp
, ip
, XFS_TRANS_DQ_BCOUNT
, qdelta
);
1298 /* Update dest isize if needed. */
1299 newlen
= XFS_FSB_TO_B(mp
, dmap
->br_startoff
+ dmap
->br_blockcount
);
1300 newlen
= min_t(xfs_off_t
, newlen
, new_isize
);
1301 if (newlen
> i_size_read(VFS_I(ip
))) {
1302 trace_xfs_reflink_update_inode_size(ip
, newlen
);
1303 i_size_write(VFS_I(ip
), newlen
);
1304 ip
->i_disk_size
= newlen
;
1305 xfs_trans_log_inode(tp
, ip
, XFS_ILOG_CORE
);
1308 /* Commit everything and unlock. */
1309 error
= xfs_trans_commit(tp
);
1313 xfs_trans_cancel(tp
);
1315 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
1318 trace_xfs_reflink_remap_extent_error(ip
, error
, _RET_IP_
);
1322 /* Remap a range of one file to the other. */
1324 xfs_reflink_remap_blocks(
1325 struct xfs_inode
*src
,
1327 struct xfs_inode
*dest
,
1332 struct xfs_bmbt_irec imap
;
1333 struct xfs_mount
*mp
= src
->i_mount
;
1334 xfs_fileoff_t srcoff
= XFS_B_TO_FSBT(mp
, pos_in
);
1335 xfs_fileoff_t destoff
= XFS_B_TO_FSBT(mp
, pos_out
);
1337 xfs_filblks_t remapped_len
= 0;
1338 xfs_off_t new_isize
= pos_out
+ remap_len
;
1342 len
= min_t(xfs_filblks_t
, XFS_B_TO_FSB(mp
, remap_len
),
1345 trace_xfs_reflink_remap_blocks(src
, srcoff
, len
, dest
, destoff
);
1348 unsigned int lock_mode
;
1350 /* Read extent from the source file */
1352 lock_mode
= xfs_ilock_data_map_shared(src
);
1353 error
= xfs_bmapi_read(src
, srcoff
, len
, &imap
, &nimaps
, 0);
1354 xfs_iunlock(src
, lock_mode
);
1358 * The caller supposedly flushed all dirty pages in the source
1359 * file range, which means that writeback should have allocated
1360 * or deleted all delalloc reservations in that range. If we
1361 * find one, that's a good sign that something is seriously
1364 ASSERT(nimaps
== 1 && imap
.br_startoff
== srcoff
);
1365 if (imap
.br_startblock
== DELAYSTARTBLOCK
) {
1366 ASSERT(imap
.br_startblock
!= DELAYSTARTBLOCK
);
1367 xfs_bmap_mark_sick(src
, XFS_DATA_FORK
);
1368 error
= -EFSCORRUPTED
;
1372 trace_xfs_reflink_remap_extent_src(src
, &imap
);
1374 /* Remap into the destination file at the given offset. */
1375 imap
.br_startoff
= destoff
;
1376 error
= xfs_reflink_remap_extent(dest
, &imap
, new_isize
);
1380 if (fatal_signal_pending(current
)) {
1385 /* Advance drange/srange */
1386 srcoff
+= imap
.br_blockcount
;
1387 destoff
+= imap
.br_blockcount
;
1388 len
-= imap
.br_blockcount
;
1389 remapped_len
+= imap
.br_blockcount
;
1394 trace_xfs_reflink_remap_blocks_error(dest
, error
, _RET_IP_
);
1395 *remapped
= min_t(loff_t
, remap_len
,
1396 XFS_FSB_TO_B(src
->i_mount
, remapped_len
));
1401 * If we're reflinking to a point past the destination file's EOF, we must
1402 * zero any speculative post-EOF preallocations that sit between the old EOF
1403 * and the destination file offset.
1406 xfs_reflink_zero_posteof(
1407 struct xfs_inode
*ip
,
1410 loff_t isize
= i_size_read(VFS_I(ip
));
1415 trace_xfs_zero_eof(ip
, isize
, pos
- isize
);
1416 return xfs_zero_range(ip
, isize
, pos
- isize
, NULL
);
1420 * Prepare two files for range cloning. Upon a successful return both inodes
1421 * will have the iolock and mmaplock held, the page cache of the out file will
1422 * be truncated, and any leases on the out file will have been broken. This
1423 * function borrows heavily from xfs_file_aio_write_checks.
1425 * The VFS allows partial EOF blocks to "match" for dedupe even though it hasn't
1426 * checked that the bytes beyond EOF physically match. Hence we cannot use the
1427 * EOF block in the source dedupe range because it's not a complete block match,
1428 * hence can introduce a corruption into the file that has it's block replaced.
1430 * In similar fashion, the VFS file cloning also allows partial EOF blocks to be
1431 * "block aligned" for the purposes of cloning entire files. However, if the
1432 * source file range includes the EOF block and it lands within the existing EOF
1433 * of the destination file, then we can expose stale data from beyond the source
1434 * file EOF in the destination file.
1436 * XFS doesn't support partial block sharing, so in both cases we have check
1437 * these cases ourselves. For dedupe, we can simply round the length to dedupe
1438 * down to the previous whole block and ignore the partial EOF block. While this
1439 * means we can't dedupe the last block of a file, this is an acceptible
1440 * tradeoff for simplicity on implementation.
1442 * For cloning, we want to share the partial EOF block if it is also the new EOF
1443 * block of the destination file. If the partial EOF block lies inside the
1444 * existing destination EOF, then we have to abort the clone to avoid exposing
1445 * stale data in the destination file. Hence we reject these clone attempts with
1446 * -EINVAL in this case.
1449 xfs_reflink_remap_prep(
1450 struct file
*file_in
,
1452 struct file
*file_out
,
1455 unsigned int remap_flags
)
1457 struct inode
*inode_in
= file_inode(file_in
);
1458 struct xfs_inode
*src
= XFS_I(inode_in
);
1459 struct inode
*inode_out
= file_inode(file_out
);
1460 struct xfs_inode
*dest
= XFS_I(inode_out
);
1463 /* Lock both files against IO */
1464 ret
= xfs_ilock2_io_mmap(src
, dest
);
1468 /* Check file eligibility and prepare for block sharing. */
1470 /* Don't reflink realtime inodes */
1471 if (XFS_IS_REALTIME_INODE(src
) || XFS_IS_REALTIME_INODE(dest
))
1474 /* Don't share DAX file data with non-DAX file. */
1475 if (IS_DAX(inode_in
) != IS_DAX(inode_out
))
1478 if (!IS_DAX(inode_in
))
1479 ret
= generic_remap_file_range_prep(file_in
, pos_in
, file_out
,
1480 pos_out
, len
, remap_flags
);
1482 ret
= dax_remap_file_range_prep(file_in
, pos_in
, file_out
,
1483 pos_out
, len
, remap_flags
, &xfs_read_iomap_ops
);
1484 if (ret
|| *len
== 0)
1487 /* Attach dquots to dest inode before changing block map */
1488 ret
= xfs_qm_dqattach(dest
);
1493 * Zero existing post-eof speculative preallocations in the destination
1496 ret
= xfs_reflink_zero_posteof(dest
, pos_out
);
1500 /* Set flags and remap blocks. */
1501 ret
= xfs_reflink_set_inode_flag(src
, dest
);
1506 * If pos_out > EOF, we may have dirtied blocks between EOF and
1507 * pos_out. In that case, we need to extend the flush and unmap to cover
1508 * from EOF to the end of the copy length.
1510 if (pos_out
> XFS_ISIZE(dest
)) {
1511 loff_t flen
= *len
+ (pos_out
- XFS_ISIZE(dest
));
1512 ret
= xfs_flush_unmap_range(dest
, XFS_ISIZE(dest
), flen
);
1514 ret
= xfs_flush_unmap_range(dest
, pos_out
, *len
);
1519 xfs_iflags_set(src
, XFS_IREMAPPING
);
1520 if (inode_in
!= inode_out
)
1521 xfs_ilock_demote(src
, XFS_IOLOCK_EXCL
| XFS_MMAPLOCK_EXCL
);
1525 xfs_iunlock2_io_mmap(src
, dest
);
1529 /* Does this inode need the reflink flag? */
1531 xfs_reflink_inode_has_shared_extents(
1532 struct xfs_trans
*tp
,
1533 struct xfs_inode
*ip
,
1536 struct xfs_bmbt_irec got
;
1537 struct xfs_mount
*mp
= ip
->i_mount
;
1538 struct xfs_ifork
*ifp
;
1539 struct xfs_iext_cursor icur
;
1543 ifp
= xfs_ifork_ptr(ip
, XFS_DATA_FORK
);
1544 error
= xfs_iread_extents(tp
, ip
, XFS_DATA_FORK
);
1548 *has_shared
= false;
1549 found
= xfs_iext_lookup_extent(ip
, ifp
, 0, &icur
, &got
);
1551 struct xfs_perag
*pag
;
1552 xfs_agblock_t agbno
;
1557 if (isnullstartblock(got
.br_startblock
) ||
1558 got
.br_state
!= XFS_EXT_NORM
)
1561 pag
= xfs_perag_get(mp
, XFS_FSB_TO_AGNO(mp
, got
.br_startblock
));
1562 agbno
= XFS_FSB_TO_AGBNO(mp
, got
.br_startblock
);
1563 aglen
= got
.br_blockcount
;
1564 error
= xfs_reflink_find_shared(pag
, tp
, agbno
, aglen
,
1565 &rbno
, &rlen
, false);
1570 /* Is there still a shared block here? */
1571 if (rbno
!= NULLAGBLOCK
) {
1576 found
= xfs_iext_next_extent(ifp
, &icur
, &got
);
1583 * Clear the inode reflink flag if there are no shared extents.
1585 * The caller is responsible for joining the inode to the transaction passed in.
1586 * The inode will be joined to the transaction that is returned to the caller.
1589 xfs_reflink_clear_inode_flag(
1590 struct xfs_inode
*ip
,
1591 struct xfs_trans
**tpp
)
1596 ASSERT(xfs_is_reflink_inode(ip
));
1598 error
= xfs_reflink_inode_has_shared_extents(*tpp
, ip
, &needs_flag
);
1599 if (error
|| needs_flag
)
1603 * We didn't find any shared blocks so turn off the reflink flag.
1604 * First, get rid of any leftover CoW mappings.
1606 error
= xfs_reflink_cancel_cow_blocks(ip
, tpp
, 0, XFS_MAX_FILEOFF
,
1611 /* Clear the inode flag. */
1612 trace_xfs_reflink_unset_inode_flag(ip
);
1613 ip
->i_diflags2
&= ~XFS_DIFLAG2_REFLINK
;
1614 xfs_inode_clear_cowblocks_tag(ip
);
1615 xfs_trans_log_inode(*tpp
, ip
, XFS_ILOG_CORE
);
1621 * Clear the inode reflink flag if there are no shared extents and the size
1625 xfs_reflink_try_clear_inode_flag(
1626 struct xfs_inode
*ip
)
1628 struct xfs_mount
*mp
= ip
->i_mount
;
1629 struct xfs_trans
*tp
;
1632 /* Start a rolling transaction to remove the mappings */
1633 error
= xfs_trans_alloc(mp
, &M_RES(mp
)->tr_write
, 0, 0, 0, &tp
);
1637 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
1638 xfs_trans_ijoin(tp
, ip
, 0);
1640 error
= xfs_reflink_clear_inode_flag(ip
, &tp
);
1644 error
= xfs_trans_commit(tp
);
1648 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
1651 xfs_trans_cancel(tp
);
1653 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
1658 * Pre-COW all shared blocks within a given byte range of a file and turn off
1659 * the reflink flag if we unshare all of the file's blocks.
1662 xfs_reflink_unshare(
1663 struct xfs_inode
*ip
,
1667 struct inode
*inode
= VFS_I(ip
);
1670 if (!xfs_is_reflink_inode(ip
))
1673 trace_xfs_reflink_unshare(ip
, offset
, len
);
1675 inode_dio_wait(inode
);
1678 error
= dax_file_unshare(inode
, offset
, len
,
1679 &xfs_dax_write_iomap_ops
);
1681 error
= iomap_file_unshare(inode
, offset
, len
,
1682 &xfs_buffered_write_iomap_ops
);
1686 error
= filemap_write_and_wait_range(inode
->i_mapping
, offset
,
1691 /* Turn off the reflink flag if possible. */
1692 error
= xfs_reflink_try_clear_inode_flag(ip
);
1698 trace_xfs_reflink_unshare_error(ip
, error
, _RET_IP_
);