2 * Copyright (c) 2011-2014 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * TRANSACTION AND FLUSH HANDLING
38 * Deceptively simple but actually fairly difficult to implement properly is
39 * how I would describe it.
41 * Flushing generally occurs bottom-up but requires a top-down scan to
42 * locate chains with MODIFIED and/or UPDATE bits set. The ONFLUSH flag
43 * tells how to recurse downward to find these chains.
46 #include <sys/cdefs.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/types.h>
57 #define HAMMER2_FLUSH_DEPTH_LIMIT 10 /* stack recursion limit */
61 * Recursively flush the specified chain. The chain is locked and
62 * referenced by the caller and will remain so on return. The chain
63 * will remain referenced throughout but can temporarily lose its
64 * lock during the recursion to avoid unnecessarily stalling user
67 struct hammer2_flush_info
{
68 hammer2_chain_t
*parent
;
69 hammer2_trans_t
*trans
;
73 struct h2_flush_list flushq
;
74 hammer2_xid_t sync_xid
; /* memory synchronization point */
75 hammer2_chain_t
*debug
;
78 typedef struct hammer2_flush_info hammer2_flush_info_t
;
80 static void hammer2_flush_core(hammer2_flush_info_t
*info
,
81 hammer2_chain_t
*chain
, int deleting
);
82 static int hammer2_flush_recurse(hammer2_chain_t
*child
, void *data
);
85 * For now use a global transaction manager. What we ultimately want to do
86 * is give each non-overlapping hmp/pmp group its own transaction manager.
88 * Transactions govern XID tracking on the physical media (the hmp), but they
89 * also govern TID tracking which is per-PFS and thus might cross multiple
90 * hmp's. So we can't just stuff tmanage into hammer2_dev or
94 hammer2_trans_manage_init(hammer2_trans_manage_t
*tman
)
96 lockinit(&tman
->translk
, "h2trans", 0, 0);
97 TAILQ_INIT(&tman
->transq
);
99 tman
->alloc_xid
= tman
->flush_xid
+ 1;
103 hammer2_trans_newxid(hammer2_pfs_t
*pmp
)
108 xid
= atomic_fetchadd_int(&pmp
->tmanage
.alloc_xid
, 1);
116 * Transaction support functions for writing to the filesystem.
118 * Initializing a new transaction allocates a transaction ID. Typically
119 * passed a pmp (hmp passed as NULL), indicating a cluster transaction. Can
120 * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
121 * media target. The latter mode is used by the recovery code.
123 * TWO TRANSACTION IDs can run concurrently, where one is a flush and the
124 * other is a set of any number of concurrent filesystem operations. We
125 * can either have <running_fs_ops> + <waiting_flush> + <blocked_fs_ops>
126 * or we can have <running_flush> + <concurrent_fs_ops>.
128 * During a flush, new fs_ops are only blocked until the fs_ops prior to
129 * the flush complete. The new fs_ops can then run concurrent with the flush.
131 * Buffer-cache transactions operate as fs_ops but never block. A
132 * buffer-cache flush will run either before or after the current pending
133 * flush depending on its state.
136 hammer2_trans_init(hammer2_trans_t
*trans
, hammer2_pfs_t
*pmp
, int flags
)
138 hammer2_trans_manage_t
*tman
;
139 hammer2_trans_t
*head
;
141 tman
= &pmp
->tmanage
;
143 bzero(trans
, sizeof(*trans
));
145 trans
->flags
= flags
;
146 trans
->td
= curthread
;
148 lockmgr(&tman
->translk
, LK_EXCLUSIVE
);
150 if (flags
& HAMMER2_TRANS_ISFLUSH
) {
152 * If multiple flushes are trying to run we have to
153 * wait until it is our turn. All flushes are serialized.
155 * We queue ourselves and then wait to become the head
156 * of the queue, allowing all prior flushes to complete.
158 * Multiple normal transactions can share the current
159 * transaction id but a flush transaction needs its own
160 * unique TID for proper block table update accounting.
164 tman
->flush_xid
= hammer2_trans_newxid(pmp
);
165 trans
->sync_xid
= tman
->flush_xid
;
166 trans
->modify_tid
= pmp
->modify_tid
;
167 TAILQ_INSERT_TAIL(&tman
->transq
, trans
, entry
);
168 if (TAILQ_FIRST(&tman
->transq
) != trans
) {
170 while (trans
->blocked
) {
171 lksleep(&trans
->sync_xid
, &tman
->translk
,
175 } else if (tman
->flushcnt
== 0) {
177 * No flushes are pending, we can go. Use prior flush_xid + 1.
179 * WARNING! Also see hammer2_chain_setflush()
181 TAILQ_INSERT_TAIL(&tman
->transq
, trans
, entry
);
182 trans
->sync_xid
= tman
->flush_xid
+ 1;
184 /* XXX improve/optimize inode allocation */
185 } else if (trans
->flags
& HAMMER2_TRANS_BUFCACHE
) {
187 * A buffer cache transaction is requested while a flush
188 * is in progress. The flush's PREFLUSH flag must be set
191 * The buffer cache flush takes on the main flush's
194 TAILQ_FOREACH(head
, &tman
->transq
, entry
) {
195 if (head
->flags
& HAMMER2_TRANS_ISFLUSH
)
199 KKASSERT(head
->flags
& HAMMER2_TRANS_PREFLUSH
);
200 trans
->flags
|= HAMMER2_TRANS_PREFLUSH
;
201 TAILQ_INSERT_AFTER(&tman
->transq
, head
, trans
, entry
);
202 trans
->sync_xid
= head
->sync_xid
;
203 trans
->modify_tid
= head
->modify_tid
;
204 trans
->flags
|= HAMMER2_TRANS_CONCURRENT
;
205 /* not allowed to block */
208 * A normal transaction is requested while a flush is in
209 * progress. We insert after the current flush and may
212 * WARNING! Also see hammer2_chain_setflush()
214 TAILQ_FOREACH(head
, &tman
->transq
, entry
) {
215 if (head
->flags
& HAMMER2_TRANS_ISFLUSH
)
219 TAILQ_INSERT_AFTER(&tman
->transq
, head
, trans
, entry
);
220 trans
->sync_xid
= head
->sync_xid
+ 1;
221 trans
->flags
|= HAMMER2_TRANS_CONCURRENT
;
224 * XXX for now we must block new transactions, synchronous
225 * flush mode is on by default.
227 * If synchronous flush mode is enabled concurrent
228 * frontend transactions during the flush are not
229 * allowed (except we don't have a choice for buffer
232 if (hammer2_synchronous_flush
> 0 ||
233 TAILQ_FIRST(&tman
->transq
) != head
) {
235 while (trans
->blocked
) {
236 lksleep(&trans
->sync_xid
, &tman
->translk
,
241 if (flags
& HAMMER2_TRANS_NEWINODE
) {
244 * Super-root transaction, all new inodes have an
245 * inode number of 1. Normal pfs inode cache
246 * semantics are not used.
248 trans
->inode_tid
= 1;
253 if (pmp
->inode_tid
< HAMMER2_INODE_START
)
254 pmp
->inode_tid
= HAMMER2_INODE_START
;
255 trans
->inode_tid
= pmp
->inode_tid
++;
259 lockmgr(&tman
->translk
, LK_RELEASE
);
263 hammer2_trans_assert_strategy(hammer2_pfs_t
*pmp
)
265 hammer2_trans_manage_t
*tman
;
266 hammer2_trans_t
*head
;
268 tman
= &pmp
->tmanage
;
269 lockmgr(&tman
->translk
, LK_EXCLUSIVE
);
270 if (tman
->flushcnt
) {
271 TAILQ_FOREACH(head
, &tman
->transq
, entry
) {
272 if (head
->flags
& HAMMER2_TRANS_ISFLUSH
)
276 KKASSERT(head
->flags
& HAMMER2_TRANS_PREFLUSH
);
278 lockmgr(&tman
->translk
, LK_RELEASE
);
282 hammer2_trans_done(hammer2_trans_t
*trans
)
284 hammer2_trans_manage_t
*tman
;
285 hammer2_trans_t
*head
;
286 hammer2_trans_t
*scan
;
288 tman
= &trans
->pmp
->tmanage
;
293 lockmgr(&tman
->translk
, LK_EXCLUSIVE
);
294 TAILQ_REMOVE(&tman
->transq
, trans
, entry
);
295 head
= TAILQ_FIRST(&tman
->transq
);
298 * Adjust flushcnt if this was a flush, clear TRANS_CONCURRENT
299 * up through the next flush. (If the head is a flush then we
300 * stop there, unlike the unblock code following this section).
302 if (trans
->flags
& HAMMER2_TRANS_ISFLUSH
) {
305 while (scan
&& (scan
->flags
& HAMMER2_TRANS_ISFLUSH
) == 0) {
306 atomic_clear_int(&scan
->flags
,
307 HAMMER2_TRANS_CONCURRENT
);
308 scan
= TAILQ_NEXT(scan
, entry
);
313 * Unblock the head of the queue and any additional transactions
314 * up to the next flush. The head can be a flush and it will be
315 * unblocked along with the non-flush transactions following it
316 * (which are allowed to run concurrently with it).
318 * In synchronous flush mode we stop if the head transaction is
321 if (head
&& head
->blocked
) {
323 wakeup(&head
->sync_xid
);
325 if (hammer2_synchronous_flush
> 0)
328 scan
= TAILQ_NEXT(head
, entry
);
329 while (scan
&& (scan
->flags
& HAMMER2_TRANS_ISFLUSH
) == 0) {
332 wakeup(&scan
->sync_xid
);
334 scan
= TAILQ_NEXT(scan
, entry
);
337 lockmgr(&tman
->translk
, LK_RELEASE
);
341 * Chains undergoing destruction are removed from the in-memory topology.
342 * To avoid getting lost these chains are placed on the delayed flush
343 * queue which will properly dispose of them.
345 * We do this instead of issuing an immediate flush in order to give
346 * recursive deletions (rm -rf, etc) a chance to remove more of the
347 * hierarchy, potentially allowing an enormous amount of write I/O to
351 hammer2_delayed_flush(hammer2_trans_t
*trans
, hammer2_chain_t
*chain
)
353 if ((chain
->flags
& HAMMER2_CHAIN_DELAYED
) == 0) {
354 hammer2_spin_ex(&chain
->hmp
->list_spin
);
355 if ((chain
->flags
& (HAMMER2_CHAIN_DELAYED
|
356 HAMMER2_CHAIN_DEFERRED
)) == 0) {
357 atomic_set_int(&chain
->flags
, HAMMER2_CHAIN_DELAYED
|
358 HAMMER2_CHAIN_DEFERRED
);
359 TAILQ_INSERT_TAIL(&chain
->hmp
->flushq
,
361 hammer2_chain_ref(chain
);
363 hammer2_spin_unex(&chain
->hmp
->list_spin
);
368 * Flush the chain and all modified sub-chains through the specified
369 * synchronization point, propagating parent chain modifications, modify_tid,
370 * and mirror_tid updates back up as needed.
372 * Caller must have interlocked against any non-flush-related modifying
373 * operations in progress whos XXX values are less than or equal
374 * to the passed sync_xid.
376 * Caller must have already vetted synchronization points to ensure they
377 * are properly flushed. Only snapshots and cluster flushes can create
378 * these sorts of synchronization points.
380 * This routine can be called from several places but the most important
383 * chain is locked on call and will remain locked on return. The chain's
384 * UPDATE flag indicates that its parent's block table (which is not yet
385 * part of the flush) should be updated. The chain may be replaced by
386 * the call if it was modified.
389 hammer2_flush(hammer2_trans_t
*trans
, hammer2_chain_t
*chain
, int istop
)
391 hammer2_chain_t
*scan
;
392 hammer2_flush_info_t info
;
397 * Execute the recursive flush and handle deferrals.
399 * Chains can be ridiculously long (thousands deep), so to
400 * avoid blowing out the kernel stack the recursive flush has a
401 * depth limit. Elements at the limit are placed on a list
402 * for re-execution after the stack has been popped.
404 bzero(&info
, sizeof(info
));
405 TAILQ_INIT(&info
.flushq
);
407 info
.sync_xid
= trans
->sync_xid
;
408 info
.cache_index
= -1;
411 * Calculate parent (can be NULL), if not NULL the flush core
412 * expects the parent to be referenced so it can easily lock/unlock
413 * it without it getting ripped up.
415 if ((info
.parent
= chain
->parent
) != NULL
)
416 hammer2_chain_ref(info
.parent
);
419 * Extra ref needed because flush_core expects it when replacing
422 hammer2_chain_ref(chain
);
428 * Move hmp->flushq to info.flushq if non-empty so it can
431 if (TAILQ_FIRST(&hmp
->flushq
) != NULL
) {
432 hammer2_spin_ex(&chain
->hmp
->list_spin
);
433 TAILQ_CONCAT(&info
.flushq
, &hmp
->flushq
, flush_node
);
434 hammer2_spin_unex(&chain
->hmp
->list_spin
);
438 * Unwind deep recursions which had been deferred. This
439 * can leave the FLUSH_* bits set for these chains, which
440 * will be handled when we [re]flush chain after the unwind.
442 while ((scan
= TAILQ_FIRST(&info
.flushq
)) != NULL
) {
443 KKASSERT(scan
->flags
& HAMMER2_CHAIN_DEFERRED
);
444 TAILQ_REMOVE(&info
.flushq
, scan
, flush_node
);
445 atomic_clear_int(&scan
->flags
, HAMMER2_CHAIN_DEFERRED
|
446 HAMMER2_CHAIN_DELAYED
);
449 * Now that we've popped back up we can do a secondary
450 * recursion on the deferred elements.
452 * NOTE: hammer2_flush() may replace scan.
454 if (hammer2_debug
& 0x0040)
455 kprintf("deferred flush %p\n", scan
);
456 hammer2_chain_lock(scan
, HAMMER2_RESOLVE_MAYBE
);
457 hammer2_flush(trans
, scan
, 0);
458 hammer2_chain_unlock(scan
);
459 hammer2_chain_drop(scan
); /* ref from deferral */
465 info
.diddeferral
= 0;
466 hammer2_flush_core(&info
, chain
, istop
);
469 * Only loop if deep recursions have been deferred.
471 if (TAILQ_EMPTY(&info
.flushq
))
474 if (++loops
% 1000 == 0) {
475 kprintf("hammer2_flush: excessive loops on %p\n",
477 if (hammer2_debug
& 0x100000)
481 hammer2_chain_drop(chain
);
483 hammer2_chain_drop(info
.parent
);
487 * This is the core of the chain flushing code. The chain is locked by the
488 * caller and must also have an extra ref on it by the caller, and remains
489 * locked and will have an extra ref on return. Upon return, the caller can
490 * test the UPDATE bit on the child to determine if the parent needs updating.
492 * (1) Determine if this node is a candidate for the flush, return if it is
493 * not. fchain and vchain are always candidates for the flush.
495 * (2) If we recurse too deep the chain is entered onto the deferral list and
496 * the current flush stack is aborted until after the deferral list is
499 * (3) Recursively flush live children (rbtree). This can create deferrals.
500 * A successful flush clears the MODIFIED and UPDATE bits on the children
501 * and typically causes the parent to be marked MODIFIED as the children
502 * update the parent's block table. A parent might already be marked
503 * MODIFIED due to a deletion (whos blocktable update in the parent is
504 * handled by the frontend), or if the parent itself is modified by the
505 * frontend for other reasons.
507 * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
508 * Deleted-but-open inodes can still be individually flushed via the
511 * (5) Note that an unmodified child may still need the block table in its
512 * parent updated (e.g. rename/move). The child will have UPDATE set
515 * WARNING ON BREF MODIFY_TID/MIRROR_TID
517 * blockref.modify_tid is consistent only within a PFS, and will not be
518 * consistent during synchronization. mirror_tid is consistent across the
519 * block device regardless of the PFS.
522 hammer2_flush_core(hammer2_flush_info_t
*info
, hammer2_chain_t
*chain
,
525 hammer2_chain_t
*parent
;
530 * (1) Optimize downward recursion to locate nodes needing action.
531 * Nothing to do if none of these flags are set.
533 if ((chain
->flags
& HAMMER2_CHAIN_FLUSH_MASK
) == 0) {
534 if (hammer2_debug
& 0x200) {
535 if (info
->debug
== NULL
)
543 diddeferral
= info
->diddeferral
;
544 parent
= info
->parent
; /* can be NULL */
547 * Downward search recursion
549 if (chain
->flags
& (HAMMER2_CHAIN_DEFERRED
| HAMMER2_CHAIN_DELAYED
)) {
554 } else if (info
->depth
== HAMMER2_FLUSH_DEPTH_LIMIT
) {
556 * Recursion depth reached.
558 KKASSERT((chain
->flags
& HAMMER2_CHAIN_DELAYED
) == 0);
559 hammer2_chain_ref(chain
);
560 TAILQ_INSERT_TAIL(&info
->flushq
, chain
, flush_node
);
561 atomic_set_int(&chain
->flags
, HAMMER2_CHAIN_DEFERRED
);
563 } else if ((chain
->flags
& HAMMER2_CHAIN_PFSBOUNDARY
) && istop
== 0) {
565 * We do not recurse through PFSROOTs. PFSROOT flushes are
566 * handled by the related pmp's (whether mounted or not,
567 * including during recovery).
569 * But we must still process the PFSROOT chains for block
570 * table updates in their parent (which IS part of our flush).
572 * Note that the volume root, vchain, does not set this flag.
575 } else if (chain
->flags
& HAMMER2_CHAIN_ONFLUSH
) {
577 * Downward recursion search (actual flush occurs bottom-up).
578 * pre-clear ONFLUSH. It can get set again due to races,
579 * which we want so the scan finds us again in the next flush.
580 * These races can also include
582 * Flush recursions stop at PFSROOT boundaries. Each PFS
583 * must be individually flushed and then the root must
586 atomic_clear_int(&chain
->flags
, HAMMER2_CHAIN_ONFLUSH
);
587 info
->parent
= chain
;
588 hammer2_spin_ex(&chain
->core
.spin
);
589 RB_SCAN(hammer2_chain_tree
, &chain
->core
.rbtree
,
590 NULL
, hammer2_flush_recurse
, info
);
591 hammer2_spin_unex(&chain
->core
.spin
);
592 info
->parent
= parent
;
593 if (info
->diddeferral
)
594 hammer2_chain_setflush(info
->trans
, chain
);
598 * Now we are in the bottom-up part of the recursion.
600 * Do not update chain if lower layers were deferred.
602 if (info
->diddeferral
)
606 * Propagate the DESTROY flag downwards. This dummies up the flush
607 * code and tries to invalidate related buffer cache buffers to
608 * avoid the disk write.
610 if (parent
&& (parent
->flags
& HAMMER2_CHAIN_DESTROY
))
611 atomic_set_int(&chain
->flags
, HAMMER2_CHAIN_DESTROY
);
614 * Chain was already modified or has become modified, flush it out.
617 if ((hammer2_debug
& 0x200) &&
619 (chain
->flags
& (HAMMER2_CHAIN_MODIFIED
| HAMMER2_CHAIN_UPDATE
))) {
620 hammer2_chain_t
*scan
= chain
;
622 kprintf("DISCONNECTED FLUSH %p->%p\n", info
->debug
, chain
);
624 kprintf(" chain %p [%08x] bref=%016jx:%02x\n",
626 scan
->bref
.key
, scan
->bref
.type
);
627 if (scan
== info
->debug
)
633 if (chain
->flags
& HAMMER2_CHAIN_MODIFIED
) {
635 * Dispose of the modified bit.
637 * UPDATE should already be set.
638 * bref.mirror_tid should already be set.
640 KKASSERT((chain
->flags
& HAMMER2_CHAIN_UPDATE
) ||
641 chain
== &hmp
->vchain
);
642 atomic_clear_int(&chain
->flags
, HAMMER2_CHAIN_MODIFIED
);
645 * Manage threads waiting for excessive dirty memory to
649 hammer2_pfs_memory_wakeup(chain
->pmp
);
651 if ((chain
->flags
& HAMMER2_CHAIN_UPDATE
) ||
652 chain
== &hmp
->vchain
||
653 chain
== &hmp
->fchain
) {
655 * Drop the ref from the MODIFIED bit we cleared,
658 hammer2_chain_drop(chain
);
661 * Drop the ref from the MODIFIED bit we cleared and
662 * set a ref for the UPDATE bit we are setting. Net
665 atomic_set_int(&chain
->flags
, HAMMER2_CHAIN_UPDATE
);
669 * Issue the flush. This is indirect via the DIO.
671 * NOTE: A DELETED node that reaches this point must be
672 * flushed for synchronization point consistency.
674 * NOTE: Even though MODIFIED was already set, the related DIO
675 * might not be dirty due to a system buffer cache
676 * flush and must be set dirty if we are going to make
677 * further modifications to the buffer. Chains with
678 * embedded data don't need this.
680 if (hammer2_debug
& 0x1000) {
681 kprintf("Flush %p.%d %016jx/%d sync_xid=%08x "
683 chain
, chain
->bref
.type
,
684 chain
->bref
.key
, chain
->bref
.keybits
,
686 chain
->bref
.data_off
);
688 if (hammer2_debug
& 0x2000) {
689 Debugger("Flush hell");
693 * Update chain CRCs for flush.
695 * NOTE: Volume headers are NOT flushed here as they require
696 * special processing.
698 switch(chain
->bref
.type
) {
699 case HAMMER2_BREF_TYPE_FREEMAP
:
701 * Update the volume header's freemap_tid to the
702 * freemap's flushing mirror_tid.
704 * (note: embedded data, do not call setdirty)
706 KKASSERT(hmp
->vchain
.flags
& HAMMER2_CHAIN_MODIFIED
);
707 KKASSERT(chain
== &hmp
->fchain
);
708 hmp
->voldata
.freemap_tid
= chain
->bref
.mirror_tid
;
709 kprintf("sync freemap mirror_tid %08jx\n",
710 (intmax_t)chain
->bref
.mirror_tid
);
713 * The freemap can be flushed independently of the
714 * main topology, but for the case where it is
715 * flushed in the same transaction, and flushed
716 * before vchain (a case we want to allow for
717 * performance reasons), make sure modifications
718 * made during the flush under vchain use a new
721 * Otherwise the mount recovery code will get confused.
723 ++hmp
->voldata
.mirror_tid
;
725 case HAMMER2_BREF_TYPE_VOLUME
:
727 * The free block table is flushed by
728 * hammer2_vfs_sync() before it flushes vchain.
729 * We must still hold fchain locked while copying
730 * voldata to volsync, however.
732 * (note: embedded data, do not call setdirty)
734 hammer2_chain_lock(&hmp
->fchain
,
735 HAMMER2_RESOLVE_ALWAYS
);
736 hammer2_voldata_lock(hmp
);
737 kprintf("sync volume mirror_tid %08jx\n",
738 (intmax_t)chain
->bref
.mirror_tid
);
741 * Update the volume header's mirror_tid to the
742 * main topology's flushing mirror_tid. It is
743 * possible that voldata.mirror_tid is already
744 * beyond bref.mirror_tid due to the bump we made
745 * above in BREF_TYPE_FREEMAP.
747 if (hmp
->voldata
.mirror_tid
< chain
->bref
.mirror_tid
) {
748 hmp
->voldata
.mirror_tid
=
749 chain
->bref
.mirror_tid
;
753 * The volume header is flushed manually by the
754 * syncer, not here. All we do here is adjust the
757 KKASSERT(chain
->data
!= NULL
);
758 KKASSERT(chain
->dio
== NULL
);
760 hmp
->voldata
.icrc_sects
[HAMMER2_VOL_ICRC_SECT1
]=
762 (char *)&hmp
->voldata
+
763 HAMMER2_VOLUME_ICRC1_OFF
,
764 HAMMER2_VOLUME_ICRC1_SIZE
);
765 hmp
->voldata
.icrc_sects
[HAMMER2_VOL_ICRC_SECT0
]=
767 (char *)&hmp
->voldata
+
768 HAMMER2_VOLUME_ICRC0_OFF
,
769 HAMMER2_VOLUME_ICRC0_SIZE
);
770 hmp
->voldata
.icrc_volheader
=
772 (char *)&hmp
->voldata
+
773 HAMMER2_VOLUME_ICRCVH_OFF
,
774 HAMMER2_VOLUME_ICRCVH_SIZE
);
776 kprintf("syncvolhdr %016jx %016jx\n",
777 hmp
->voldata
.mirror_tid
,
778 hmp
->vchain
.bref
.mirror_tid
);
779 hmp
->volsync
= hmp
->voldata
;
780 atomic_set_int(&chain
->flags
, HAMMER2_CHAIN_VOLUMESYNC
);
781 hammer2_voldata_unlock(hmp
);
782 hammer2_chain_unlock(&hmp
->fchain
);
784 case HAMMER2_BREF_TYPE_DATA
:
786 * Data elements have already been flushed via the
787 * logical file buffer cache. Their hash was set in
788 * the bref by the vop_write code. Do not re-dirty.
790 * Make sure any device buffer(s) have been flushed
791 * out here (there aren't usually any to flush) XXX.
794 case HAMMER2_BREF_TYPE_INDIRECT
:
795 case HAMMER2_BREF_TYPE_FREEMAP_NODE
:
796 case HAMMER2_BREF_TYPE_FREEMAP_LEAF
:
798 * Buffer I/O will be cleaned up when the volume is
799 * flushed (but the kernel is free to flush it before
802 KKASSERT((chain
->flags
& HAMMER2_CHAIN_EMBEDDED
) == 0);
803 hammer2_chain_setcheck(chain
, chain
->data
);
805 case HAMMER2_BREF_TYPE_INODE
:
807 * NOTE: We must call io_setdirty() to make any late
808 * changes to the inode data, the system might
809 * have already flushed the buffer.
811 if (chain
->data
->ipdata
.op_flags
&
812 HAMMER2_OPFLAG_PFSROOT
) {
814 * non-NULL pmp if mounted as a PFS. We must
815 * sync fields cached in the pmp? XXX
817 hammer2_inode_data_t
*ipdata
;
819 hammer2_io_setdirty(chain
->dio
);
820 ipdata
= &chain
->data
->ipdata
;
823 chain
->pmp
->inode_tid
;
826 /* can't be mounted as a PFS */
829 KKASSERT((chain
->flags
& HAMMER2_CHAIN_EMBEDDED
) == 0);
830 hammer2_chain_setcheck(chain
, chain
->data
);
833 KKASSERT(chain
->flags
& HAMMER2_CHAIN_EMBEDDED
);
834 panic("hammer2_flush_core: unsupported "
841 * If the chain was destroyed try to avoid unnecessary I/O.
842 * (this only really works if the DIO system buffer is the
843 * same size as chain->bytes).
845 if ((chain
->flags
& HAMMER2_CHAIN_DESTROY
) && chain
->dio
) {
846 hammer2_io_setinval(chain
->dio
, chain
->bytes
);
851 * If UPDATE is set the parent block table may need to be updated.
853 * NOTE: UPDATE may be set on vchain or fchain in which case
854 * parent could be NULL. It's easiest to allow the case
855 * and test for NULL. parent can also wind up being NULL
856 * due to a deletion so we need to handle the case anyway.
858 * If no parent exists we can just clear the UPDATE bit. If the
859 * chain gets reattached later on the bit will simply get set
862 if ((chain
->flags
& HAMMER2_CHAIN_UPDATE
) && parent
== NULL
) {
863 atomic_clear_int(&chain
->flags
, HAMMER2_CHAIN_UPDATE
);
864 hammer2_chain_drop(chain
);
868 * The chain may need its blockrefs updated in the parent. This
869 * requires some fancy footwork.
871 if (chain
->flags
& HAMMER2_CHAIN_UPDATE
) {
872 hammer2_blockref_t
*base
;
876 * Both parent and chain must be locked. This requires
877 * temporarily unlocking the chain. We have to deal with
878 * the case where the chain might be reparented or modified
879 * while it was unlocked.
881 hammer2_chain_unlock(chain
);
882 hammer2_chain_lock(parent
, HAMMER2_RESOLVE_ALWAYS
);
883 hammer2_chain_lock(chain
, HAMMER2_RESOLVE_MAYBE
);
884 if (chain
->parent
!= parent
) {
885 kprintf("PARENT MISMATCH ch=%p p=%p/%p\n",
886 chain
, chain
->parent
, parent
);
887 hammer2_chain_unlock(parent
);
892 * Check race condition. If someone got in and modified
893 * it again while it was unlocked, we have to loop up.
895 if (chain
->flags
& HAMMER2_CHAIN_MODIFIED
) {
896 hammer2_chain_unlock(parent
);
897 kprintf("hammer2_flush: chain %p flush-mod race\n",
903 * Clear UPDATE flag, mark parent modified, update its
904 * modify_tid if necessary, and adjust the parent blockmap.
906 if (chain
->flags
& HAMMER2_CHAIN_UPDATE
) {
907 atomic_clear_int(&chain
->flags
, HAMMER2_CHAIN_UPDATE
);
908 hammer2_chain_drop(chain
);
914 * Avoid actually modifying and updating the parent if it
915 * was flagged for destruction. This can greatly reduce
916 * disk I/O in large tree removals because the
917 * hammer2_io_setinval() call in the upward recursion
918 * (see MODIFIED code above) can only handle a few cases.
920 if (parent
->flags
& HAMMER2_CHAIN_DESTROY
) {
921 if (parent
->bref
.modify_tid
< chain
->bref
.modify_tid
) {
922 parent
->bref
.modify_tid
=
923 chain
->bref
.modify_tid
;
925 atomic_clear_int(&chain
->flags
, HAMMER2_CHAIN_BMAPPED
|
926 HAMMER2_CHAIN_BMAPUPD
);
927 hammer2_chain_unlock(parent
);
932 * We are updating the parent's blockmap, the parent must
935 hammer2_chain_modify(info
->trans
, parent
, 0);
936 if (parent
->bref
.modify_tid
< chain
->bref
.modify_tid
)
937 parent
->bref
.modify_tid
= chain
->bref
.modify_tid
;
940 * Calculate blockmap pointer
942 switch(parent
->bref
.type
) {
943 case HAMMER2_BREF_TYPE_INODE
:
945 * Access the inode's block array. However, there is
946 * no block array if the inode is flagged DIRECTDATA.
949 (parent
->data
->ipdata
.op_flags
&
950 HAMMER2_OPFLAG_DIRECTDATA
) == 0) {
951 base
= &parent
->data
->
952 ipdata
.u
.blockset
.blockref
[0];
956 count
= HAMMER2_SET_COUNT
;
958 case HAMMER2_BREF_TYPE_INDIRECT
:
959 case HAMMER2_BREF_TYPE_FREEMAP_NODE
:
961 base
= &parent
->data
->npdata
[0];
964 count
= parent
->bytes
/ sizeof(hammer2_blockref_t
);
966 case HAMMER2_BREF_TYPE_VOLUME
:
967 base
= &chain
->hmp
->voldata
.sroot_blockset
.blockref
[0];
968 count
= HAMMER2_SET_COUNT
;
970 case HAMMER2_BREF_TYPE_FREEMAP
:
971 base
= &parent
->data
->npdata
[0];
972 count
= HAMMER2_SET_COUNT
;
977 panic("hammer2_flush_core: "
978 "unrecognized blockref type: %d",
985 * We synchronize pending statistics at this time. Delta
986 * adjustments designated for the current and upper level
989 if (base
&& (chain
->flags
& HAMMER2_CHAIN_BMAPUPD
)) {
990 if (chain
->flags
& HAMMER2_CHAIN_BMAPPED
) {
991 hammer2_spin_ex(&parent
->core
.spin
);
992 hammer2_base_delete(info
->trans
, parent
,
994 &info
->cache_index
, chain
);
995 hammer2_spin_unex(&parent
->core
.spin
);
996 /* base_delete clears both bits */
998 atomic_clear_int(&chain
->flags
,
999 HAMMER2_CHAIN_BMAPUPD
);
1002 if (base
&& (chain
->flags
& HAMMER2_CHAIN_BMAPPED
) == 0) {
1003 hammer2_spin_ex(&parent
->core
.spin
);
1004 hammer2_base_insert(info
->trans
, parent
,
1006 &info
->cache_index
, chain
);
1007 hammer2_spin_unex(&parent
->core
.spin
);
1008 /* base_insert sets BMAPPED */
1010 hammer2_chain_unlock(parent
);
1016 * Final cleanup after flush
1019 KKASSERT(chain
->refs
> 0);
1020 if (hammer2_debug
& 0x200) {
1021 if (info
->debug
== chain
)
1027 * Flush recursion helper, called from flush_core, calls flush_core.
1029 * Flushes the children of the caller's chain (info->parent), restricted
1030 * by sync_tid. Set info->domodify if the child's blockref must propagate
1031 * back up to the parent.
1033 * Ripouts can move child from rbtree to dbtree or dbq but the caller's
1034 * flush scan order prevents any chains from being lost. A child can be
1035 * executes more than once.
1037 * WARNING! If we do not call hammer2_flush_core() we must update
1038 * bref.mirror_tid ourselves to indicate that the flush has
1039 * processed the child.
1041 * WARNING! parent->core spinlock is held on entry and return.
1044 hammer2_flush_recurse(hammer2_chain_t
*child
, void *data
)
1046 hammer2_flush_info_t
*info
= data
;
1047 /*hammer2_trans_t *trans = info->trans;*/
1048 hammer2_chain_t
*parent
= info
->parent
;
1051 * (child can never be fchain or vchain so a special check isn't
1054 * We must ref the child before unlocking the spinlock.
1056 * The caller has added a ref to the parent so we can temporarily
1057 * unlock it in order to lock the child.
1059 hammer2_chain_ref(child
);
1060 hammer2_spin_unex(&parent
->core
.spin
);
1062 hammer2_chain_unlock(parent
);
1063 hammer2_chain_lock(child
, HAMMER2_RESOLVE_MAYBE
);
1066 * Recurse and collect deferral data. We're in the media flush,
1067 * this can cross PFS boundaries.
1069 if (child
->flags
& HAMMER2_CHAIN_FLUSH_MASK
) {
1071 hammer2_flush_core(info
, child
, 0);
1073 } else if (hammer2_debug
& 0x200) {
1074 if (info
->debug
== NULL
)
1075 info
->debug
= child
;
1077 hammer2_flush_core(info
, child
, 0);
1079 if (info
->debug
== child
)
1084 * Relock to continue the loop
1086 hammer2_chain_unlock(child
);
1087 hammer2_chain_lock(parent
, HAMMER2_RESOLVE_MAYBE
);
1088 hammer2_chain_drop(child
);
1089 KKASSERT(info
->parent
== parent
);
1090 hammer2_spin_ex(&parent
->core
.spin
);