2 * Copyright (c) 2013-2019 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/fcntl.h>
40 #include <sys/mount.h>
41 #include <sys/vnode.h>
42 #include <sys/mountctl.h>
43 #include <vm/vm_kern.h>
44 #include <vm/vm_extern.h>
49 * breadth-first search
51 typedef struct hammer2_chain_save
{
52 TAILQ_ENTRY(hammer2_chain_save
) entry
;
53 hammer2_chain_t
*chain
;
55 } hammer2_chain_save_t
;
57 TAILQ_HEAD(hammer2_chain_save_list
, hammer2_chain_save
);
58 typedef struct hammer2_chain_save_list hammer2_chain_save_list_t
;
60 typedef struct hammer2_bulkfree_info
{
63 hammer2_off_t sbase
; /* sub-loop iteration */
65 hammer2_bmap_data_t
*bmap
;
67 long count_10_00
; /* staged->free */
68 long count_11_10
; /* allocated->staged */
69 long count_00_11
; /* (should not happen) */
70 long count_01_11
; /* (should not happen) */
71 long count_10_11
; /* staged->allocated */
73 long count_linadjusts
;
74 long count_inodes_scanned
;
75 long count_dirents_scanned
;
76 long count_dedup_factor
;
77 long count_bytes_scanned
;
78 long count_chains_scanned
;
79 long count_chains_reported
;
82 hammer2_off_t adj_free
;
84 hammer2_tid_t saved_mirror_tid
;
86 hammer2_chain_save_list_t list
;
87 hammer2_dedup_t
*dedup
;
89 } hammer2_bulkfree_info_t
;
91 static int h2_bulkfree_test(hammer2_bulkfree_info_t
*info
,
92 hammer2_blockref_t
*bref
, int pri
, int saved_error
);
95 * General bulk scan function with callback. Called with a referenced
96 * but UNLOCKED parent. The parent is returned in the same state.
100 hammer2_bulk_scan(hammer2_chain_t
*parent
,
101 int (*func
)(hammer2_bulkfree_info_t
*info
,
102 hammer2_blockref_t
*bref
),
103 hammer2_bulkfree_info_t
*info
)
105 hammer2_blockref_t bref
;
106 hammer2_chain_t
*chain
;
107 hammer2_chain_save_t
*tail
;
108 hammer2_chain_save_t
*save
;
120 hammer2_chain_lock(parent
, HAMMER2_RESOLVE_ALWAYS
|
121 HAMMER2_RESOLVE_SHARED
);
122 tail
= TAILQ_LAST(&info
->list
, hammer2_chain_save_list
);
125 * The parent was previously retrieved NODATA and thus has not
126 * tested the CRC. Now that we have locked it normally, check
127 * for a CRC problem and skip it if we found one. The bulk scan
128 * cannot safely traverse invalid block tables (we could end up
129 * in an endless loop or cause a panic).
131 if (parent
->error
& HAMMER2_ERROR_CHECK
) {
132 error
= parent
->error
;
137 * Report which PFS is being scanned
139 if (parent
->bref
.type
== HAMMER2_BREF_TYPE_INODE
&&
140 (parent
->bref
.flags
& HAMMER2_BREF_FLAG_PFSROOT
)) {
141 kprintf("hammer2_bulkfree: Scanning %s\n",
142 parent
->data
->ipdata
.filename
);
146 * Generally loop on the contents if we have not been flagged
149 * Remember that these chains are completely isolated from
150 * the frontend, so we can release locks temporarily without
154 error
|= hammer2_chain_scan(parent
, &chain
, &bref
, &first
,
155 HAMMER2_LOOKUP_NODATA
|
156 HAMMER2_LOOKUP_SHARED
);
159 * Handle EOF or other error at current level. This stops
162 if (error
& ~HAMMER2_ERROR_CHECK
)
166 * Account for dirents before thre data_off test, since most
167 * dirents do not need a data reference.
169 if (bref
.type
== HAMMER2_BREF_TYPE_DIRENT
)
170 ++info
->count_dirents_scanned
;
173 * Ignore brefs without data (typically dirents)
175 if ((bref
.data_off
& ~HAMMER2_OFF_MASK_RADIX
) == 0)
179 * Process bref, chain is only non-NULL if the bref
180 * might be recursable (its possible that we sometimes get
181 * a non-NULL chain where the bref cannot be recursed).
183 * If we already ran down this tree we do not have to do it
184 * again, but we must still recover any cumulative error
185 * recorded from the time we did.
188 e2
= h2_bulkfree_test(info
, &bref
, 1, 0);
190 error
|= e2
& ~HAMMER2_ERROR_EOF
;
194 if (bref
.type
== HAMMER2_BREF_TYPE_INODE
)
195 ++info
->count_inodes_scanned
;
197 error
|= func(info
, &bref
);
198 if (error
& ~HAMMER2_ERROR_CHECK
)
202 * A non-null chain is always returned if it is
203 * recursive, otherwise a non-null chain might be
204 * returned but usually is not when not recursive.
210 info
->count_bytes_scanned
+= chain
->bytes
;
211 ++info
->count_chains_scanned
;
213 if (info
->count_chains_scanned
>=
214 info
->count_chains_reported
+ 1000000 ||
215 (info
->count_chains_scanned
< 1000000 &&
216 info
->count_chains_scanned
>=
217 info
->count_chains_reported
+ 100000)) {
218 kprintf(" chains %-7ld inodes %-7ld "
219 "dirents %-7ld bytes %5ldMB\n",
220 info
->count_chains_scanned
,
221 info
->count_inodes_scanned
,
222 info
->count_dirents_scanned
,
223 info
->count_bytes_scanned
/ 1000000);
224 info
->count_chains_reported
=
225 info
->count_chains_scanned
;
230 * Else check type and setup depth-first scan.
232 * Account for bytes actually read.
234 switch(chain
->bref
.type
) {
235 case HAMMER2_BREF_TYPE_INODE
:
236 case HAMMER2_BREF_TYPE_FREEMAP_NODE
:
237 case HAMMER2_BREF_TYPE_INDIRECT
:
238 case HAMMER2_BREF_TYPE_VOLUME
:
239 case HAMMER2_BREF_TYPE_FREEMAP
:
241 if (chain
->error
& HAMMER2_ERROR_CHECK
) {
243 * Cannot safely recurse chains with crc
244 * errors, even in emergency mode.
247 } else if (info
->depth
> 16) {
248 save
= kmalloc(sizeof(*save
), M_HAMMER2
,
251 hammer2_chain_ref(chain
);
252 TAILQ_INSERT_TAIL(&info
->list
, save
, entry
);
257 int savepri
= info
->pri
;
259 hammer2_chain_unlock(chain
);
260 hammer2_chain_unlock(parent
);
262 rup_error
|= hammer2_bulk_scan(chain
,
264 info
->pri
+= savepri
;
265 hammer2_chain_lock(parent
,
266 HAMMER2_RESOLVE_ALWAYS
|
267 HAMMER2_RESOLVE_SHARED
);
268 hammer2_chain_lock(chain
,
269 HAMMER2_RESOLVE_ALWAYS
|
270 HAMMER2_RESOLVE_SHARED
);
274 case HAMMER2_BREF_TYPE_DATA
:
277 /* does not recurse */
280 if (rup_error
& HAMMER2_ERROR_ABORTED
)
284 hammer2_chain_unlock(chain
);
285 hammer2_chain_drop(chain
);
289 * If this is a PFSROOT, also re-run any defered elements
290 * added during our scan so we can report any cumulative errors
293 if (parent
->bref
.type
== HAMMER2_BREF_TYPE_INODE
&&
294 (parent
->bref
.flags
& HAMMER2_BREF_FLAG_PFSROOT
)) {
298 save
= tail
? TAILQ_NEXT(tail
, entry
) :
299 TAILQ_FIRST(&info
->list
);
303 TAILQ_REMOVE(&info
->list
, save
, entry
);
306 rup_error
|= hammer2_bulk_scan(save
->chain
, func
, info
);
307 hammer2_chain_drop(save
->chain
);
308 kfree(save
, M_HAMMER2
);
316 * Report which PFS the errors were encountered in.
318 if (parent
->bref
.type
== HAMMER2_BREF_TYPE_INODE
&&
319 (parent
->bref
.flags
& HAMMER2_BREF_FLAG_PFSROOT
) &&
320 (error
& ~HAMMER2_ERROR_EOF
)) {
321 kprintf("hammer2_bulkfree: Encountered errors (%08x) "
322 "while scanning \"%s\"\n",
323 error
, parent
->data
->ipdata
.filename
);
327 * Save with higher pri now that we know what it is.
329 h2_bulkfree_test(info
, &parent
->bref
, info
->pri
+ 1,
330 (error
& ~HAMMER2_ERROR_EOF
));
333 hammer2_chain_unlock(parent
);
335 return (error
& ~HAMMER2_ERROR_EOF
);
342 * Chain flush (partial synchronization) XXX removed
343 * Scan the whole topology - build in-memory freemap (mark 11)
344 * Reconcile the in-memory freemap against the on-disk freemap.
345 * ondisk xx -> ondisk 11 (if allocated)
346 * ondisk 11 -> ondisk 10 (if free in-memory)
347 * ondisk 10 -> ondisk 00 (if free in-memory) - on next pass
350 * The topology scan may have to be performed multiple times to window
351 * freemaps which are too large to fit in kernel memory.
353 * Races are handled using a double-transition (11->10, 10->00). The bulkfree
354 * scan snapshots the volume root's blockset and thus can run concurrent with
355 * normal operations, as long as a full flush is made between each pass to
356 * synchronize any modified chains (otherwise their blocks might be improperly
359 * Temporary memory in multiples of 64KB is required to reconstruct the leaf
360 * hammer2_bmap_data blocks so they can later be compared against the live
361 * freemap. Each 64KB block represents 128 x 16KB x 1024 = ~2 GB of storage.
362 * A 32MB save area thus represents around ~1 TB. The temporary memory
363 * allocated can be specified. If it is not sufficient multiple topology
364 * passes will be made.
368 * Bulkfree callback info
370 static void hammer2_bulkfree_thread(void *arg __unused
);
371 static void cbinfo_bmap_init(hammer2_bulkfree_info_t
*cbinfo
, size_t size
);
372 static int h2_bulkfree_callback(hammer2_bulkfree_info_t
*cbinfo
,
373 hammer2_blockref_t
*bref
);
374 static int h2_bulkfree_sync(hammer2_bulkfree_info_t
*cbinfo
);
375 static void h2_bulkfree_sync_adjust(hammer2_bulkfree_info_t
*cbinfo
,
376 hammer2_off_t data_off
, hammer2_bmap_data_t
*live
,
377 hammer2_bmap_data_t
*bmap
, hammer2_key_t alloc_base
);
380 hammer2_bulkfree_init(hammer2_dev_t
*hmp
)
382 hammer2_thr_create(&hmp
->bfthr
, NULL
, hmp
,
383 hmp
->devrepname
, -1, -1,
384 hammer2_bulkfree_thread
);
388 hammer2_bulkfree_uninit(hammer2_dev_t
*hmp
)
390 hammer2_thr_delete(&hmp
->bfthr
);
394 hammer2_bulkfree_thread(void *arg
)
396 hammer2_thread_t
*thr
= arg
;
397 hammer2_ioc_bulkfree_t bfi
;
401 hammer2_thr_wait_any(thr
,
402 HAMMER2_THREAD_STOP
|
403 HAMMER2_THREAD_FREEZE
|
404 HAMMER2_THREAD_UNFREEZE
|
405 HAMMER2_THREAD_REMASTER
,
410 if (flags
& HAMMER2_THREAD_STOP
)
412 if (flags
& HAMMER2_THREAD_FREEZE
) {
413 hammer2_thr_signal2(thr
, HAMMER2_THREAD_FROZEN
,
414 HAMMER2_THREAD_FREEZE
);
417 if (flags
& HAMMER2_THREAD_UNFREEZE
) {
418 hammer2_thr_signal2(thr
, 0,
419 HAMMER2_THREAD_FROZEN
|
420 HAMMER2_THREAD_UNFREEZE
);
423 if (flags
& HAMMER2_THREAD_FROZEN
)
425 if (flags
& HAMMER2_THREAD_REMASTER
) {
426 hammer2_thr_signal2(thr
, 0, HAMMER2_THREAD_REMASTER
);
427 bzero(&bfi
, sizeof(bfi
));
428 bfi
.size
= 8192 * 1024;
429 /* hammer2_bulkfree_pass(thr->hmp, &bfi); */
433 hammer2_thr_signal(thr
, HAMMER2_THREAD_STOPPED
);
434 /* structure can go invalid at this point */
438 hammer2_bulkfree_pass(hammer2_dev_t
*hmp
, hammer2_chain_t
*vchain
,
439 hammer2_ioc_bulkfree_t
*bfi
)
441 hammer2_bulkfree_info_t cbinfo
;
442 hammer2_chain_save_t
*save
;
448 * We have to clear the live dedup cache as it might have entries
449 * that are freeable as of now. Any new entries in the dedup cache
450 * made after this point, even if they become freeable, will have
451 * previously been fully allocated and will be protected by the
454 hammer2_dedup_clear(hmp
);
457 * Setup for free pass using the buffer size specified by the
458 * hammer2 utility, 32K-aligned.
460 bzero(&cbinfo
, sizeof(cbinfo
));
461 size
= (bfi
->size
+ HAMMER2_FREEMAP_LEVELN_PSIZE
- 1) &
462 ~(size_t)(HAMMER2_FREEMAP_LEVELN_PSIZE
- 1);
465 * Cap at 1/4 physical memory (hammer2 utility will not normally
466 * ever specify a buffer this big, but leave the option available).
468 if (size
> kmem_lim_size() * 1024 * 1024 / 4) {
469 size
= kmem_lim_size() * 1024 * 1024 / 4;
470 kprintf("hammer2: Warning: capping bulkfree buffer at %jdM\n",
471 (intmax_t)size
/ (1024 * 1024));
474 #define HAMMER2_FREEMAP_SIZEDIV \
475 (HAMMER2_FREEMAP_LEVEL1_SIZE / HAMMER2_FREEMAP_LEVELN_PSIZE)
476 #define HAMMER2_FREEMAP_SIZEMASK (HAMMER2_FREEMAP_SIZEDIV - 1)
479 * Cap at the size needed to cover the whole volume to avoid
480 * making an unnecessarily large allocation.
482 if (size
> hmp
->total_size
/ HAMMER2_FREEMAP_SIZEDIV
) {
483 size
= (hmp
->total_size
+ HAMMER2_FREEMAP_SIZEMASK
) /
484 HAMMER2_FREEMAP_SIZEDIV
;
488 * Minimum bitmap buffer size, then align to a LEVELN_PSIZE (32K)
491 if (size
< 1024 * 1024)
493 size
= (size
+ HAMMER2_FREEMAP_LEVELN_PSIZE
- 1) &
494 ~(size_t)(HAMMER2_FREEMAP_LEVELN_PSIZE
- 1);
497 cbinfo
.bmap
= kmem_alloc_swapbacked(&cbinfo
.kp
, size
, VM_SUBSYS_HAMMER
);
498 cbinfo
.saved_mirror_tid
= hmp
->voldata
.mirror_tid
;
500 cbinfo
.dedup
= kmalloc(sizeof(*cbinfo
.dedup
) * HAMMER2_DEDUP_HEUR_SIZE
,
501 M_HAMMER2
, M_WAITOK
| M_ZERO
);
503 kprintf("hammer2: bulkfree buf=%jdM\n",
504 (intmax_t)size
/ (1024 * 1024));
507 * Normalize start point to a 2GB boundary. We operate on a
508 * 64KB leaf bitmap boundary which represents 2GB of storage.
510 cbinfo
.sbase
= bfi
->sbase
;
511 if (cbinfo
.sbase
> hmp
->total_size
)
512 cbinfo
.sbase
= hmp
->total_size
;
513 cbinfo
.sbase
&= ~HAMMER2_FREEMAP_LEVEL1_MASK
;
514 TAILQ_INIT(&cbinfo
.list
);
516 cbinfo
.bulkfree_ticks
= ticks
;
519 * Loop on a full meta-data scan as many times as required to
520 * get through all available storage.
523 while (cbinfo
.sbase
< hmp
->total_size
) {
525 * We have enough ram to represent (incr) bytes of storage.
526 * Each 64KB of ram represents 2GB of storage.
528 * We must also clean out our de-duplication heuristic for
529 * each (incr) bytes of storage, otherwise we wind up not
530 * scanning meta-data for later areas of storage because
531 * they had already been scanned in earlier areas of storage.
532 * Since the ranging is different, we have to restart
533 * the dedup heuristic too.
537 cbinfo_bmap_init(&cbinfo
, size
);
538 bzero(cbinfo
.dedup
, sizeof(*cbinfo
.dedup
) *
539 HAMMER2_DEDUP_HEUR_SIZE
);
540 cbinfo
.count_inodes_scanned
= 0;
541 cbinfo
.count_dirents_scanned
= 0;
542 cbinfo
.count_bytes_scanned
= 0;
543 cbinfo
.count_chains_scanned
= 0;
544 cbinfo
.count_chains_reported
= 0;
546 incr
= size
/ HAMMER2_FREEMAP_LEVELN_PSIZE
*
547 HAMMER2_FREEMAP_LEVEL1_SIZE
;
548 if (hmp
->total_size
- cbinfo
.sbase
<= incr
) {
549 cbinfo
.sstop
= hmp
->total_size
;
552 cbinfo
.sstop
= cbinfo
.sbase
+ incr
;
555 kprintf("hammer2: pass %016jx-%016jx ",
556 (intmax_t)cbinfo
.sbase
,
557 (intmax_t)cbinfo
.sstop
);
558 if (allmedia
&& cbinfo
.sbase
== 0)
559 kprintf("(all media)\n");
561 kprintf("(remaining media)\n");
563 kprintf("(%jdGB of media)\n",
564 (intmax_t)incr
/ (1024L*1024*1024));
567 * Scan topology for stuff inside this range.
569 * NOTE - By not using a transaction the operation can
570 * run concurrent with the frontend as well as
573 * We cannot safely set a mtid without a transaction,
574 * and in fact we don't want to set one anyway. We
575 * want the bulkfree to be passive and no interfere
576 * with crash recovery.
578 #undef HAMMER2_BULKFREE_TRANS /* undef - don't use transaction */
579 #ifdef HAMMER2_BULKFREE_TRANS
580 hammer2_trans_init(hmp
->spmp
, 0);
581 cbinfo
.mtid
= hammer2_trans_sub(hmp
->spmp
);
586 error
|= hammer2_bulk_scan(vchain
,
587 h2_bulkfree_callback
, &cbinfo
);
589 while ((save
= TAILQ_FIRST(&cbinfo
.list
)) != NULL
&&
590 (error
& ~HAMMER2_ERROR_CHECK
) == 0) {
591 TAILQ_REMOVE(&cbinfo
.list
, save
, entry
);
593 error
|= hammer2_bulk_scan(save
->chain
,
594 h2_bulkfree_callback
,
596 hammer2_chain_drop(save
->chain
);
597 kfree(save
, M_HAMMER2
);
600 TAILQ_REMOVE(&cbinfo
.list
, save
, entry
);
601 hammer2_chain_drop(save
->chain
);
602 kfree(save
, M_HAMMER2
);
603 save
= TAILQ_FIRST(&cbinfo
.list
);
607 * If the complete scan succeeded we can synchronize our
608 * in-memory freemap against live storage. If an abort
609 * occured we cannot safely synchronize our partially
610 * filled-out in-memory freemap.
612 * We still synchronize on CHECK failures. That is, we still
613 * want bulkfree to operate even if the filesystem has defects.
615 if (error
& ~HAMMER2_ERROR_CHECK
) {
616 kprintf("bulkfree lastdrop %d %d error=0x%04x\n",
617 vchain
->refs
, vchain
->core
.chain_count
, error
);
619 if (error
& HAMMER2_ERROR_CHECK
) {
620 kprintf("bulkfree lastdrop %d %d "
621 "(with check errors)\n",
622 vchain
->refs
, vchain
->core
.chain_count
);
624 kprintf("bulkfree lastdrop %d %d\n",
625 vchain
->refs
, vchain
->core
.chain_count
);
628 error
= h2_bulkfree_sync(&cbinfo
);
630 hammer2_voldata_lock(hmp
);
631 hammer2_voldata_modify(hmp
);
632 hmp
->voldata
.allocator_free
+= cbinfo
.adj_free
;
633 hammer2_voldata_unlock(hmp
);
637 * Cleanup for next loop.
639 #ifdef HAMMER2_BULKFREE_TRANS
640 hammer2_trans_done(hmp
->spmp
, 0);
642 if (error
& ~HAMMER2_ERROR_CHECK
)
644 cbinfo
.sbase
= cbinfo
.sstop
;
647 kmem_free_swapbacked(&cbinfo
.kp
);
648 kfree(cbinfo
.dedup
, M_HAMMER2
);
651 bfi
->sstop
= cbinfo
.sbase
;
653 incr
= bfi
->sstop
/ (hmp
->total_size
/ 10000);
657 kprintf("bulkfree pass statistics (%d.%02d%% storage processed):\n",
661 if (error
& ~HAMMER2_ERROR_CHECK
) {
662 kprintf(" bulkfree was aborted\n");
664 if (error
& HAMMER2_ERROR_CHECK
) {
665 kprintf(" WARNING: bulkfree "
666 "encountered CRC errors\n");
668 kprintf(" transition->free %ld\n", cbinfo
.count_10_00
);
669 kprintf(" transition->staged %ld\n", cbinfo
.count_11_10
);
670 kprintf(" ERR(00)->allocated %ld\n", cbinfo
.count_00_11
);
671 kprintf(" ERR(01)->allocated %ld\n", cbinfo
.count_01_11
);
672 kprintf(" staged->allocated %ld\n", cbinfo
.count_10_11
);
673 kprintf(" ~2MB segs cleaned %ld\n", cbinfo
.count_l0cleans
);
674 kprintf(" linear adjusts %ld\n",
675 cbinfo
.count_linadjusts
);
676 kprintf(" dedup factor %ld\n",
677 cbinfo
.count_dedup_factor
);
684 cbinfo_bmap_init(hammer2_bulkfree_info_t
*cbinfo
, size_t size
)
686 hammer2_bmap_data_t
*bmap
= cbinfo
->bmap
;
687 hammer2_key_t key
= cbinfo
->sbase
;
691 lokey
= (cbinfo
->hmp
->voldata
.allocator_beg
+ HAMMER2_SEGMASK64
) &
693 hikey
= cbinfo
->hmp
->total_size
& ~HAMMER2_SEGMASK64
;
697 bzero(bmap
, sizeof(*bmap
));
698 if (lokey
< H2FMBASE(key
, HAMMER2_FREEMAP_LEVEL1_RADIX
))
699 lokey
= H2FMBASE(key
, HAMMER2_FREEMAP_LEVEL1_RADIX
);
700 if (lokey
< H2FMZONEBASE(key
) + HAMMER2_ZONE_SEG64
)
701 lokey
= H2FMZONEBASE(key
) + HAMMER2_ZONE_SEG64
;
702 if (key
< lokey
|| key
>= hikey
) {
703 memset(bmap
->bitmapq
, -1,
704 sizeof(bmap
->bitmapq
));
706 bmap
->linear
= HAMMER2_SEGSIZE
;
708 bmap
->avail
= HAMMER2_FREEMAP_LEVEL0_SIZE
;
710 size
-= sizeof(*bmap
);
711 key
+= HAMMER2_FREEMAP_LEVEL0_SIZE
;
717 h2_bulkfree_callback(hammer2_bulkfree_info_t
*cbinfo
, hammer2_blockref_t
*bref
)
719 hammer2_bmap_data_t
*bmap
;
720 hammer2_off_t data_off
;
726 * Check for signal and allow yield to userland during scan.
728 if (hammer2_signal_check(&cbinfo
->save_time
))
729 return HAMMER2_ERROR_ABORTED
;
732 * Deal with kernel thread cpu or I/O hogging by limiting the
733 * number of chains scanned per second to hammer2_bulkfree_tps.
734 * Ignore leaf records (DIRENT and DATA), no per-record I/O is
735 * involved for those since we don't load their data.
737 if (bref
->type
!= HAMMER2_BREF_TYPE_DATA
&&
738 bref
->type
!= HAMMER2_BREF_TYPE_DIRENT
) {
739 ++cbinfo
->bulkfree_calls
;
740 if (cbinfo
->bulkfree_calls
> hammer2_bulkfree_tps
) {
741 int dticks
= ticks
- cbinfo
->bulkfree_ticks
;
745 tsleep(&cbinfo
->bulkfree_ticks
, 0,
746 "h2bw", hz
- dticks
);
748 cbinfo
->bulkfree_calls
= 0;
749 cbinfo
->bulkfree_ticks
= ticks
;
754 * Calculate the data offset and determine if it is within
755 * the current freemap range being gathered.
757 data_off
= bref
->data_off
& ~HAMMER2_OFF_MASK_RADIX
;
758 if (data_off
< cbinfo
->sbase
|| data_off
>= cbinfo
->sstop
)
760 if (data_off
< cbinfo
->hmp
->voldata
.allocator_beg
)
762 if (data_off
>= cbinfo
->hmp
->total_size
)
766 * Calculate the information needed to generate the in-memory
769 * Hammer2 does not allow allocations to cross the L1 (2GB) boundary,
770 * it's a problem if it does. (Or L0 (2MB) for that matter).
772 radix
= (int)(bref
->data_off
& HAMMER2_OFF_MASK_RADIX
);
773 KKASSERT(radix
!= 0);
774 bytes
= (size_t)1 << radix
;
775 class = (bref
->type
<< 8) | HAMMER2_PBUFRADIX
;
777 if (data_off
+ bytes
> cbinfo
->sstop
) {
778 kprintf("hammer2_bulkfree_scan: illegal 2GB boundary "
779 "%016jx %016jx/%d\n",
780 (intmax_t)bref
->data_off
,
783 bytes
= cbinfo
->sstop
- data_off
; /* XXX */
787 * Convert to a storage offset relative to the beginning of the
788 * storage range we are collecting. Then lookup the level0 bmap entry.
790 data_off
-= cbinfo
->sbase
;
791 bmap
= cbinfo
->bmap
+ (data_off
>> HAMMER2_FREEMAP_LEVEL0_RADIX
);
794 * Convert data_off to a bmap-relative value (~4MB storage range).
795 * Adjust linear, class, and avail.
797 * Hammer2 does not allow allocations to cross the L0 (4MB) boundary,
799 data_off
&= HAMMER2_FREEMAP_LEVEL0_MASK
;
800 if (data_off
+ bytes
> HAMMER2_FREEMAP_LEVEL0_SIZE
) {
801 kprintf("hammer2_bulkfree_scan: illegal 4MB boundary "
802 "%016jx %016jx/%d\n",
803 (intmax_t)bref
->data_off
,
806 bytes
= HAMMER2_FREEMAP_LEVEL0_SIZE
- data_off
;
809 if (bmap
->class == 0) {
811 bmap
->avail
= HAMMER2_FREEMAP_LEVEL0_SIZE
;
815 * NOTE: bmap->class does not have to match class. Classification
816 * is relaxed when free space is low, so some mixing can occur.
822 if (bmap
->class != class) {
823 kprintf("hammer2_bulkfree_scan: illegal mixed class "
824 "%016jx %016jx/%d (%04x vs %04x)\n",
825 (intmax_t)bref
->data_off
,
833 * Just record the highest byte-granular offset for now. Do not
834 * match against allocations which are in multiples of whole blocks.
836 * Make sure that any in-block linear offset at least covers the
837 * data range. This can cause bmap->linear to become block-aligned.
839 if (bytes
& HAMMER2_FREEMAP_BLOCK_MASK
) {
840 if (bmap
->linear
< (int32_t)data_off
+ (int32_t)bytes
)
841 bmap
->linear
= (int32_t)data_off
+ (int32_t)bytes
;
842 } else if (bmap
->linear
>= (int32_t)data_off
&&
843 bmap
->linear
< (int32_t)data_off
+ (int32_t)bytes
) {
844 bmap
->linear
= (int32_t)data_off
+ (int32_t)bytes
;
848 * Adjust the hammer2_bitmap_t bitmap[HAMMER2_BMAP_ELEMENTS].
849 * 64-bit entries, 2 bits per entry, to code 11.
851 * NOTE: data_off mask to 524288, shift right by 14 (radix for 16384),
852 * and multiply shift amount by 2 for sets of 2 bits.
854 * NOTE: The allocation can be smaller than HAMMER2_FREEMAP_BLOCK_SIZE.
855 * also, data_off may not be FREEMAP_BLOCK_SIZE aligned.
858 hammer2_bitmap_t bmask
;
861 bindex
= (int)data_off
>> (HAMMER2_FREEMAP_BLOCK_RADIX
+
862 HAMMER2_BMAP_INDEX_RADIX
);
863 bmask
= (hammer2_bitmap_t
)3 <<
864 ((((int)data_off
& HAMMER2_BMAP_INDEX_MASK
) >>
865 HAMMER2_FREEMAP_BLOCK_RADIX
) << 1);
868 * NOTE! The (avail) calculation is bitmap-granular. Multiple
869 * sub-granular records can wind up at the same bitmap
872 if ((bmap
->bitmapq
[bindex
] & bmask
) == 0) {
873 if (bytes
< HAMMER2_FREEMAP_BLOCK_SIZE
) {
874 bmap
->avail
-= HAMMER2_FREEMAP_BLOCK_SIZE
;
876 bmap
->avail
-= bytes
;
878 bmap
->bitmapq
[bindex
] |= bmask
;
880 data_off
+= HAMMER2_FREEMAP_BLOCK_SIZE
;
881 if (bytes
< HAMMER2_FREEMAP_BLOCK_SIZE
)
884 bytes
-= HAMMER2_FREEMAP_BLOCK_SIZE
;
890 * Synchronize the in-memory bitmap with the live freemap. This is not a
891 * direct copy. Instead the bitmaps must be compared:
893 * In-memory Live-freemap
894 * 00 11 -> 10 (do nothing if live modified)
895 * 10 -> 00 (do nothing if live modified)
896 * 11 10 -> 11 handles race against live
897 * ** -> 11 nominally warn of corruption
899 * We must also fixup the hints in HAMMER2_BREF_TYPE_FREEMAP_LEAF.
902 h2_bulkfree_sync(hammer2_bulkfree_info_t
*cbinfo
)
904 hammer2_off_t data_off
;
906 hammer2_key_t key_dummy
;
907 hammer2_bmap_data_t
*bmap
;
908 hammer2_bmap_data_t
*live
;
909 hammer2_chain_t
*live_parent
;
910 hammer2_chain_t
*live_chain
;
914 kprintf("hammer2_bulkfree - range ");
916 if (cbinfo
->sbase
< cbinfo
->hmp
->voldata
.allocator_beg
)
918 (intmax_t)cbinfo
->hmp
->voldata
.allocator_beg
);
921 (intmax_t)cbinfo
->sbase
);
923 if (cbinfo
->sstop
> cbinfo
->hmp
->total_size
)
925 (intmax_t)cbinfo
->hmp
->total_size
);
928 (intmax_t)cbinfo
->sstop
);
930 data_off
= cbinfo
->sbase
;
933 live_parent
= &cbinfo
->hmp
->fchain
;
934 hammer2_chain_ref(live_parent
);
935 hammer2_chain_lock(live_parent
, HAMMER2_RESOLVE_ALWAYS
);
940 * Iterate each hammer2_bmap_data_t line (128 bytes) managing
943 while (data_off
< cbinfo
->sstop
) {
945 * The freemap is not used below allocator_beg or beyond
949 if (data_off
< cbinfo
->hmp
->voldata
.allocator_beg
)
951 if (data_off
>= cbinfo
->hmp
->total_size
)
955 * Locate the freemap leaf on the live filesystem
957 key
= (data_off
& ~HAMMER2_FREEMAP_LEVEL1_MASK
);
959 if (live_chain
== NULL
|| live_chain
->bref
.key
!= key
) {
961 hammer2_chain_unlock(live_chain
);
962 hammer2_chain_drop(live_chain
);
964 live_chain
= hammer2_chain_lookup(
968 key
+ HAMMER2_FREEMAP_LEVEL1_MASK
,
970 HAMMER2_LOOKUP_ALWAYS
);
972 kprintf("hammer2_bulkfree: freemap lookup "
973 "error near %016jx, error %s\n",
975 hammer2_error_str(live_chain
->error
));
979 if (live_chain
== NULL
) {
981 * XXX if we implement a full recovery mode we need
982 * to create/recreate missing freemap chains if our
983 * bmap has any allocated blocks.
986 bmap
->avail
!= HAMMER2_FREEMAP_LEVEL0_SIZE
) {
987 kprintf("hammer2_bulkfree: cannot locate "
988 "live leaf for allocated data "
994 if (live_chain
->error
) {
995 kprintf("hammer2_bulkfree: unable to access freemap "
996 "near %016jx, error %s\n",
998 hammer2_error_str(live_chain
->error
));
999 hammer2_chain_unlock(live_chain
);
1000 hammer2_chain_drop(live_chain
);
1005 bmapindex
= (data_off
& HAMMER2_FREEMAP_LEVEL1_MASK
) >>
1006 HAMMER2_FREEMAP_LEVEL0_RADIX
;
1007 live
= &live_chain
->data
->bmdata
[bmapindex
];
1010 * Shortcut if the bitmaps match and the live linear
1011 * indicator is sane. We can't do a perfect check of
1012 * live->linear because the only real requirement is that
1013 * if it is not block-aligned, that it not cover the space
1014 * within its current block which overlaps one of the data
1015 * ranges we scan. We don't retain enough fine-grained
1016 * data in our scan to be able to set it exactly.
1018 * TODO - we could shortcut this by testing that both
1019 * live->class and bmap->class are 0, and both avails are
1020 * set to HAMMER2_FREEMAP_LEVEL0_SIZE (4MB).
1022 if (bcmp(live
->bitmapq
, bmap
->bitmapq
,
1023 sizeof(bmap
->bitmapq
)) == 0 &&
1024 live
->linear
>= bmap
->linear
) {
1027 if (hammer2_debug
& 1) {
1028 kprintf("live %016jx %04d.%04x (avail=%d)\n",
1029 data_off
, bmapindex
, live
->class, live
->avail
);
1032 hammer2_chain_modify(live_chain
, cbinfo
->mtid
, 0, 0);
1033 live_chain
->bref
.check
.freemap
.bigmask
= -1;
1034 cbinfo
->hmp
->freemap_relaxed
= 0; /* reset heuristic */
1035 live
= &live_chain
->data
->bmdata
[bmapindex
];
1037 h2_bulkfree_sync_adjust(cbinfo
, data_off
, live
, bmap
,
1038 live_chain
->bref
.key
+
1040 HAMMER2_FREEMAP_LEVEL0_SIZE
);
1042 data_off
+= HAMMER2_FREEMAP_LEVEL0_SIZE
;
1046 hammer2_chain_unlock(live_chain
);
1047 hammer2_chain_drop(live_chain
);
1050 hammer2_chain_unlock(live_parent
);
1051 hammer2_chain_drop(live_parent
);
1057 * Merge the bulkfree bitmap against the existing bitmap.
1061 h2_bulkfree_sync_adjust(hammer2_bulkfree_info_t
*cbinfo
,
1062 hammer2_off_t data_off
, hammer2_bmap_data_t
*live
,
1063 hammer2_bmap_data_t
*bmap
, hammer2_key_t alloc_base
)
1067 hammer2_off_t tmp_off
;
1068 hammer2_bitmap_t lmask
;
1069 hammer2_bitmap_t mmask
;
1073 for (bindex
= 0; bindex
< HAMMER2_BMAP_ELEMENTS
; ++bindex
) {
1074 lmask
= live
->bitmapq
[bindex
]; /* live */
1075 mmask
= bmap
->bitmapq
[bindex
]; /* snapshotted bulkfree */
1076 if (lmask
== mmask
) {
1077 tmp_off
+= HAMMER2_BMAP_INDEX_SIZE
;
1082 scount
< HAMMER2_BMAP_BITS_PER_ELEMENT
;
1084 if ((mmask
& 3) == 0) {
1086 * in-memory 00 live 11 -> 10
1089 * Storage might be marked allocated or
1090 * staged and must be remarked staged or
1093 switch (lmask
& 3) {
1097 kprintf("hammer2_bulkfree: cannot "
1098 "transition m=00/l=01\n");
1100 case 2: /* 10 -> 00 */
1101 live
->bitmapq
[bindex
] &=
1102 ~((hammer2_bitmap_t
)2 << scount
);
1104 HAMMER2_FREEMAP_BLOCK_SIZE
;
1106 HAMMER2_FREEMAP_LEVEL0_SIZE
) {
1108 HAMMER2_FREEMAP_LEVEL0_SIZE
;
1111 HAMMER2_FREEMAP_BLOCK_SIZE
;
1112 ++cbinfo
->count_10_00
;
1113 hammer2_io_dedup_assert(
1116 HAMMER2_FREEMAP_BLOCK_RADIX
,
1117 HAMMER2_FREEMAP_BLOCK_SIZE
);
1119 case 3: /* 11 -> 10 */
1120 live
->bitmapq
[bindex
] &=
1121 ~((hammer2_bitmap_t
)1 << scount
);
1122 ++cbinfo
->count_11_10
;
1123 hammer2_io_dedup_delete(
1125 HAMMER2_BREF_TYPE_DATA
,
1127 HAMMER2_FREEMAP_BLOCK_RADIX
,
1128 HAMMER2_FREEMAP_BLOCK_SIZE
);
1131 } else if ((mmask
& 3) == 3) {
1133 * in-memory 11 live 10 -> 11
1136 * Storage might be incorrectly marked free
1137 * or staged and must be remarked fully
1140 switch (lmask
& 3) {
1142 ++cbinfo
->count_00_11
;
1144 HAMMER2_FREEMAP_BLOCK_SIZE
;
1146 HAMMER2_FREEMAP_BLOCK_SIZE
;
1147 if ((int32_t)live
->avail
< 0)
1151 ++cbinfo
->count_01_11
;
1153 case 2: /* 10 -> 11 */
1154 ++cbinfo
->count_10_11
;
1159 live
->bitmapq
[bindex
] |=
1160 ((hammer2_bitmap_t
)3 << scount
);
1164 tmp_off
+= HAMMER2_FREEMAP_BLOCK_SIZE
;
1169 * Determine if the live bitmap is completely free and reset its
1170 * fields if so. Otherwise check to see if we can reduce the linear
1173 for (bindex
= HAMMER2_BMAP_ELEMENTS
- 1; bindex
>= 0; --bindex
) {
1174 if (live
->bitmapq
[bindex
] != 0)
1179 * Completely empty, reset entire segment
1182 kprintf("hammer2: cleanseg %016jx.%04x (%d)\n",
1183 alloc_base
, live
->class, live
->avail
);
1185 live
->avail
= HAMMER2_FREEMAP_LEVEL0_SIZE
;
1188 ++cbinfo
->count_l0cleans
;
1189 } else if (bindex
< 7) {
1191 * Partially full, bitmapq[bindex] != 0. Our bulkfree pass
1192 * does not record enough information to set live->linear
1195 * NOTE: Setting live->linear to a sub-block (16K) boundary
1196 * forces the live code to iterate to the next fully
1197 * free block. It does NOT mean that all blocks above
1198 * live->linear are available.
1200 * Setting live->linear to a fragmentary (less than
1201 * 16K) boundary allows allocations to iterate within
1204 if (live
->linear
< bmap
->linear
&&
1205 ((live
->linear
^ bmap
->linear
) &
1206 ~HAMMER2_FREEMAP_BLOCK_MASK
) == 0) {
1208 * If greater than but still within the same
1209 * sub-block as live we can adjust linear upward.
1211 live
->linear
= bmap
->linear
;
1212 ++cbinfo
->count_linadjusts
;
1215 * Otherwise adjust to the nearest higher or same
1216 * sub-block boundary. The live system may have
1217 * bounced live->linear around so we cannot make any
1218 * assumptions with regards to available fragmentary
1222 (bmap
->linear
+ HAMMER2_FREEMAP_BLOCK_MASK
) &
1223 ~HAMMER2_FREEMAP_BLOCK_MASK
;
1224 ++cbinfo
->count_linadjusts
;
1228 * Completely full, effectively disable the linear iterator
1230 live
->linear
= HAMMER2_SEGSIZE
;
1235 kprintf("%016jx %04d.%04x (avail=%7d) "
1236 "%08x %08x %08x %08x %08x %08x %08x %08x\n",
1239 HAMMER2_FREEMAP_LEVEL1_MASK
) >>
1240 HAMMER2_FREEMAP_LEVEL0_RADIX
),
1243 bmap
->bitmap
[0], bmap
->bitmap
[1],
1244 bmap
->bitmap
[2], bmap
->bitmap
[3],
1245 bmap
->bitmap
[4], bmap
->bitmap
[5],
1246 bmap
->bitmap
[6], bmap
->bitmap
[7]);
1252 * BULKFREE DEDUP HEURISTIC
1254 * WARNING! This code is SMP safe but the heuristic allows SMP collisions.
1255 * All fields must be loaded into locals and validated.
1259 h2_bulkfree_test(hammer2_bulkfree_info_t
*cbinfo
, hammer2_blockref_t
*bref
,
1260 int pri
, int saved_error
)
1262 hammer2_dedup_t
*dedup
;
1267 n
= hammer2_icrc32(&bref
->data_off
, sizeof(bref
->data_off
));
1268 dedup
= cbinfo
->dedup
+ (n
& (HAMMER2_DEDUP_HEUR_MASK
& ~7));
1270 for (i
= best
= 0; i
< 8; ++i
) {
1271 if (dedup
[i
].data_off
== bref
->data_off
) {
1272 if (dedup
[i
].ticks
< pri
)
1273 dedup
[i
].ticks
= pri
;
1275 cbinfo
->count_dedup_factor
+= dedup
[i
].ticks
;
1276 return (dedup
[i
].saved_error
| HAMMER2_ERROR_EOF
);
1278 if (dedup
[i
].ticks
< dedup
[best
].ticks
)
1281 dedup
[best
].data_off
= bref
->data_off
;
1282 dedup
[best
].ticks
= pri
;
1283 dedup
[best
].saved_error
= saved_error
;