4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
24 * Copyright (c) 2014 Integros [integros.com]
27 #include <sys/bpobj.h>
28 #include <sys/zfs_context.h>
29 #include <sys/refcount.h>
30 #include <sys/dsl_pool.h>
31 #include <sys/zfeature.h>
35 * Return an empty bpobj, preferably the empty dummy one (dp_empty_bpobj).
38 bpobj_alloc_empty(objset_t
*os
, int blocksize
, dmu_tx_t
*tx
)
40 spa_t
*spa
= dmu_objset_spa(os
);
41 dsl_pool_t
*dp
= dmu_objset_pool(os
);
43 if (spa_feature_is_enabled(spa
, SPA_FEATURE_EMPTY_BPOBJ
)) {
44 if (!spa_feature_is_active(spa
, SPA_FEATURE_EMPTY_BPOBJ
)) {
45 ASSERT0(dp
->dp_empty_bpobj
);
47 bpobj_alloc(os
, SPA_OLD_MAXBLOCKSIZE
, tx
);
49 DMU_POOL_DIRECTORY_OBJECT
,
50 DMU_POOL_EMPTY_BPOBJ
, sizeof (uint64_t), 1,
51 &dp
->dp_empty_bpobj
, tx
) == 0);
53 spa_feature_incr(spa
, SPA_FEATURE_EMPTY_BPOBJ
, tx
);
54 ASSERT(dp
->dp_empty_bpobj
!= 0);
55 return (dp
->dp_empty_bpobj
);
57 return (bpobj_alloc(os
, blocksize
, tx
));
62 bpobj_decr_empty(objset_t
*os
, dmu_tx_t
*tx
)
64 dsl_pool_t
*dp
= dmu_objset_pool(os
);
66 spa_feature_decr(dmu_objset_spa(os
), SPA_FEATURE_EMPTY_BPOBJ
, tx
);
67 if (!spa_feature_is_active(dmu_objset_spa(os
),
68 SPA_FEATURE_EMPTY_BPOBJ
)) {
69 VERIFY3U(0, ==, zap_remove(dp
->dp_meta_objset
,
70 DMU_POOL_DIRECTORY_OBJECT
,
71 DMU_POOL_EMPTY_BPOBJ
, tx
));
72 VERIFY3U(0, ==, dmu_object_free(os
, dp
->dp_empty_bpobj
, tx
));
73 dp
->dp_empty_bpobj
= 0;
78 bpobj_alloc(objset_t
*os
, int blocksize
, dmu_tx_t
*tx
)
82 if (spa_version(dmu_objset_spa(os
)) < SPA_VERSION_BPOBJ_ACCOUNT
)
84 else if (spa_version(dmu_objset_spa(os
)) < SPA_VERSION_DEADLISTS
)
87 size
= sizeof (bpobj_phys_t
);
89 return (dmu_object_alloc(os
, DMU_OT_BPOBJ
, blocksize
,
90 DMU_OT_BPOBJ_HDR
, size
, tx
));
94 bpobj_free(objset_t
*os
, uint64_t obj
, dmu_tx_t
*tx
)
98 dmu_object_info_t doi
;
100 dmu_buf_t
*dbuf
= NULL
;
102 ASSERT(obj
!= dmu_objset_pool(os
)->dp_empty_bpobj
);
103 VERIFY3U(0, ==, bpobj_open(&bpo
, os
, obj
));
105 mutex_enter(&bpo
.bpo_lock
);
107 if (!bpo
.bpo_havesubobj
|| bpo
.bpo_phys
->bpo_subobjs
== 0)
110 VERIFY3U(0, ==, dmu_object_info(os
, bpo
.bpo_phys
->bpo_subobjs
, &doi
));
111 epb
= doi
.doi_data_block_size
/ sizeof (uint64_t);
113 for (i
= bpo
.bpo_phys
->bpo_num_subobjs
- 1; i
>= 0; i
--) {
115 uint64_t offset
, blkoff
;
117 offset
= i
* sizeof (uint64_t);
118 blkoff
= P2PHASE(i
, epb
);
120 if (dbuf
== NULL
|| dbuf
->db_offset
> offset
) {
122 dmu_buf_rele(dbuf
, FTAG
);
123 VERIFY3U(0, ==, dmu_buf_hold(os
,
124 bpo
.bpo_phys
->bpo_subobjs
, offset
, FTAG
, &dbuf
, 0));
127 ASSERT3U(offset
, >=, dbuf
->db_offset
);
128 ASSERT3U(offset
, <, dbuf
->db_offset
+ dbuf
->db_size
);
130 objarray
= dbuf
->db_data
;
131 bpobj_free(os
, objarray
[blkoff
], tx
);
134 dmu_buf_rele(dbuf
, FTAG
);
137 VERIFY3U(0, ==, dmu_object_free(os
, bpo
.bpo_phys
->bpo_subobjs
, tx
));
140 mutex_exit(&bpo
.bpo_lock
);
143 VERIFY3U(0, ==, dmu_object_free(os
, obj
, tx
));
147 bpobj_open(bpobj_t
*bpo
, objset_t
*os
, uint64_t object
)
149 dmu_object_info_t doi
;
152 err
= dmu_object_info(os
, object
, &doi
);
156 bzero(bpo
, sizeof (*bpo
));
157 mutex_init(&bpo
->bpo_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
159 ASSERT(bpo
->bpo_dbuf
== NULL
);
160 ASSERT(bpo
->bpo_phys
== NULL
);
162 ASSERT3U(doi
.doi_type
, ==, DMU_OT_BPOBJ
);
163 ASSERT3U(doi
.doi_bonus_type
, ==, DMU_OT_BPOBJ_HDR
);
165 err
= dmu_bonus_hold(os
, object
, bpo
, &bpo
->bpo_dbuf
);
170 bpo
->bpo_object
= object
;
171 bpo
->bpo_epb
= doi
.doi_data_block_size
>> SPA_BLKPTRSHIFT
;
172 bpo
->bpo_havecomp
= (doi
.doi_bonus_size
> BPOBJ_SIZE_V0
);
173 bpo
->bpo_havesubobj
= (doi
.doi_bonus_size
> BPOBJ_SIZE_V1
);
174 bpo
->bpo_phys
= bpo
->bpo_dbuf
->db_data
;
179 bpobj_close(bpobj_t
*bpo
)
181 /* Lame workaround for closing a bpobj that was never opened. */
182 if (bpo
->bpo_object
== 0)
185 dmu_buf_rele(bpo
->bpo_dbuf
, bpo
);
186 if (bpo
->bpo_cached_dbuf
!= NULL
)
187 dmu_buf_rele(bpo
->bpo_cached_dbuf
, bpo
);
188 bpo
->bpo_dbuf
= NULL
;
189 bpo
->bpo_phys
= NULL
;
190 bpo
->bpo_cached_dbuf
= NULL
;
193 mutex_destroy(&bpo
->bpo_lock
);
197 bpobj_hasentries(bpobj_t
*bpo
)
199 return (bpo
->bpo_phys
->bpo_num_blkptrs
!= 0 ||
200 (bpo
->bpo_havesubobj
&& bpo
->bpo_phys
->bpo_num_subobjs
!= 0));
204 bpobj_iterate_impl(bpobj_t
*bpo
, bpobj_itor_t func
, void *arg
, dmu_tx_t
*tx
,
207 dmu_object_info_t doi
;
211 dmu_buf_t
*dbuf
= NULL
;
213 mutex_enter(&bpo
->bpo_lock
);
216 dmu_buf_will_dirty(bpo
->bpo_dbuf
, tx
);
218 for (i
= bpo
->bpo_phys
->bpo_num_blkptrs
- 1; i
>= 0; i
--) {
221 uint64_t offset
, blkoff
;
223 offset
= i
* sizeof (blkptr_t
);
224 blkoff
= P2PHASE(i
, bpo
->bpo_epb
);
226 if (dbuf
== NULL
|| dbuf
->db_offset
> offset
) {
228 dmu_buf_rele(dbuf
, FTAG
);
229 err
= dmu_buf_hold(bpo
->bpo_os
, bpo
->bpo_object
, offset
,
235 ASSERT3U(offset
, >=, dbuf
->db_offset
);
236 ASSERT3U(offset
, <, dbuf
->db_offset
+ dbuf
->db_size
);
238 bparray
= dbuf
->db_data
;
239 bp
= &bparray
[blkoff
];
240 err
= func(arg
, bp
, tx
);
244 bpo
->bpo_phys
->bpo_bytes
-=
245 bp_get_dsize_sync(dmu_objset_spa(bpo
->bpo_os
), bp
);
246 ASSERT3S(bpo
->bpo_phys
->bpo_bytes
, >=, 0);
247 if (bpo
->bpo_havecomp
) {
248 bpo
->bpo_phys
->bpo_comp
-= BP_GET_PSIZE(bp
);
249 bpo
->bpo_phys
->bpo_uncomp
-= BP_GET_UCSIZE(bp
);
251 bpo
->bpo_phys
->bpo_num_blkptrs
--;
252 ASSERT3S(bpo
->bpo_phys
->bpo_num_blkptrs
, >=, 0);
256 dmu_buf_rele(dbuf
, FTAG
);
260 VERIFY3U(0, ==, dmu_free_range(bpo
->bpo_os
, bpo
->bpo_object
,
261 (i
+ 1) * sizeof (blkptr_t
), -1ULL, tx
));
263 if (err
|| !bpo
->bpo_havesubobj
|| bpo
->bpo_phys
->bpo_subobjs
== 0)
266 ASSERT(bpo
->bpo_havecomp
);
267 err
= dmu_object_info(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
, &doi
);
269 mutex_exit(&bpo
->bpo_lock
);
272 ASSERT3U(doi
.doi_type
, ==, DMU_OT_BPOBJ_SUBOBJ
);
273 epb
= doi
.doi_data_block_size
/ sizeof (uint64_t);
275 for (i
= bpo
->bpo_phys
->bpo_num_subobjs
- 1; i
>= 0; i
--) {
277 uint64_t offset
, blkoff
;
279 uint64_t used_before
, comp_before
, uncomp_before
;
280 uint64_t used_after
, comp_after
, uncomp_after
;
282 offset
= i
* sizeof (uint64_t);
283 blkoff
= P2PHASE(i
, epb
);
285 if (dbuf
== NULL
|| dbuf
->db_offset
> offset
) {
287 dmu_buf_rele(dbuf
, FTAG
);
288 err
= dmu_buf_hold(bpo
->bpo_os
,
289 bpo
->bpo_phys
->bpo_subobjs
, offset
, FTAG
, &dbuf
, 0);
294 ASSERT3U(offset
, >=, dbuf
->db_offset
);
295 ASSERT3U(offset
, <, dbuf
->db_offset
+ dbuf
->db_size
);
297 objarray
= dbuf
->db_data
;
298 err
= bpobj_open(&sublist
, bpo
->bpo_os
, objarray
[blkoff
]);
302 err
= bpobj_space(&sublist
,
303 &used_before
, &comp_before
, &uncomp_before
);
305 bpobj_close(&sublist
);
309 err
= bpobj_iterate_impl(&sublist
, func
, arg
, tx
, free
);
311 VERIFY3U(0, ==, bpobj_space(&sublist
,
312 &used_after
, &comp_after
, &uncomp_after
));
313 bpo
->bpo_phys
->bpo_bytes
-= used_before
- used_after
;
314 ASSERT3S(bpo
->bpo_phys
->bpo_bytes
, >=, 0);
315 bpo
->bpo_phys
->bpo_comp
-= comp_before
- comp_after
;
316 bpo
->bpo_phys
->bpo_uncomp
-=
317 uncomp_before
- uncomp_after
;
320 bpobj_close(&sublist
);
324 err
= dmu_object_free(bpo
->bpo_os
,
325 objarray
[blkoff
], tx
);
328 bpo
->bpo_phys
->bpo_num_subobjs
--;
329 ASSERT3S(bpo
->bpo_phys
->bpo_num_subobjs
, >=, 0);
333 dmu_buf_rele(dbuf
, FTAG
);
337 VERIFY3U(0, ==, dmu_free_range(bpo
->bpo_os
,
338 bpo
->bpo_phys
->bpo_subobjs
,
339 (i
+ 1) * sizeof (uint64_t), -1ULL, tx
));
343 /* If there are no entries, there should be no bytes. */
344 if (!bpobj_hasentries(bpo
)) {
345 ASSERT0(bpo
->bpo_phys
->bpo_bytes
);
346 ASSERT0(bpo
->bpo_phys
->bpo_comp
);
347 ASSERT0(bpo
->bpo_phys
->bpo_uncomp
);
350 mutex_exit(&bpo
->bpo_lock
);
355 * Iterate and remove the entries. If func returns nonzero, iteration
356 * will stop and that entry will not be removed.
359 bpobj_iterate(bpobj_t
*bpo
, bpobj_itor_t func
, void *arg
, dmu_tx_t
*tx
)
361 return (bpobj_iterate_impl(bpo
, func
, arg
, tx
, B_TRUE
));
365 * Iterate the entries. If func returns nonzero, iteration will stop.
368 bpobj_iterate_nofree(bpobj_t
*bpo
, bpobj_itor_t func
, void *arg
, dmu_tx_t
*tx
)
370 return (bpobj_iterate_impl(bpo
, func
, arg
, tx
, B_FALSE
));
374 bpobj_enqueue_subobj(bpobj_t
*bpo
, uint64_t subobj
, dmu_tx_t
*tx
)
377 uint64_t used
, comp
, uncomp
, subsubobjs
;
379 ASSERT(bpo
->bpo_havesubobj
);
380 ASSERT(bpo
->bpo_havecomp
);
381 ASSERT(bpo
->bpo_object
!= dmu_objset_pool(bpo
->bpo_os
)->dp_empty_bpobj
);
383 if (subobj
== dmu_objset_pool(bpo
->bpo_os
)->dp_empty_bpobj
) {
384 bpobj_decr_empty(bpo
->bpo_os
, tx
);
388 VERIFY3U(0, ==, bpobj_open(&subbpo
, bpo
->bpo_os
, subobj
));
389 VERIFY3U(0, ==, bpobj_space(&subbpo
, &used
, &comp
, &uncomp
));
391 if (!bpobj_hasentries(&subbpo
)) {
392 /* No point in having an empty subobj. */
393 bpobj_close(&subbpo
);
394 bpobj_free(bpo
->bpo_os
, subobj
, tx
);
398 dmu_buf_will_dirty(bpo
->bpo_dbuf
, tx
);
399 if (bpo
->bpo_phys
->bpo_subobjs
== 0) {
400 bpo
->bpo_phys
->bpo_subobjs
= dmu_object_alloc(bpo
->bpo_os
,
401 DMU_OT_BPOBJ_SUBOBJ
, SPA_OLD_MAXBLOCKSIZE
,
405 dmu_object_info_t doi
;
406 ASSERT0(dmu_object_info(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
, &doi
));
407 ASSERT3U(doi
.doi_type
, ==, DMU_OT_BPOBJ_SUBOBJ
);
409 mutex_enter(&bpo
->bpo_lock
);
410 dmu_write(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
,
411 bpo
->bpo_phys
->bpo_num_subobjs
* sizeof (subobj
),
412 sizeof (subobj
), &subobj
, tx
);
413 bpo
->bpo_phys
->bpo_num_subobjs
++;
416 * If subobj has only one block of subobjs, then move subobj's
417 * subobjs to bpo's subobj list directly. This reduces
418 * recursion in bpobj_iterate due to nested subobjs.
420 subsubobjs
= subbpo
.bpo_phys
->bpo_subobjs
;
421 if (subsubobjs
!= 0) {
422 dmu_object_info_t doi
;
424 VERIFY3U(0, ==, dmu_object_info(bpo
->bpo_os
, subsubobjs
, &doi
));
425 if (doi
.doi_max_offset
== doi
.doi_data_block_size
) {
427 uint64_t numsubsub
= subbpo
.bpo_phys
->bpo_num_subobjs
;
429 VERIFY3U(0, ==, dmu_buf_hold(bpo
->bpo_os
, subsubobjs
,
430 0, FTAG
, &subdb
, 0));
432 * Make sure that we are not asking dmu_write()
433 * to write more data than we have in our buffer.
435 VERIFY3U(subdb
->db_size
, >=,
436 numsubsub
* sizeof (subobj
));
437 dmu_write(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
,
438 bpo
->bpo_phys
->bpo_num_subobjs
* sizeof (subobj
),
439 numsubsub
* sizeof (subobj
), subdb
->db_data
, tx
);
440 dmu_buf_rele(subdb
, FTAG
);
441 bpo
->bpo_phys
->bpo_num_subobjs
+= numsubsub
;
443 dmu_buf_will_dirty(subbpo
.bpo_dbuf
, tx
);
444 subbpo
.bpo_phys
->bpo_subobjs
= 0;
445 VERIFY3U(0, ==, dmu_object_free(bpo
->bpo_os
,
449 bpo
->bpo_phys
->bpo_bytes
+= used
;
450 bpo
->bpo_phys
->bpo_comp
+= comp
;
451 bpo
->bpo_phys
->bpo_uncomp
+= uncomp
;
452 mutex_exit(&bpo
->bpo_lock
);
454 bpobj_close(&subbpo
);
458 bpobj_enqueue(bpobj_t
*bpo
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
460 blkptr_t stored_bp
= *bp
;
465 ASSERT(!BP_IS_HOLE(bp
));
466 ASSERT(bpo
->bpo_object
!= dmu_objset_pool(bpo
->bpo_os
)->dp_empty_bpobj
);
468 if (BP_IS_EMBEDDED(bp
)) {
470 * The bpobj will compress better without the payload.
472 * Note that we store EMBEDDED bp's because they have an
473 * uncompressed size, which must be accounted for. An
474 * alternative would be to add their size to bpo_uncomp
475 * without storing the bp, but that would create additional
476 * complications: bpo_uncomp would be inconsistent with the
477 * set of BP's stored, and bpobj_iterate() wouldn't visit
478 * all the space accounted for in the bpobj.
480 bzero(&stored_bp
, sizeof (stored_bp
));
481 stored_bp
.blk_prop
= bp
->blk_prop
;
482 stored_bp
.blk_birth
= bp
->blk_birth
;
483 } else if (!BP_GET_DEDUP(bp
)) {
484 /* The bpobj will compress better without the checksum */
485 bzero(&stored_bp
.blk_cksum
, sizeof (stored_bp
.blk_cksum
));
488 /* We never need the fill count. */
489 stored_bp
.blk_fill
= 0;
491 mutex_enter(&bpo
->bpo_lock
);
493 offset
= bpo
->bpo_phys
->bpo_num_blkptrs
* sizeof (stored_bp
);
494 blkoff
= P2PHASE(bpo
->bpo_phys
->bpo_num_blkptrs
, bpo
->bpo_epb
);
496 if (bpo
->bpo_cached_dbuf
== NULL
||
497 offset
< bpo
->bpo_cached_dbuf
->db_offset
||
498 offset
>= bpo
->bpo_cached_dbuf
->db_offset
+
499 bpo
->bpo_cached_dbuf
->db_size
) {
500 if (bpo
->bpo_cached_dbuf
)
501 dmu_buf_rele(bpo
->bpo_cached_dbuf
, bpo
);
502 VERIFY3U(0, ==, dmu_buf_hold(bpo
->bpo_os
, bpo
->bpo_object
,
503 offset
, bpo
, &bpo
->bpo_cached_dbuf
, 0));
506 dmu_buf_will_dirty(bpo
->bpo_cached_dbuf
, tx
);
507 bparray
= bpo
->bpo_cached_dbuf
->db_data
;
508 bparray
[blkoff
] = stored_bp
;
510 dmu_buf_will_dirty(bpo
->bpo_dbuf
, tx
);
511 bpo
->bpo_phys
->bpo_num_blkptrs
++;
512 bpo
->bpo_phys
->bpo_bytes
+=
513 bp_get_dsize_sync(dmu_objset_spa(bpo
->bpo_os
), bp
);
514 if (bpo
->bpo_havecomp
) {
515 bpo
->bpo_phys
->bpo_comp
+= BP_GET_PSIZE(bp
);
516 bpo
->bpo_phys
->bpo_uncomp
+= BP_GET_UCSIZE(bp
);
518 mutex_exit(&bpo
->bpo_lock
);
521 struct space_range_arg
{
532 space_range_cb(void *arg
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
534 struct space_range_arg
*sra
= arg
;
536 if (bp
->blk_birth
> sra
->mintxg
&& bp
->blk_birth
<= sra
->maxtxg
) {
537 if (dsl_pool_sync_context(spa_get_dsl(sra
->spa
)))
538 sra
->used
+= bp_get_dsize_sync(sra
->spa
, bp
);
540 sra
->used
+= bp_get_dsize(sra
->spa
, bp
);
541 sra
->comp
+= BP_GET_PSIZE(bp
);
542 sra
->uncomp
+= BP_GET_UCSIZE(bp
);
548 bpobj_space(bpobj_t
*bpo
, uint64_t *usedp
, uint64_t *compp
, uint64_t *uncompp
)
550 mutex_enter(&bpo
->bpo_lock
);
552 *usedp
= bpo
->bpo_phys
->bpo_bytes
;
553 if (bpo
->bpo_havecomp
) {
554 *compp
= bpo
->bpo_phys
->bpo_comp
;
555 *uncompp
= bpo
->bpo_phys
->bpo_uncomp
;
556 mutex_exit(&bpo
->bpo_lock
);
559 mutex_exit(&bpo
->bpo_lock
);
560 return (bpobj_space_range(bpo
, 0, UINT64_MAX
,
561 usedp
, compp
, uncompp
));
566 * Return the amount of space in the bpobj which is:
567 * mintxg < blk_birth <= maxtxg
570 bpobj_space_range(bpobj_t
*bpo
, uint64_t mintxg
, uint64_t maxtxg
,
571 uint64_t *usedp
, uint64_t *compp
, uint64_t *uncompp
)
573 struct space_range_arg sra
= { 0 };
577 * As an optimization, if they want the whole txg range, just
578 * get bpo_bytes rather than iterating over the bps.
580 if (mintxg
< TXG_INITIAL
&& maxtxg
== UINT64_MAX
&& bpo
->bpo_havecomp
)
581 return (bpobj_space(bpo
, usedp
, compp
, uncompp
));
583 sra
.spa
= dmu_objset_spa(bpo
->bpo_os
);
587 err
= bpobj_iterate_nofree(bpo
, space_range_cb
, &sra
, NULL
);
590 *uncompp
= sra
.uncomp
;