4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
27 #include <sys/zfs_context.h>
29 #include <sys/spa_impl.h>
33 #include <sys/dmu_tx.h>
35 #include <sys/zio_checksum.h>
36 #include <sys/zio_compress.h>
38 static const ddt_ops_t
*ddt_ops
[DDT_TYPES
] = {
42 static const char *ddt_class_name
[DDT_CLASSES
] = {
49 ddt_object_create(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
52 spa_t
*spa
= ddt
->ddt_spa
;
53 objset_t
*os
= ddt
->ddt_os
;
54 uint64_t *objectp
= &ddt
->ddt_object
[type
][class];
55 boolean_t prehash
= zio_checksum_table
[ddt
->ddt_checksum
].ci_dedup
;
56 char name
[DDT_NAMELEN
];
58 ddt_object_name(ddt
, type
, class, name
);
60 ASSERT(*objectp
== 0);
61 VERIFY(ddt_ops
[type
]->ddt_op_create(os
, objectp
, tx
, prehash
) == 0);
62 ASSERT(*objectp
!= 0);
64 VERIFY(zap_add(os
, DMU_POOL_DIRECTORY_OBJECT
, name
,
65 sizeof (uint64_t), 1, objectp
, tx
) == 0);
67 VERIFY(zap_add(os
, spa
->spa_ddt_stat_object
, name
,
68 sizeof (uint64_t), sizeof (ddt_histogram_t
) / sizeof (uint64_t),
69 &ddt
->ddt_histogram
[type
][class], tx
) == 0);
73 ddt_object_destroy(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
76 spa_t
*spa
= ddt
->ddt_spa
;
77 objset_t
*os
= ddt
->ddt_os
;
78 uint64_t *objectp
= &ddt
->ddt_object
[type
][class];
79 char name
[DDT_NAMELEN
];
81 ddt_object_name(ddt
, type
, class, name
);
83 ASSERT(*objectp
!= 0);
84 ASSERT(ddt_object_count(ddt
, type
, class) == 0);
85 ASSERT(ddt_histogram_empty(&ddt
->ddt_histogram
[type
][class]));
86 VERIFY(zap_remove(os
, DMU_POOL_DIRECTORY_OBJECT
, name
, tx
) == 0);
87 VERIFY(zap_remove(os
, spa
->spa_ddt_stat_object
, name
, tx
) == 0);
88 VERIFY(ddt_ops
[type
]->ddt_op_destroy(os
, *objectp
, tx
) == 0);
94 ddt_object_load(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class)
96 char name
[DDT_NAMELEN
];
99 ddt_object_name(ddt
, type
, class, name
);
101 error
= zap_lookup(ddt
->ddt_os
, DMU_POOL_DIRECTORY_OBJECT
, name
,
102 sizeof (uint64_t), 1, &ddt
->ddt_object
[type
][class]);
107 error
= zap_lookup(ddt
->ddt_os
, ddt
->ddt_spa
->spa_ddt_stat_object
, name
,
108 sizeof (uint64_t), sizeof (ddt_histogram_t
) / sizeof (uint64_t),
109 &ddt
->ddt_histogram
[type
][class]);
116 ddt_object_sync(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
119 char name
[DDT_NAMELEN
];
121 ddt_object_name(ddt
, type
, class, name
);
123 VERIFY(zap_update(ddt
->ddt_os
, ddt
->ddt_spa
->spa_ddt_stat_object
, name
,
124 sizeof (uint64_t), sizeof (ddt_histogram_t
) / sizeof (uint64_t),
125 &ddt
->ddt_histogram
[type
][class], tx
) == 0);
129 ddt_object_lookup(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
132 if (!ddt_object_exists(ddt
, type
, class))
135 return (ddt_ops
[type
]->ddt_op_lookup(ddt
->ddt_os
,
136 ddt
->ddt_object
[type
][class], dde
));
140 ddt_object_update(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
141 ddt_entry_t
*dde
, dmu_tx_t
*tx
)
143 ASSERT(ddt_object_exists(ddt
, type
, class));
145 return (ddt_ops
[type
]->ddt_op_update(ddt
->ddt_os
,
146 ddt
->ddt_object
[type
][class], dde
, tx
));
150 ddt_object_remove(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
151 ddt_entry_t
*dde
, dmu_tx_t
*tx
)
153 ASSERT(ddt_object_exists(ddt
, type
, class));
155 return (ddt_ops
[type
]->ddt_op_remove(ddt
->ddt_os
,
156 ddt
->ddt_object
[type
][class], dde
, tx
));
160 ddt_object_walk(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
161 ddt_entry_t
*dde
, uint64_t *walk
)
163 ASSERT(ddt_object_exists(ddt
, type
, class));
165 return (ddt_ops
[type
]->ddt_op_walk(ddt
->ddt_os
,
166 ddt
->ddt_object
[type
][class], dde
, walk
));
170 ddt_object_count(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class)
172 ASSERT(ddt_object_exists(ddt
, type
, class));
174 return (ddt_ops
[type
]->ddt_op_count(ddt
->ddt_os
,
175 ddt
->ddt_object
[type
][class]));
179 ddt_object_info(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
180 dmu_object_info_t
*doi
)
182 if (!ddt_object_exists(ddt
, type
, class))
185 return (dmu_object_info(ddt
->ddt_os
, ddt
->ddt_object
[type
][class],
190 ddt_object_exists(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class)
192 return (!!ddt
->ddt_object
[type
][class]);
196 ddt_object_name(ddt_t
*ddt
, enum ddt_type type
, enum ddt_class
class,
199 (void) sprintf(name
, DMU_POOL_DDT
,
200 zio_checksum_table
[ddt
->ddt_checksum
].ci_name
,
201 ddt_ops
[type
]->ddt_op_name
, ddt_class_name
[class]);
205 ddt_bp_fill(const ddt_phys_t
*ddp
, blkptr_t
*bp
, uint64_t txg
)
209 for (int d
= 0; d
< SPA_DVAS_PER_BP
; d
++)
210 bp
->blk_dva
[d
] = ddp
->ddp_dva
[d
];
211 BP_SET_BIRTH(bp
, txg
, ddp
->ddp_phys_birth
);
215 ddt_bp_create(const ddt_t
*ddt
, const ddt_key_t
*ddk
, const ddt_phys_t
*ddp
,
221 ddt_bp_fill(ddp
, bp
, ddp
->ddp_phys_birth
);
223 bp
->blk_cksum
= ddk
->ddk_cksum
;
225 BP_SET_LSIZE(bp
, DDK_GET_LSIZE(ddk
));
226 BP_SET_PSIZE(bp
, DDK_GET_PSIZE(ddk
));
227 BP_SET_COMPRESS(bp
, DDK_GET_COMPRESS(ddk
));
228 BP_SET_CHECKSUM(bp
, ddt
->ddt_checksum
);
229 BP_SET_TYPE(bp
, DMU_OT_NONE
);
232 BP_SET_BYTEORDER(bp
, ZFS_HOST_BYTEORDER
);
236 ddt_key_fill(ddt_key_t
*ddk
, const blkptr_t
*bp
)
238 ddk
->ddk_cksum
= bp
->blk_cksum
;
241 DDK_SET_LSIZE(ddk
, BP_GET_LSIZE(bp
));
242 DDK_SET_PSIZE(ddk
, BP_GET_PSIZE(bp
));
243 DDK_SET_COMPRESS(ddk
, BP_GET_COMPRESS(bp
));
247 ddt_phys_fill(ddt_phys_t
*ddp
, const blkptr_t
*bp
)
249 ASSERT(ddp
->ddp_phys_birth
== 0);
251 for (int d
= 0; d
< SPA_DVAS_PER_BP
; d
++)
252 ddp
->ddp_dva
[d
] = bp
->blk_dva
[d
];
253 ddp
->ddp_phys_birth
= BP_PHYSICAL_BIRTH(bp
);
257 ddt_phys_clear(ddt_phys_t
*ddp
)
259 bzero(ddp
, sizeof (*ddp
));
263 ddt_phys_addref(ddt_phys_t
*ddp
)
269 ddt_phys_decref(ddt_phys_t
*ddp
)
271 ASSERT((int64_t)ddp
->ddp_refcnt
> 0);
276 ddt_phys_free(ddt_t
*ddt
, ddt_key_t
*ddk
, ddt_phys_t
*ddp
, uint64_t txg
)
280 ddt_bp_create(ddt
, ddk
, ddp
, &blk
);
282 zio_free(ddt
->ddt_spa
, txg
, &blk
);
286 ddt_phys_select(const ddt_entry_t
*dde
, const blkptr_t
*bp
)
288 ddt_phys_t
*ddp
= (ddt_phys_t
*)dde
->dde_phys
;
290 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++) {
291 if (DVA_EQUAL(BP_IDENTITY(bp
), &ddp
->ddp_dva
[0]) &&
292 BP_PHYSICAL_BIRTH(bp
) == ddp
->ddp_phys_birth
)
299 ddt_phys_total_refcnt(const ddt_entry_t
*dde
)
303 for (int p
= DDT_PHYS_SINGLE
; p
<= DDT_PHYS_TRIPLE
; p
++)
304 refcnt
+= dde
->dde_phys
[p
].ddp_refcnt
;
310 ddt_stat_generate(ddt_t
*ddt
, ddt_entry_t
*dde
, ddt_stat_t
*dds
)
312 spa_t
*spa
= ddt
->ddt_spa
;
313 ddt_phys_t
*ddp
= dde
->dde_phys
;
314 ddt_key_t
*ddk
= &dde
->dde_key
;
315 uint64_t lsize
= DDK_GET_LSIZE(ddk
);
316 uint64_t psize
= DDK_GET_PSIZE(ddk
);
318 bzero(dds
, sizeof (*dds
));
320 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++) {
322 uint64_t refcnt
= ddp
->ddp_refcnt
;
324 if (ddp
->ddp_phys_birth
== 0)
327 for (int d
= 0; d
< SPA_DVAS_PER_BP
; d
++)
328 dsize
+= dva_get_dsize_sync(spa
, &ddp
->ddp_dva
[d
]);
330 dds
->dds_blocks
+= 1;
331 dds
->dds_lsize
+= lsize
;
332 dds
->dds_psize
+= psize
;
333 dds
->dds_dsize
+= dsize
;
335 dds
->dds_ref_blocks
+= refcnt
;
336 dds
->dds_ref_lsize
+= lsize
* refcnt
;
337 dds
->dds_ref_psize
+= psize
* refcnt
;
338 dds
->dds_ref_dsize
+= dsize
* refcnt
;
343 ddt_stat_add(ddt_stat_t
*dst
, const ddt_stat_t
*src
, uint64_t neg
)
345 const uint64_t *s
= (const uint64_t *)src
;
346 uint64_t *d
= (uint64_t *)dst
;
347 uint64_t *d_end
= (uint64_t *)(dst
+ 1);
349 ASSERT(neg
== 0 || neg
== -1ULL); /* add or subtract */
352 *d
++ += (*s
++ ^ neg
) - neg
;
356 ddt_stat_update(ddt_t
*ddt
, ddt_entry_t
*dde
, uint64_t neg
)
359 ddt_histogram_t
*ddh
;
362 ddt_stat_generate(ddt
, dde
, &dds
);
364 bucket
= highbit(dds
.dds_ref_blocks
) - 1;
367 ddh
= &ddt
->ddt_histogram
[dde
->dde_type
][dde
->dde_class
];
369 ddt_stat_add(&ddh
->ddh_stat
[bucket
], &dds
, neg
);
373 ddt_histogram_add(ddt_histogram_t
*dst
, const ddt_histogram_t
*src
)
375 for (int h
= 0; h
< 64; h
++)
376 ddt_stat_add(&dst
->ddh_stat
[h
], &src
->ddh_stat
[h
], 0);
380 ddt_histogram_stat(ddt_stat_t
*dds
, const ddt_histogram_t
*ddh
)
382 bzero(dds
, sizeof (*dds
));
384 for (int h
= 0; h
< 64; h
++)
385 ddt_stat_add(dds
, &ddh
->ddh_stat
[h
], 0);
389 ddt_histogram_empty(const ddt_histogram_t
*ddh
)
391 const uint64_t *s
= (const uint64_t *)ddh
;
392 const uint64_t *s_end
= (const uint64_t *)(ddh
+ 1);
402 ddt_get_dedup_stats(spa_t
*spa
, ddt_stat_t
*dds_total
)
404 ddt_histogram_t ddh_total
= { 0 };
406 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++) {
407 ddt_t
*ddt
= spa
->spa_ddt
[c
];
408 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
409 for (enum ddt_class
class = 0; class < DDT_CLASSES
;
411 ddt_histogram_add(&ddh_total
,
412 &ddt
->ddt_histogram
[type
][class]);
417 ddt_histogram_stat(dds_total
, &ddh_total
);
421 ddt_get_dedup_dspace(spa_t
*spa
)
423 ddt_stat_t dds_total
= { 0 };
425 ddt_get_dedup_stats(spa
, &dds_total
);
426 return (dds_total
.dds_ref_dsize
- dds_total
.dds_dsize
);
430 ddt_get_pool_dedup_ratio(spa_t
*spa
)
432 ddt_stat_t dds_total
= { 0 };
434 ddt_get_dedup_stats(spa
, &dds_total
);
435 if (dds_total
.dds_dsize
== 0)
438 return (dds_total
.dds_ref_dsize
* 100 / dds_total
.dds_dsize
);
442 ddt_ditto_copies_needed(ddt_t
*ddt
, ddt_entry_t
*dde
, ddt_phys_t
*ddp_willref
)
444 spa_t
*spa
= ddt
->ddt_spa
;
445 uint64_t total_refcnt
= 0;
446 uint64_t ditto
= spa
->spa_dedup_ditto
;
447 int total_copies
= 0;
448 int desired_copies
= 0;
450 for (int p
= DDT_PHYS_SINGLE
; p
<= DDT_PHYS_TRIPLE
; p
++) {
451 ddt_phys_t
*ddp
= &dde
->dde_phys
[p
];
452 zio_t
*zio
= dde
->dde_lead_zio
[p
];
453 uint64_t refcnt
= ddp
->ddp_refcnt
; /* committed refs */
455 refcnt
+= zio
->io_parent_count
; /* pending refs */
456 if (ddp
== ddp_willref
)
457 refcnt
++; /* caller's ref */
459 total_refcnt
+= refcnt
;
464 if (ditto
== 0 || ditto
> UINT32_MAX
)
467 if (total_refcnt
>= 1)
469 if (total_refcnt
>= ditto
)
471 if (total_refcnt
>= ditto
* ditto
)
474 return (MAX(desired_copies
, total_copies
) - total_copies
);
478 ddt_ditto_copies_present(ddt_entry_t
*dde
)
480 ddt_phys_t
*ddp
= &dde
->dde_phys
[DDT_PHYS_DITTO
];
481 dva_t
*dva
= ddp
->ddp_dva
;
482 int copies
= 0 - DVA_GET_GANG(dva
);
484 for (int d
= 0; d
< SPA_DVAS_PER_BP
; d
++, dva
++)
485 if (DVA_IS_VALID(dva
))
488 ASSERT(copies
>= 0 && copies
< SPA_DVAS_PER_BP
);
494 ddt_compress(void *src
, uchar_t
*dst
, size_t s_len
, size_t d_len
)
496 uchar_t
*version
= dst
++;
497 int cpfunc
= ZIO_COMPRESS_ZLE
;
498 zio_compress_info_t
*ci
= &zio_compress_table
[cpfunc
];
501 ASSERT(d_len
>= s_len
+ 1); /* no compression plus version byte */
503 c_len
= ci
->ci_compress(src
, dst
, s_len
, d_len
- 1, ci
->ci_level
);
505 if (c_len
== s_len
) {
506 cpfunc
= ZIO_COMPRESS_OFF
;
507 bcopy(src
, dst
, s_len
);
510 *version
= (ZFS_HOST_BYTEORDER
& DDT_COMPRESS_BYTEORDER_MASK
) | cpfunc
;
516 ddt_decompress(uchar_t
*src
, void *dst
, size_t s_len
, size_t d_len
)
518 uchar_t version
= *src
++;
519 int cpfunc
= version
& DDT_COMPRESS_FUNCTION_MASK
;
520 zio_compress_info_t
*ci
= &zio_compress_table
[cpfunc
];
522 if (ci
->ci_decompress
!= NULL
)
523 (void) ci
->ci_decompress(src
, dst
, s_len
, d_len
, ci
->ci_level
);
525 bcopy(src
, dst
, d_len
);
527 if ((version
^ ZFS_HOST_BYTEORDER
) & DDT_COMPRESS_BYTEORDER_MASK
)
528 byteswap_uint64_array(dst
, d_len
);
532 ddt_select_by_checksum(spa_t
*spa
, enum zio_checksum c
)
534 return (spa
->spa_ddt
[c
]);
538 ddt_select(spa_t
*spa
, const blkptr_t
*bp
)
540 return (spa
->spa_ddt
[BP_GET_CHECKSUM(bp
)]);
544 ddt_enter(ddt_t
*ddt
)
546 mutex_enter(&ddt
->ddt_lock
);
552 mutex_exit(&ddt
->ddt_lock
);
556 ddt_alloc(const ddt_key_t
*ddk
)
560 dde
= kmem_zalloc(sizeof (ddt_entry_t
), KM_SLEEP
);
561 cv_init(&dde
->dde_cv
, NULL
, CV_DEFAULT
, NULL
);
569 ddt_free(ddt_entry_t
*dde
)
571 ASSERT(!dde
->dde_loading
);
573 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++)
574 ASSERT(dde
->dde_lead_zio
[p
] == NULL
);
576 if (dde
->dde_repair_data
!= NULL
)
577 zio_buf_free(dde
->dde_repair_data
,
578 DDK_GET_PSIZE(&dde
->dde_key
));
580 cv_destroy(&dde
->dde_cv
);
581 kmem_free(dde
, sizeof (*dde
));
585 ddt_remove(ddt_t
*ddt
, ddt_entry_t
*dde
)
587 ASSERT(MUTEX_HELD(&ddt
->ddt_lock
));
589 avl_remove(&ddt
->ddt_tree
, dde
);
594 ddt_lookup(ddt_t
*ddt
, const blkptr_t
*bp
, boolean_t add
)
596 ddt_entry_t
*dde
, dde_search
;
598 enum ddt_class
class;
602 ASSERT(MUTEX_HELD(&ddt
->ddt_lock
));
604 ddt_key_fill(&dde_search
.dde_key
, bp
);
606 dde
= avl_find(&ddt
->ddt_tree
, &dde_search
, &where
);
610 dde
= ddt_alloc(&dde_search
.dde_key
);
611 avl_insert(&ddt
->ddt_tree
, dde
, where
);
614 while (dde
->dde_loading
)
615 cv_wait(&dde
->dde_cv
, &ddt
->ddt_lock
);
620 dde
->dde_loading
= B_TRUE
;
626 for (type
= 0; type
< DDT_TYPES
; type
++) {
627 for (class = 0; class < DDT_CLASSES
; class++) {
628 error
= ddt_object_lookup(ddt
, type
, class, dde
);
636 ASSERT(error
== 0 || error
== ENOENT
);
640 ASSERT(dde
->dde_loaded
== B_FALSE
);
641 ASSERT(dde
->dde_loading
== B_TRUE
);
643 dde
->dde_type
= type
; /* will be DDT_TYPES if no entry found */
644 dde
->dde_class
= class; /* will be DDT_CLASSES if no entry found */
645 dde
->dde_loaded
= B_TRUE
;
646 dde
->dde_loading
= B_FALSE
;
649 ddt_stat_update(ddt
, dde
, -1ULL);
651 cv_broadcast(&dde
->dde_cv
);
657 ddt_entry_compare(const void *x1
, const void *x2
)
659 const ddt_entry_t
*dde1
= x1
;
660 const ddt_entry_t
*dde2
= x2
;
661 const uint64_t *u1
= (const uint64_t *)&dde1
->dde_key
;
662 const uint64_t *u2
= (const uint64_t *)&dde2
->dde_key
;
664 for (int i
= 0; i
< DDT_KEY_WORDS
; i
++) {
675 ddt_table_alloc(spa_t
*spa
, enum zio_checksum c
)
679 ddt
= kmem_zalloc(sizeof (*ddt
), KM_SLEEP
);
681 mutex_init(&ddt
->ddt_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
682 avl_create(&ddt
->ddt_tree
, ddt_entry_compare
,
683 sizeof (ddt_entry_t
), offsetof(ddt_entry_t
, dde_node
));
684 avl_create(&ddt
->ddt_repair_tree
, ddt_entry_compare
,
685 sizeof (ddt_entry_t
), offsetof(ddt_entry_t
, dde_node
));
686 ddt
->ddt_checksum
= c
;
688 ddt
->ddt_os
= spa
->spa_meta_objset
;
694 ddt_table_free(ddt_t
*ddt
)
696 ASSERT(avl_numnodes(&ddt
->ddt_tree
) == 0);
697 ASSERT(avl_numnodes(&ddt
->ddt_repair_tree
) == 0);
698 avl_destroy(&ddt
->ddt_tree
);
699 avl_destroy(&ddt
->ddt_repair_tree
);
700 mutex_destroy(&ddt
->ddt_lock
);
701 kmem_free(ddt
, sizeof (*ddt
));
705 ddt_create(spa_t
*spa
)
707 spa
->spa_dedup_checksum
= ZIO_DEDUPCHECKSUM
;
709 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++)
710 spa
->spa_ddt
[c
] = ddt_table_alloc(spa
, c
);
720 error
= zap_lookup(spa
->spa_meta_objset
, DMU_POOL_DIRECTORY_OBJECT
,
721 DMU_POOL_DDT_STATS
, sizeof (uint64_t), 1,
722 &spa
->spa_ddt_stat_object
);
725 return (error
== ENOENT
? 0 : error
);
727 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++) {
728 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
729 for (enum ddt_class
class = 0; class < DDT_CLASSES
;
731 ddt_t
*ddt
= spa
->spa_ddt
[c
];
732 error
= ddt_object_load(ddt
, type
, class);
733 if (error
!= 0 && error
!= ENOENT
)
743 ddt_unload(spa_t
*spa
)
745 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++) {
746 if (spa
->spa_ddt
[c
]) {
747 ddt_table_free(spa
->spa_ddt
[c
]);
748 spa
->spa_ddt
[c
] = NULL
;
754 ddt_repair_start(ddt_t
*ddt
, const blkptr_t
*bp
)
759 ddt_key_fill(&ddk
, bp
);
761 dde
= ddt_alloc(&ddk
);
763 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
764 for (enum ddt_class
class = 0; class < DDT_CLASSES
; class++) {
766 * We can only do repair if there are multiple copies
767 * of the block. For anything in the UNIQUE class,
768 * there's definitely only one copy, so don't even try.
770 if (class != DDT_CLASS_UNIQUE
&&
771 ddt_object_lookup(ddt
, type
, class, dde
) == 0)
776 bzero(dde
->dde_phys
, sizeof (dde
->dde_phys
));
782 ddt_repair_done(ddt_t
*ddt
, ddt_entry_t
*dde
)
788 if (dde
->dde_repair_data
!= NULL
&& spa_writeable(ddt
->ddt_spa
) &&
789 avl_find(&ddt
->ddt_repair_tree
, dde
, &where
) == NULL
)
790 avl_insert(&ddt
->ddt_repair_tree
, dde
, where
);
798 ddt_repair_entry_done(zio_t
*zio
)
800 ddt_entry_t
*rdde
= zio
->io_private
;
806 ddt_repair_entry(ddt_t
*ddt
, ddt_entry_t
*dde
, ddt_entry_t
*rdde
, zio_t
*rio
)
808 ddt_phys_t
*ddp
= dde
->dde_phys
;
809 ddt_phys_t
*rddp
= rdde
->dde_phys
;
810 ddt_key_t
*ddk
= &dde
->dde_key
;
811 ddt_key_t
*rddk
= &rdde
->dde_key
;
815 zio
= zio_null(rio
, rio
->io_spa
, NULL
,
816 ddt_repair_entry_done
, rdde
, rio
->io_flags
);
818 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++, rddp
++) {
819 if (ddp
->ddp_phys_birth
== 0 ||
820 ddp
->ddp_phys_birth
!= rddp
->ddp_phys_birth
||
821 bcmp(ddp
->ddp_dva
, rddp
->ddp_dva
, sizeof (ddp
->ddp_dva
)))
823 ddt_bp_create(ddt
, ddk
, ddp
, &blk
);
824 zio_nowait(zio_rewrite(zio
, zio
->io_spa
, 0, &blk
,
825 rdde
->dde_repair_data
, DDK_GET_PSIZE(rddk
), NULL
, NULL
,
826 ZIO_PRIORITY_SYNC_WRITE
, ZIO_DDT_CHILD_FLAGS(zio
), NULL
));
833 ddt_repair_table(ddt_t
*ddt
, zio_t
*rio
)
835 spa_t
*spa
= ddt
->ddt_spa
;
836 ddt_entry_t
*dde
, *rdde_next
, *rdde
;
837 avl_tree_t
*t
= &ddt
->ddt_repair_tree
;
840 if (spa_sync_pass(spa
) > 1)
844 for (rdde
= avl_first(t
); rdde
!= NULL
; rdde
= rdde_next
) {
845 rdde_next
= AVL_NEXT(t
, rdde
);
846 avl_remove(&ddt
->ddt_repair_tree
, rdde
);
848 ddt_bp_create(ddt
, &rdde
->dde_key
, NULL
, &blk
);
849 dde
= ddt_repair_start(ddt
, &blk
);
850 ddt_repair_entry(ddt
, dde
, rdde
, rio
);
851 ddt_repair_done(ddt
, dde
);
858 ddt_sync_entry(ddt_t
*ddt
, ddt_entry_t
*dde
, dmu_tx_t
*tx
, uint64_t txg
)
860 ddt_phys_t
*ddp
= dde
->dde_phys
;
861 ddt_key_t
*ddk
= &dde
->dde_key
;
862 enum ddt_type otype
= dde
->dde_type
;
863 enum ddt_type ntype
= DDT_TYPE_CURRENT
;
864 enum ddt_class oclass
= dde
->dde_class
;
865 enum ddt_class nclass
;
866 uint64_t total_refcnt
= 0;
868 ASSERT(dde
->dde_loaded
);
869 ASSERT(!dde
->dde_loading
);
871 for (int p
= 0; p
< DDT_PHYS_TYPES
; p
++, ddp
++) {
872 ASSERT(dde
->dde_lead_zio
[p
] == NULL
);
873 ASSERT((int64_t)ddp
->ddp_refcnt
>= 0);
874 if (ddp
->ddp_phys_birth
== 0) {
875 ASSERT(ddp
->ddp_refcnt
== 0);
878 if (p
== DDT_PHYS_DITTO
) {
879 if (ddt_ditto_copies_needed(ddt
, dde
, NULL
) == 0)
880 ddt_phys_free(ddt
, ddk
, ddp
, txg
);
883 if (ddp
->ddp_refcnt
== 0)
884 ddt_phys_free(ddt
, ddk
, ddp
, txg
);
885 total_refcnt
+= ddp
->ddp_refcnt
;
888 if (dde
->dde_phys
[DDT_PHYS_DITTO
].ddp_phys_birth
!= 0)
889 nclass
= DDT_CLASS_DITTO
;
890 else if (total_refcnt
> 1)
891 nclass
= DDT_CLASS_DUPLICATE
;
893 nclass
= DDT_CLASS_UNIQUE
;
895 if (otype
!= DDT_TYPES
&&
896 (otype
!= ntype
|| oclass
!= nclass
|| total_refcnt
== 0)) {
897 VERIFY(ddt_object_remove(ddt
, otype
, oclass
, dde
, tx
) == 0);
898 ASSERT(ddt_object_lookup(ddt
, otype
, oclass
, dde
) == ENOENT
);
901 if (total_refcnt
!= 0) {
902 dde
->dde_type
= ntype
;
903 dde
->dde_class
= nclass
;
904 ddt_stat_update(ddt
, dde
, 0);
905 if (!ddt_object_exists(ddt
, ntype
, nclass
))
906 ddt_object_create(ddt
, ntype
, nclass
, tx
);
907 VERIFY(ddt_object_update(ddt
, ntype
, nclass
, dde
, tx
) == 0);
912 ddt_sync_table(ddt_t
*ddt
, dmu_tx_t
*tx
, uint64_t txg
)
914 spa_t
*spa
= ddt
->ddt_spa
;
918 if (avl_numnodes(&ddt
->ddt_tree
) == 0)
921 ASSERT(spa_sync_pass(spa
) == 1);
922 ASSERT(spa
->spa_uberblock
.ub_version
>= SPA_VERSION_DEDUP
);
924 if (spa
->spa_ddt_stat_object
== 0) {
925 spa
->spa_ddt_stat_object
= zap_create(ddt
->ddt_os
,
926 DMU_OT_DDT_STATS
, DMU_OT_NONE
, 0, tx
);
927 VERIFY(zap_add(ddt
->ddt_os
, DMU_POOL_DIRECTORY_OBJECT
,
928 DMU_POOL_DDT_STATS
, sizeof (uint64_t), 1,
929 &spa
->spa_ddt_stat_object
, tx
) == 0);
932 while ((dde
= avl_destroy_nodes(&ddt
->ddt_tree
, &cookie
)) != NULL
) {
933 ddt_sync_entry(ddt
, dde
, tx
, txg
);
937 for (enum ddt_type type
= 0; type
< DDT_TYPES
; type
++) {
938 for (enum ddt_class
class = 0; class < DDT_CLASSES
; class++) {
939 if (!ddt_object_exists(ddt
, type
, class))
941 ddt_object_sync(ddt
, type
, class, tx
);
942 if (ddt_object_count(ddt
, type
, class) == 0)
943 ddt_object_destroy(ddt
, type
, class, tx
);
949 ddt_sync(spa_t
*spa
, uint64_t txg
)
952 zio_t
*rio
= zio_root(spa
, NULL
, NULL
,
953 ZIO_FLAG_CANFAIL
| ZIO_FLAG_SPECULATIVE
);
955 ASSERT(spa_syncing_txg(spa
) == txg
);
957 tx
= dmu_tx_create_assigned(spa
->spa_dsl_pool
, txg
);
959 for (enum zio_checksum c
= 0; c
< ZIO_CHECKSUM_FUNCTIONS
; c
++) {
960 ddt_t
*ddt
= spa
->spa_ddt
[c
];
963 ddt_sync_table(ddt
, tx
, txg
);
964 ddt_repair_table(ddt
, rio
);
967 (void) zio_wait(rio
);