Print this page
*** NO COMMENTS ***
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/ddt.c
+++ new/usr/src/uts/common/fs/zfs/ddt.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25
26 26 #include <sys/zfs_context.h>
27 27 #include <sys/spa.h>
28 28 #include <sys/spa_impl.h>
29 29 #include <sys/zio.h>
30 30 #include <sys/ddt.h>
31 31 #include <sys/zap.h>
32 32 #include <sys/dmu_tx.h>
33 33 #include <sys/arc.h>
34 34 #include <sys/dsl_pool.h>
35 35 #include <sys/zio_checksum.h>
36 36 #include <sys/zio_compress.h>
37 37 #include <sys/dsl_scan.h>
38 38
39 39 /*
40 40 * Enable/disable prefetching of dedup-ed blocks which are going to be freed.
41 41 */
42 42 int zfs_dedup_prefetch = 1;
43 43
44 44 static const ddt_ops_t *ddt_ops[DDT_TYPES] = {
45 45 &ddt_zap_ops,
46 46 };
47 47
48 48 static const char *ddt_class_name[DDT_CLASSES] = {
49 49 "ditto",
50 50 "duplicate",
51 51 "unique",
52 52 };
53 53
54 54 static void
55 55 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
56 56 dmu_tx_t *tx)
57 57 {
58 58 spa_t *spa = ddt->ddt_spa;
59 59 objset_t *os = ddt->ddt_os;
60 60 uint64_t *objectp = &ddt->ddt_object[type][class];
61 61 boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup;
62 62 char name[DDT_NAMELEN];
63 63
64 64 ddt_object_name(ddt, type, class, name);
65 65
66 66 ASSERT(*objectp == 0);
67 67 VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0);
68 68 ASSERT(*objectp != 0);
69 69
70 70 VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name,
71 71 sizeof (uint64_t), 1, objectp, tx) == 0);
72 72
73 73 VERIFY(zap_add(os, spa->spa_ddt_stat_object, name,
74 74 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
75 75 &ddt->ddt_histogram[type][class], tx) == 0);
76 76 }
77 77
78 78 static void
79 79 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
80 80 dmu_tx_t *tx)
81 81 {
82 82 spa_t *spa = ddt->ddt_spa;
83 83 objset_t *os = ddt->ddt_os;
84 84 uint64_t *objectp = &ddt->ddt_object[type][class];
85 85 char name[DDT_NAMELEN];
86 86
87 87 ddt_object_name(ddt, type, class, name);
88 88
89 89 ASSERT(*objectp != 0);
90 90 ASSERT(ddt_object_count(ddt, type, class) == 0);
91 91 ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class]));
92 92 VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0);
93 93 VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0);
94 94 VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0);
95 95 bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t));
96 96
97 97 *objectp = 0;
98 98 }
99 99
100 100 static int
101 101 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
102 102 {
103 103 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
104 104 dmu_object_info_t doi;
105 105 char name[DDT_NAMELEN];
106 106 int error;
107 107
108 108 ddt_object_name(ddt, type, class, name);
109 109
110 110 error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name,
111 111 sizeof (uint64_t), 1, &ddt->ddt_object[type][class]);
112 112
113 113 if (error)
114 114 return (error);
115 115
116 116 error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,
117 117 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
118 118 &ddt->ddt_histogram[type][class]);
119 119
120 120 /*
121 121 * Seed the cached statistics.
122 122 */
123 123 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);
124 124
125 125 ddo->ddo_count = ddt_object_count(ddt, type, class);
126 126 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
127 127 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
128 128
129 129 ASSERT(error == 0);
130 130 return (error);
131 131 }
132 132
133 133 static void
134 134 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
135 135 dmu_tx_t *tx)
136 136 {
137 137 ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
138 138 dmu_object_info_t doi;
139 139 char name[DDT_NAMELEN];
140 140
141 141 ddt_object_name(ddt, type, class, name);
142 142
143 143 VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,
144 144 sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
145 145 &ddt->ddt_histogram[type][class], tx) == 0);
146 146
147 147 /*
148 148 * Cache DDT statistics; this is the only time they'll change.
149 149 */
150 150 VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);
151 151
152 152 ddo->ddo_count = ddt_object_count(ddt, type, class);
153 153 ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
154 154 ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
155 155 }
156 156
157 157 static int
158 158 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
159 159 ddt_entry_t *dde)
160 160 {
161 161 if (!ddt_object_exists(ddt, type, class))
162 162 return (ENOENT);
163 163
164 164 return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os,
165 165 ddt->ddt_object[type][class], dde));
166 166 }
167 167
168 168 static void
169 169 ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
170 170 ddt_entry_t *dde)
171 171 {
172 172 if (!ddt_object_exists(ddt, type, class))
173 173 return;
174 174
175 175 ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os,
176 176 ddt->ddt_object[type][class], dde);
177 177 }
178 178
179 179 int
180 180 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
181 181 ddt_entry_t *dde, dmu_tx_t *tx)
182 182 {
183 183 ASSERT(ddt_object_exists(ddt, type, class));
184 184
185 185 return (ddt_ops[type]->ddt_op_update(ddt->ddt_os,
186 186 ddt->ddt_object[type][class], dde, tx));
187 187 }
188 188
189 189 static int
190 190 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
191 191 ddt_entry_t *dde, dmu_tx_t *tx)
192 192 {
193 193 ASSERT(ddt_object_exists(ddt, type, class));
194 194
195 195 return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os,
196 196 ddt->ddt_object[type][class], dde, tx));
197 197 }
198 198
199 199 int
200 200 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
201 201 uint64_t *walk, ddt_entry_t *dde)
202 202 {
203 203 ASSERT(ddt_object_exists(ddt, type, class));
204 204
205 205 return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os,
206 206 ddt->ddt_object[type][class], dde, walk));
207 207 }
208 208
209 209 uint64_t
210 210 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
211 211 {
212 212 ASSERT(ddt_object_exists(ddt, type, class));
213 213
214 214 return (ddt_ops[type]->ddt_op_count(ddt->ddt_os,
215 215 ddt->ddt_object[type][class]));
216 216 }
217 217
218 218 int
219 219 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
220 220 dmu_object_info_t *doi)
221 221 {
222 222 if (!ddt_object_exists(ddt, type, class))
223 223 return (ENOENT);
224 224
225 225 return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class],
226 226 doi));
227 227 }
228 228
229 229 boolean_t
230 230 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
231 231 {
232 232 return (!!ddt->ddt_object[type][class]);
233 233 }
234 234
235 235 void
236 236 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
237 237 char *name)
238 238 {
239 239 (void) sprintf(name, DMU_POOL_DDT,
240 240 zio_checksum_table[ddt->ddt_checksum].ci_name,
241 241 ddt_ops[type]->ddt_op_name, ddt_class_name[class]);
242 242 }
243 243
244 244 void
245 245 ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg)
246 246 {
247 247 ASSERT(txg != 0);
248 248
249 249 for (int d = 0; d < SPA_DVAS_PER_BP; d++)
250 250 bp->blk_dva[d] = ddp->ddp_dva[d];
251 251 BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth);
252 252 }
253 253
254 254 void
255 255 ddt_bp_create(enum zio_checksum checksum,
256 256 const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp)
257 257 {
258 258 BP_ZERO(bp);
259 259
260 260 if (ddp != NULL)
261 261 ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth);
262 262
263 263 bp->blk_cksum = ddk->ddk_cksum;
264 264 bp->blk_fill = 1;
265 265
266 266 BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk));
267 267 BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk));
268 268 BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk));
269 269 BP_SET_CHECKSUM(bp, checksum);
270 270 BP_SET_TYPE(bp, DMU_OT_DEDUP);
271 271 BP_SET_LEVEL(bp, 0);
272 272 BP_SET_DEDUP(bp, 0);
273 273 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
274 274 }
275 275
276 276 void
277 277 ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp)
278 278 {
279 279 ddk->ddk_cksum = bp->blk_cksum;
|
↓ open down ↓ |
279 lines elided |
↑ open up ↑ |
280 280 ddk->ddk_prop = 0;
281 281
282 282 DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp));
283 283 DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp));
284 284 DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp));
285 285 }
286 286
287 287 void
288 288 ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp)
289 289 {
290 + ASSERT(ddp != NULL);
290 291 ASSERT(ddp->ddp_phys_birth == 0);
291 292
292 293 for (int d = 0; d < SPA_DVAS_PER_BP; d++)
293 294 ddp->ddp_dva[d] = bp->blk_dva[d];
294 295 ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp);
295 296 }
296 297
297 298 void
298 299 ddt_phys_clear(ddt_phys_t *ddp)
299 300 {
300 - bzero(ddp, sizeof (*ddp));
301 + ASSERT(ddp != NULL);
302 + if (ddp) {
303 + bzero(ddp, sizeof (*ddp));
304 + }
301 305 }
302 306
303 307 void
304 308 ddt_phys_addref(ddt_phys_t *ddp)
305 309 {
306 - ddp->ddp_refcnt++;
310 + ASSERT(ddp != NULL);
311 + if (ddp) {
312 + ddp->ddp_refcnt++;
313 + }
307 314 }
308 315
309 316 void
310 317 ddt_phys_decref(ddt_phys_t *ddp)
311 318 {
312 - ASSERT((int64_t)ddp->ddp_refcnt > 0);
313 - ddp->ddp_refcnt--;
319 +// ASSERT(ddp != NULL);
320 + if (ddp) {
321 + ASSERT((int64_t)ddp->ddp_refcnt > 0);
322 + ddp->ddp_refcnt--;
323 + }
314 324 }
315 325
316 326 void
317 327 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg)
318 328 {
319 329 blkptr_t blk;
320 330
321 331 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);
322 332 ddt_phys_clear(ddp);
323 333 zio_free(ddt->ddt_spa, txg, &blk);
324 334 }
325 335
|
↓ open down ↓ |
2 lines elided |
↑ open up ↑ |
326 336 ddt_phys_t *
327 337 ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp)
328 338 {
329 339 ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys;
330 340
331 341 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
332 342 if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) &&
333 343 BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth)
334 344 return (ddp);
335 345 }
346 + (void) printf("ddt_phys_select() found nothing for "
347 + "DVA[BP]=<%llu:%llx:%llx> and phys_birth[BP]=%llu\n",
348 + (u_longlong_t)DVA_GET_VDEV(BP_IDENTITY(bp)),
349 + (u_longlong_t)DVA_GET_OFFSET(BP_IDENTITY(bp)),
350 + (u_longlong_t)DVA_GET_ASIZE(BP_IDENTITY(bp)),
351 + (u_longlong_t)BP_PHYSICAL_BIRTH(bp)
352 + );
336 353 return (NULL);
337 354 }
338 355
339 356 uint64_t
340 357 ddt_phys_total_refcnt(const ddt_entry_t *dde)
341 358 {
342 359 uint64_t refcnt = 0;
343 360
344 361 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++)
345 362 refcnt += dde->dde_phys[p].ddp_refcnt;
346 363
347 364 return (refcnt);
348 365 }
349 366
350 367 static void
351 368 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds)
352 369 {
353 370 spa_t *spa = ddt->ddt_spa;
354 371 ddt_phys_t *ddp = dde->dde_phys;
355 372 ddt_key_t *ddk = &dde->dde_key;
356 373 uint64_t lsize = DDK_GET_LSIZE(ddk);
357 374 uint64_t psize = DDK_GET_PSIZE(ddk);
358 375
359 376 bzero(dds, sizeof (*dds));
360 377
361 378 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
362 379 uint64_t dsize = 0;
363 380 uint64_t refcnt = ddp->ddp_refcnt;
364 381
365 382 if (ddp->ddp_phys_birth == 0)
366 383 continue;
367 384
368 385 for (int d = 0; d < SPA_DVAS_PER_BP; d++)
369 386 dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]);
370 387
371 388 dds->dds_blocks += 1;
372 389 dds->dds_lsize += lsize;
373 390 dds->dds_psize += psize;
374 391 dds->dds_dsize += dsize;
375 392
376 393 dds->dds_ref_blocks += refcnt;
377 394 dds->dds_ref_lsize += lsize * refcnt;
378 395 dds->dds_ref_psize += psize * refcnt;
379 396 dds->dds_ref_dsize += dsize * refcnt;
380 397 }
381 398 }
382 399
383 400 void
384 401 ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg)
385 402 {
386 403 const uint64_t *s = (const uint64_t *)src;
387 404 uint64_t *d = (uint64_t *)dst;
388 405 uint64_t *d_end = (uint64_t *)(dst + 1);
389 406
390 407 ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */
391 408
392 409 while (d < d_end)
393 410 *d++ += (*s++ ^ neg) - neg;
394 411 }
395 412
396 413 static void
397 414 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg)
398 415 {
399 416 ddt_stat_t dds;
400 417 ddt_histogram_t *ddh;
401 418 int bucket;
402 419
403 420 ddt_stat_generate(ddt, dde, &dds);
404 421
405 422 bucket = highbit(dds.dds_ref_blocks) - 1;
406 423 ASSERT(bucket >= 0);
407 424
408 425 ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class];
409 426
410 427 ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg);
411 428 }
412 429
413 430 void
414 431 ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src)
415 432 {
416 433 for (int h = 0; h < 64; h++)
417 434 ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0);
418 435 }
419 436
420 437 void
421 438 ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh)
422 439 {
423 440 bzero(dds, sizeof (*dds));
424 441
425 442 for (int h = 0; h < 64; h++)
426 443 ddt_stat_add(dds, &ddh->ddh_stat[h], 0);
427 444 }
428 445
429 446 boolean_t
430 447 ddt_histogram_empty(const ddt_histogram_t *ddh)
431 448 {
432 449 const uint64_t *s = (const uint64_t *)ddh;
433 450 const uint64_t *s_end = (const uint64_t *)(ddh + 1);
434 451
435 452 while (s < s_end)
436 453 if (*s++ != 0)
437 454 return (B_FALSE);
438 455
439 456 return (B_TRUE);
440 457 }
441 458
442 459 void
443 460 ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total)
444 461 {
445 462 /* Sum the statistics we cached in ddt_object_sync(). */
446 463 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
447 464 ddt_t *ddt = spa->spa_ddt[c];
448 465 for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
449 466 for (enum ddt_class class = 0; class < DDT_CLASSES;
450 467 class++) {
451 468 ddt_object_t *ddo =
452 469 &ddt->ddt_object_stats[type][class];
453 470 ddo_total->ddo_count += ddo->ddo_count;
454 471 ddo_total->ddo_dspace += ddo->ddo_dspace;
455 472 ddo_total->ddo_mspace += ddo->ddo_mspace;
456 473 }
457 474 }
458 475 }
459 476
460 477 /* ... and compute the averages. */
461 478 if (ddo_total->ddo_count != 0) {
462 479 ddo_total->ddo_dspace /= ddo_total->ddo_count;
463 480 ddo_total->ddo_mspace /= ddo_total->ddo_count;
464 481 }
465 482 }
466 483
467 484 void
468 485 ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh)
469 486 {
470 487 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
471 488 ddt_t *ddt = spa->spa_ddt[c];
472 489 for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
473 490 for (enum ddt_class class = 0; class < DDT_CLASSES;
474 491 class++) {
475 492 ddt_histogram_add(ddh,
476 493 &ddt->ddt_histogram_cache[type][class]);
477 494 }
478 495 }
479 496 }
480 497 }
481 498
482 499 void
483 500 ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total)
484 501 {
485 502 ddt_histogram_t *ddh_total;
486 503
487 504 ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP);
488 505 ddt_get_dedup_histogram(spa, ddh_total);
489 506 ddt_histogram_stat(dds_total, ddh_total);
490 507 kmem_free(ddh_total, sizeof (ddt_histogram_t));
491 508 }
492 509
493 510 uint64_t
494 511 ddt_get_dedup_dspace(spa_t *spa)
495 512 {
496 513 ddt_stat_t dds_total = { 0 };
497 514
498 515 ddt_get_dedup_stats(spa, &dds_total);
499 516 return (dds_total.dds_ref_dsize - dds_total.dds_dsize);
500 517 }
501 518
502 519 uint64_t
503 520 ddt_get_pool_dedup_ratio(spa_t *spa)
504 521 {
505 522 ddt_stat_t dds_total = { 0 };
506 523
507 524 ddt_get_dedup_stats(spa, &dds_total);
508 525 if (dds_total.dds_dsize == 0)
509 526 return (100);
510 527
511 528 return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize);
512 529 }
513 530
514 531 int
515 532 ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref)
516 533 {
517 534 spa_t *spa = ddt->ddt_spa;
518 535 uint64_t total_refcnt = 0;
519 536 uint64_t ditto = spa->spa_dedup_ditto;
520 537 int total_copies = 0;
521 538 int desired_copies = 0;
522 539
523 540 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
524 541 ddt_phys_t *ddp = &dde->dde_phys[p];
525 542 zio_t *zio = dde->dde_lead_zio[p];
526 543 uint64_t refcnt = ddp->ddp_refcnt; /* committed refs */
527 544 if (zio != NULL)
528 545 refcnt += zio->io_parent_count; /* pending refs */
529 546 if (ddp == ddp_willref)
530 547 refcnt++; /* caller's ref */
531 548 if (refcnt != 0) {
532 549 total_refcnt += refcnt;
533 550 total_copies += p;
534 551 }
535 552 }
536 553
537 554 if (ditto == 0 || ditto > UINT32_MAX)
538 555 ditto = UINT32_MAX;
539 556
540 557 if (total_refcnt >= 1)
541 558 desired_copies++;
542 559 if (total_refcnt >= ditto)
543 560 desired_copies++;
544 561 if (total_refcnt >= ditto * ditto)
545 562 desired_copies++;
546 563
547 564 return (MAX(desired_copies, total_copies) - total_copies);
548 565 }
549 566
550 567 int
551 568 ddt_ditto_copies_present(ddt_entry_t *dde)
552 569 {
553 570 ddt_phys_t *ddp = &dde->dde_phys[DDT_PHYS_DITTO];
554 571 dva_t *dva = ddp->ddp_dva;
555 572 int copies = 0 - DVA_GET_GANG(dva);
556 573
557 574 for (int d = 0; d < SPA_DVAS_PER_BP; d++, dva++)
558 575 if (DVA_IS_VALID(dva))
559 576 copies++;
560 577
561 578 ASSERT(copies >= 0 && copies < SPA_DVAS_PER_BP);
562 579
563 580 return (copies);
564 581 }
565 582
566 583 size_t
567 584 ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len)
568 585 {
569 586 uchar_t *version = dst++;
570 587 int cpfunc = ZIO_COMPRESS_ZLE;
571 588 zio_compress_info_t *ci = &zio_compress_table[cpfunc];
572 589 size_t c_len;
573 590
574 591 ASSERT(d_len >= s_len + 1); /* no compression plus version byte */
575 592
576 593 c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level);
577 594
578 595 if (c_len == s_len) {
579 596 cpfunc = ZIO_COMPRESS_OFF;
580 597 bcopy(src, dst, s_len);
581 598 }
582 599
583 600 *version = (ZFS_HOST_BYTEORDER & DDT_COMPRESS_BYTEORDER_MASK) | cpfunc;
584 601
585 602 return (c_len + 1);
586 603 }
587 604
588 605 void
589 606 ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len)
590 607 {
591 608 uchar_t version = *src++;
592 609 int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK;
593 610 zio_compress_info_t *ci = &zio_compress_table[cpfunc];
594 611
595 612 if (ci->ci_decompress != NULL)
596 613 (void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level);
597 614 else
598 615 bcopy(src, dst, d_len);
599 616
600 617 if ((version ^ ZFS_HOST_BYTEORDER) & DDT_COMPRESS_BYTEORDER_MASK)
601 618 byteswap_uint64_array(dst, d_len);
602 619 }
603 620
604 621 ddt_t *
605 622 ddt_select_by_checksum(spa_t *spa, enum zio_checksum c)
606 623 {
607 624 return (spa->spa_ddt[c]);
608 625 }
609 626
610 627 ddt_t *
611 628 ddt_select(spa_t *spa, const blkptr_t *bp)
612 629 {
613 630 return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]);
614 631 }
615 632
616 633 void
617 634 ddt_enter(ddt_t *ddt)
618 635 {
619 636 mutex_enter(&ddt->ddt_lock);
620 637 }
621 638
622 639 void
623 640 ddt_exit(ddt_t *ddt)
624 641 {
625 642 mutex_exit(&ddt->ddt_lock);
626 643 }
627 644
628 645 static ddt_entry_t *
629 646 ddt_alloc(const ddt_key_t *ddk)
630 647 {
631 648 ddt_entry_t *dde;
632 649
633 650 dde = kmem_zalloc(sizeof (ddt_entry_t), KM_SLEEP);
634 651 cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL);
635 652
636 653 dde->dde_key = *ddk;
637 654
638 655 return (dde);
639 656 }
640 657
641 658 static void
642 659 ddt_free(ddt_entry_t *dde)
643 660 {
644 661 ASSERT(!dde->dde_loading);
645 662
646 663 for (int p = 0; p < DDT_PHYS_TYPES; p++)
647 664 ASSERT(dde->dde_lead_zio[p] == NULL);
648 665
649 666 if (dde->dde_repair_data != NULL)
650 667 zio_buf_free(dde->dde_repair_data,
651 668 DDK_GET_PSIZE(&dde->dde_key));
652 669
653 670 cv_destroy(&dde->dde_cv);
654 671 kmem_free(dde, sizeof (*dde));
655 672 }
656 673
657 674 void
658 675 ddt_remove(ddt_t *ddt, ddt_entry_t *dde)
659 676 {
660 677 ASSERT(MUTEX_HELD(&ddt->ddt_lock));
661 678
662 679 avl_remove(&ddt->ddt_tree, dde);
663 680 ddt_free(dde);
664 681 }
665 682
666 683 ddt_entry_t *
667 684 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add)
668 685 {
669 686 ddt_entry_t *dde, dde_search;
670 687 enum ddt_type type;
671 688 enum ddt_class class;
672 689 avl_index_t where;
673 690 int error;
674 691
675 692 ASSERT(MUTEX_HELD(&ddt->ddt_lock));
676 693
677 694 ddt_key_fill(&dde_search.dde_key, bp);
678 695
679 696 dde = avl_find(&ddt->ddt_tree, &dde_search, &where);
680 697 if (dde == NULL) {
681 698 if (!add)
682 699 return (NULL);
683 700 dde = ddt_alloc(&dde_search.dde_key);
684 701 avl_insert(&ddt->ddt_tree, dde, where);
685 702 }
686 703
687 704 while (dde->dde_loading)
688 705 cv_wait(&dde->dde_cv, &ddt->ddt_lock);
689 706
690 707 if (dde->dde_loaded)
691 708 return (dde);
692 709
693 710 dde->dde_loading = B_TRUE;
694 711
695 712 ddt_exit(ddt);
696 713
697 714 error = ENOENT;
698 715
699 716 for (type = 0; type < DDT_TYPES; type++) {
700 717 for (class = 0; class < DDT_CLASSES; class++) {
701 718 error = ddt_object_lookup(ddt, type, class, dde);
702 719 if (error != ENOENT)
703 720 break;
704 721 }
705 722 if (error != ENOENT)
706 723 break;
707 724 }
708 725
709 726 ASSERT(error == 0 || error == ENOENT);
710 727
711 728 ddt_enter(ddt);
712 729
713 730 ASSERT(dde->dde_loaded == B_FALSE);
714 731 ASSERT(dde->dde_loading == B_TRUE);
715 732
716 733 dde->dde_type = type; /* will be DDT_TYPES if no entry found */
717 734 dde->dde_class = class; /* will be DDT_CLASSES if no entry found */
718 735 dde->dde_loaded = B_TRUE;
719 736 dde->dde_loading = B_FALSE;
720 737
721 738 if (error == 0)
722 739 ddt_stat_update(ddt, dde, -1ULL);
723 740
724 741 cv_broadcast(&dde->dde_cv);
725 742
726 743 return (dde);
727 744 }
728 745
729 746 void
730 747 ddt_prefetch(spa_t *spa, const blkptr_t *bp)
731 748 {
732 749 ddt_t *ddt;
733 750 ddt_entry_t dde;
734 751
735 752 if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp))
736 753 return;
737 754
738 755 /*
739 756 * We only remove the DDT once all tables are empty and only
740 757 * prefetch dedup blocks when there are entries in the DDT.
741 758 * Thus no locking is required as the DDT can't disappear on us.
742 759 */
743 760 ddt = ddt_select(spa, bp);
744 761 ddt_key_fill(&dde.dde_key, bp);
745 762
746 763 for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
747 764 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
748 765 ddt_object_prefetch(ddt, type, class, &dde);
749 766 }
750 767 }
751 768 }
752 769
753 770 int
754 771 ddt_entry_compare(const void *x1, const void *x2)
755 772 {
756 773 const ddt_entry_t *dde1 = x1;
757 774 const ddt_entry_t *dde2 = x2;
758 775 const uint64_t *u1 = (const uint64_t *)&dde1->dde_key;
759 776 const uint64_t *u2 = (const uint64_t *)&dde2->dde_key;
760 777
761 778 for (int i = 0; i < DDT_KEY_WORDS; i++) {
762 779 if (u1[i] < u2[i])
763 780 return (-1);
764 781 if (u1[i] > u2[i])
765 782 return (1);
766 783 }
767 784
768 785 return (0);
769 786 }
770 787
771 788 static ddt_t *
772 789 ddt_table_alloc(spa_t *spa, enum zio_checksum c)
773 790 {
774 791 ddt_t *ddt;
775 792
776 793 ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP);
777 794
778 795 mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL);
779 796 avl_create(&ddt->ddt_tree, ddt_entry_compare,
780 797 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node));
781 798 avl_create(&ddt->ddt_repair_tree, ddt_entry_compare,
782 799 sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node));
783 800 ddt->ddt_checksum = c;
784 801 ddt->ddt_spa = spa;
785 802 ddt->ddt_os = spa->spa_meta_objset;
786 803
787 804 return (ddt);
788 805 }
789 806
790 807 static void
791 808 ddt_table_free(ddt_t *ddt)
792 809 {
793 810 ASSERT(avl_numnodes(&ddt->ddt_tree) == 0);
794 811 ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0);
795 812 avl_destroy(&ddt->ddt_tree);
796 813 avl_destroy(&ddt->ddt_repair_tree);
797 814 mutex_destroy(&ddt->ddt_lock);
798 815 kmem_free(ddt, sizeof (*ddt));
799 816 }
800 817
801 818 void
802 819 ddt_create(spa_t *spa)
803 820 {
804 821 spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM;
805 822
806 823 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++)
807 824 spa->spa_ddt[c] = ddt_table_alloc(spa, c);
808 825 }
809 826
810 827 int
811 828 ddt_load(spa_t *spa)
812 829 {
813 830 int error;
814 831
815 832 ddt_create(spa);
816 833
817 834 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
818 835 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1,
819 836 &spa->spa_ddt_stat_object);
820 837
821 838 if (error)
822 839 return (error == ENOENT ? 0 : error);
823 840
824 841 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
825 842 ddt_t *ddt = spa->spa_ddt[c];
826 843 for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
827 844 for (enum ddt_class class = 0; class < DDT_CLASSES;
828 845 class++) {
829 846 error = ddt_object_load(ddt, type, class);
830 847 if (error != 0 && error != ENOENT)
831 848 return (error);
832 849 }
833 850 }
834 851
835 852 /*
836 853 * Seed the cached histograms.
837 854 */
838 855 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,
839 856 sizeof (ddt->ddt_histogram));
840 857 }
841 858
842 859 return (0);
843 860 }
844 861
845 862 void
846 863 ddt_unload(spa_t *spa)
847 864 {
848 865 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
849 866 if (spa->spa_ddt[c]) {
850 867 ddt_table_free(spa->spa_ddt[c]);
851 868 spa->spa_ddt[c] = NULL;
852 869 }
853 870 }
854 871 }
855 872
856 873 boolean_t
857 874 ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp)
858 875 {
859 876 ddt_t *ddt;
860 877 ddt_entry_t dde;
861 878
862 879 if (!BP_GET_DEDUP(bp))
863 880 return (B_FALSE);
864 881
865 882 if (max_class == DDT_CLASS_UNIQUE)
866 883 return (B_TRUE);
867 884
868 885 ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)];
869 886
870 887 ddt_key_fill(&dde.dde_key, bp);
871 888
872 889 for (enum ddt_type type = 0; type < DDT_TYPES; type++)
873 890 for (enum ddt_class class = 0; class <= max_class; class++)
874 891 if (ddt_object_lookup(ddt, type, class, &dde) == 0)
875 892 return (B_TRUE);
876 893
877 894 return (B_FALSE);
878 895 }
879 896
880 897 ddt_entry_t *
881 898 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp)
882 899 {
883 900 ddt_key_t ddk;
884 901 ddt_entry_t *dde;
885 902
886 903 ddt_key_fill(&ddk, bp);
887 904
888 905 dde = ddt_alloc(&ddk);
889 906
890 907 for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
891 908 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
892 909 /*
893 910 * We can only do repair if there are multiple copies
894 911 * of the block. For anything in the UNIQUE class,
895 912 * there's definitely only one copy, so don't even try.
896 913 */
897 914 if (class != DDT_CLASS_UNIQUE &&
898 915 ddt_object_lookup(ddt, type, class, dde) == 0)
899 916 return (dde);
900 917 }
901 918 }
902 919
903 920 bzero(dde->dde_phys, sizeof (dde->dde_phys));
904 921
905 922 return (dde);
906 923 }
907 924
908 925 void
909 926 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde)
910 927 {
911 928 avl_index_t where;
912 929
913 930 ddt_enter(ddt);
914 931
915 932 if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) &&
916 933 avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL)
917 934 avl_insert(&ddt->ddt_repair_tree, dde, where);
918 935 else
919 936 ddt_free(dde);
920 937
921 938 ddt_exit(ddt);
922 939 }
923 940
924 941 static void
925 942 ddt_repair_entry_done(zio_t *zio)
926 943 {
927 944 ddt_entry_t *rdde = zio->io_private;
928 945
929 946 ddt_free(rdde);
930 947 }
931 948
932 949 static void
933 950 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio)
934 951 {
935 952 ddt_phys_t *ddp = dde->dde_phys;
936 953 ddt_phys_t *rddp = rdde->dde_phys;
937 954 ddt_key_t *ddk = &dde->dde_key;
938 955 ddt_key_t *rddk = &rdde->dde_key;
939 956 zio_t *zio;
940 957 blkptr_t blk;
941 958
942 959 zio = zio_null(rio, rio->io_spa, NULL,
943 960 ddt_repair_entry_done, rdde, rio->io_flags);
944 961
945 962 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) {
946 963 if (ddp->ddp_phys_birth == 0 ||
947 964 ddp->ddp_phys_birth != rddp->ddp_phys_birth ||
948 965 bcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva)))
949 966 continue;
950 967 ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);
951 968 zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk,
952 969 rdde->dde_repair_data, DDK_GET_PSIZE(rddk), NULL, NULL,
953 970 ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL));
954 971 }
955 972
956 973 zio_nowait(zio);
957 974 }
958 975
959 976 static void
960 977 ddt_repair_table(ddt_t *ddt, zio_t *rio)
961 978 {
962 979 spa_t *spa = ddt->ddt_spa;
963 980 ddt_entry_t *dde, *rdde_next, *rdde;
964 981 avl_tree_t *t = &ddt->ddt_repair_tree;
965 982 blkptr_t blk;
966 983
967 984 if (spa_sync_pass(spa) > 1)
968 985 return;
969 986
970 987 ddt_enter(ddt);
971 988 for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) {
972 989 rdde_next = AVL_NEXT(t, rdde);
973 990 avl_remove(&ddt->ddt_repair_tree, rdde);
974 991 ddt_exit(ddt);
975 992 ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk);
976 993 dde = ddt_repair_start(ddt, &blk);
977 994 ddt_repair_entry(ddt, dde, rdde, rio);
978 995 ddt_repair_done(ddt, dde);
979 996 ddt_enter(ddt);
980 997 }
981 998 ddt_exit(ddt);
982 999 }
983 1000
984 1001 static void
985 1002 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg)
986 1003 {
987 1004 dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool;
988 1005 ddt_phys_t *ddp = dde->dde_phys;
989 1006 ddt_key_t *ddk = &dde->dde_key;
990 1007 enum ddt_type otype = dde->dde_type;
991 1008 enum ddt_type ntype = DDT_TYPE_CURRENT;
992 1009 enum ddt_class oclass = dde->dde_class;
993 1010 enum ddt_class nclass;
994 1011 uint64_t total_refcnt = 0;
995 1012
996 1013 ASSERT(dde->dde_loaded);
997 1014 ASSERT(!dde->dde_loading);
998 1015
999 1016 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
1000 1017 ASSERT(dde->dde_lead_zio[p] == NULL);
1001 1018 ASSERT((int64_t)ddp->ddp_refcnt >= 0);
1002 1019 if (ddp->ddp_phys_birth == 0) {
1003 1020 ASSERT(ddp->ddp_refcnt == 0);
1004 1021 continue;
1005 1022 }
1006 1023 if (p == DDT_PHYS_DITTO) {
1007 1024 if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0)
1008 1025 ddt_phys_free(ddt, ddk, ddp, txg);
1009 1026 continue;
1010 1027 }
1011 1028 if (ddp->ddp_refcnt == 0)
1012 1029 ddt_phys_free(ddt, ddk, ddp, txg);
1013 1030 total_refcnt += ddp->ddp_refcnt;
1014 1031 }
1015 1032
1016 1033 if (dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth != 0)
1017 1034 nclass = DDT_CLASS_DITTO;
1018 1035 else if (total_refcnt > 1)
1019 1036 nclass = DDT_CLASS_DUPLICATE;
1020 1037 else
1021 1038 nclass = DDT_CLASS_UNIQUE;
1022 1039
1023 1040 if (otype != DDT_TYPES &&
1024 1041 (otype != ntype || oclass != nclass || total_refcnt == 0)) {
1025 1042 VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0);
1026 1043 ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT);
1027 1044 }
1028 1045
1029 1046 if (total_refcnt != 0) {
1030 1047 dde->dde_type = ntype;
1031 1048 dde->dde_class = nclass;
1032 1049 ddt_stat_update(ddt, dde, 0);
1033 1050 if (!ddt_object_exists(ddt, ntype, nclass))
1034 1051 ddt_object_create(ddt, ntype, nclass, tx);
1035 1052 VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0);
1036 1053
1037 1054 /*
1038 1055 * If the class changes, the order that we scan this bp
1039 1056 * changes. If it decreases, we could miss it, so
1040 1057 * scan it right now. (This covers both class changing
1041 1058 * while we are doing ddt_walk(), and when we are
1042 1059 * traversing.)
1043 1060 */
1044 1061 if (nclass < oclass) {
1045 1062 dsl_scan_ddt_entry(dp->dp_scan,
1046 1063 ddt->ddt_checksum, dde, tx);
1047 1064 }
1048 1065 }
1049 1066 }
1050 1067
1051 1068 static void
1052 1069 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg)
1053 1070 {
1054 1071 spa_t *spa = ddt->ddt_spa;
1055 1072 ddt_entry_t *dde;
1056 1073 void *cookie = NULL;
1057 1074
1058 1075 if (avl_numnodes(&ddt->ddt_tree) == 0)
1059 1076 return;
1060 1077
1061 1078 ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP);
1062 1079
1063 1080 if (spa->spa_ddt_stat_object == 0) {
1064 1081 spa->spa_ddt_stat_object = zap_create(ddt->ddt_os,
1065 1082 DMU_OT_DDT_STATS, DMU_OT_NONE, 0, tx);
1066 1083 VERIFY(zap_add(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT,
1067 1084 DMU_POOL_DDT_STATS, sizeof (uint64_t), 1,
1068 1085 &spa->spa_ddt_stat_object, tx) == 0);
1069 1086 }
1070 1087
1071 1088 while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) {
1072 1089 ddt_sync_entry(ddt, dde, tx, txg);
1073 1090 ddt_free(dde);
1074 1091 }
1075 1092
1076 1093 for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
1077 1094 uint64_t count = 0;
1078 1095 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
1079 1096 if (ddt_object_exists(ddt, type, class)) {
1080 1097 ddt_object_sync(ddt, type, class, tx);
1081 1098 count += ddt_object_count(ddt, type, class);
1082 1099 }
1083 1100 }
1084 1101 for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
1085 1102 if (count == 0 && ddt_object_exists(ddt, type, class))
1086 1103 ddt_object_destroy(ddt, type, class, tx);
1087 1104 }
1088 1105 }
1089 1106
1090 1107 bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache,
1091 1108 sizeof (ddt->ddt_histogram));
1092 1109 }
1093 1110
1094 1111 void
1095 1112 ddt_sync(spa_t *spa, uint64_t txg)
1096 1113 {
1097 1114 dmu_tx_t *tx;
1098 1115 zio_t *rio = zio_root(spa, NULL, NULL,
1099 1116 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE);
1100 1117
1101 1118 ASSERT(spa_syncing_txg(spa) == txg);
1102 1119
1103 1120 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
1104 1121
1105 1122 for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
1106 1123 ddt_t *ddt = spa->spa_ddt[c];
1107 1124 if (ddt == NULL)
1108 1125 continue;
1109 1126 ddt_sync_table(ddt, tx, txg);
1110 1127 ddt_repair_table(ddt, rio);
1111 1128 }
1112 1129
1113 1130 (void) zio_wait(rio);
1114 1131
1115 1132 dmu_tx_commit(tx);
1116 1133 }
1117 1134
1118 1135 int
1119 1136 ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde)
1120 1137 {
1121 1138 do {
1122 1139 do {
1123 1140 do {
1124 1141 ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum];
1125 1142 int error = ENOENT;
1126 1143 if (ddt_object_exists(ddt, ddb->ddb_type,
1127 1144 ddb->ddb_class)) {
1128 1145 error = ddt_object_walk(ddt,
1129 1146 ddb->ddb_type, ddb->ddb_class,
1130 1147 &ddb->ddb_cursor, dde);
1131 1148 }
1132 1149 dde->dde_type = ddb->ddb_type;
1133 1150 dde->dde_class = ddb->ddb_class;
1134 1151 if (error == 0)
1135 1152 return (0);
1136 1153 if (error != ENOENT)
1137 1154 return (error);
1138 1155 ddb->ddb_cursor = 0;
1139 1156 } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS);
1140 1157 ddb->ddb_checksum = 0;
1141 1158 } while (++ddb->ddb_type < DDT_TYPES);
1142 1159 ddb->ddb_type = 0;
1143 1160 } while (++ddb->ddb_class < DDT_CLASSES);
1144 1161
1145 1162 return (ENOENT);
1146 1163 }
|
↓ open down ↓ |
801 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX