1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012 by Delphix. All rights reserved.
24 */
25
26 #include <sys/zfs_context.h>
27 #include <sys/dmu_objset.h>
28 #include <sys/dmu_traverse.h>
29 #include <sys/dsl_dataset.h>
30 #include <sys/dsl_dir.h>
31 #include <sys/dsl_pool.h>
32 #include <sys/dnode.h>
33 #include <sys/spa.h>
34 #include <sys/zio.h>
35 #include <sys/dmu_impl.h>
36 #include <sys/sa.h>
37 #include <sys/sa_impl.h>
38 #include <sys/callb.h>
39
40 int zfs_pd_blks_max = 100;
41
42 typedef struct prefetch_data {
43 kmutex_t pd_mtx;
44 kcondvar_t pd_cv;
45 int pd_blks_max;
46 int pd_blks_fetched;
47 int pd_flags;
48 boolean_t pd_cancel;
49 boolean_t pd_exited;
50 } prefetch_data_t;
51
52 typedef struct traverse_data {
53 spa_t *td_spa;
54 uint64_t td_objset;
55 blkptr_t *td_rootbp;
56 uint64_t td_min_txg;
57 zbookmark_t *td_resume;
58 int td_flags;
59 prefetch_data_t *td_pfd;
60 blkptr_cb_t *td_func;
61 void *td_arg;
62 } traverse_data_t;
63
64 static int traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
65 arc_buf_t *buf, uint64_t objset, uint64_t object);
66
67 static int
68 traverse_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg)
69 {
70 traverse_data_t *td = arg;
71 zbookmark_t zb;
72
73 if (bp->blk_birth == 0)
74 return (0);
75
76 if (claim_txg == 0 && bp->blk_birth >= spa_first_txg(td->td_spa))
77 return (0);
78
79 SET_BOOKMARK(&zb, td->td_objset, ZB_ZIL_OBJECT, ZB_ZIL_LEVEL,
80 bp->blk_cksum.zc_word[ZIL_ZC_SEQ]);
81
82 (void) td->td_func(td->td_spa, zilog, bp, NULL, &zb, NULL, td->td_arg);
83
84 return (0);
85 }
86
87 static int
88 traverse_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg)
89 {
90 traverse_data_t *td = arg;
91
92 if (lrc->lrc_txtype == TX_WRITE) {
93 lr_write_t *lr = (lr_write_t *)lrc;
94 blkptr_t *bp = &lr->lr_blkptr;
95 zbookmark_t zb;
96
97 if (bp->blk_birth == 0)
98 return (0);
99
100 if (claim_txg == 0 || bp->blk_birth < claim_txg)
101 return (0);
102
103 SET_BOOKMARK(&zb, td->td_objset, lr->lr_foid,
104 ZB_ZIL_LEVEL, lr->lr_offset / BP_GET_LSIZE(bp));
105
106 (void) td->td_func(td->td_spa, zilog, bp, NULL, &zb, NULL,
107 td->td_arg);
108 }
109 return (0);
110 }
111
112 static void
113 traverse_zil(traverse_data_t *td, zil_header_t *zh)
114 {
115 uint64_t claim_txg = zh->zh_claim_txg;
116 zilog_t *zilog;
117
118 /*
119 * We only want to visit blocks that have been claimed but not yet
120 * replayed; plus, in read-only mode, blocks that are already stable.
121 */
122 if (claim_txg == 0 && spa_writeable(td->td_spa))
123 return;
124
125 zilog = zil_alloc(spa_get_dsl(td->td_spa)->dp_meta_objset, zh);
126
127 (void) zil_parse(zilog, traverse_zil_block, traverse_zil_record, td,
128 claim_txg);
129
130 zil_free(zilog);
131 }
132
133 typedef enum resume_skip {
134 RESUME_SKIP_ALL,
135 RESUME_SKIP_NONE,
136 RESUME_SKIP_CHILDREN
137 } resume_skip_t;
138
139 /*
140 * Returns RESUME_SKIP_ALL if td indicates that we are resuming a traversal and
141 * the block indicated by zb does not need to be visited at all. Returns
142 * RESUME_SKIP_CHILDREN if we are resuming a post traversal and we reach the
143 * resume point. This indicates that this block should be visited but not its
144 * children (since they must have been visited in a previous traversal).
145 * Otherwise returns RESUME_SKIP_NONE.
146 */
147 static resume_skip_t
148 resume_skip_check(traverse_data_t *td, const dnode_phys_t *dnp,
149 const zbookmark_t *zb)
150 {
151 if (td->td_resume != NULL && !ZB_IS_ZERO(td->td_resume)) {
152 /*
153 * If we already visited this bp & everything below,
154 * don't bother doing it again.
155 */
156 if (zbookmark_is_before(dnp, zb, td->td_resume))
157 return (RESUME_SKIP_ALL);
158
159 /*
160 * If we found the block we're trying to resume from, zero
161 * the bookmark out to indicate that we have resumed.
162 */
163 ASSERT3U(zb->zb_object, <=, td->td_resume->zb_object);
164 if (bcmp(zb, td->td_resume, sizeof (*zb)) == 0) {
165 bzero(td->td_resume, sizeof (*zb));
166 if (td->td_flags & TRAVERSE_POST)
167 return (RESUME_SKIP_CHILDREN);
168 }
169 }
170 return (RESUME_SKIP_NONE);
171 }
172
173 static void
174 traverse_pause(traverse_data_t *td, const zbookmark_t *zb)
175 {
176 ASSERT(td->td_resume != NULL);
177 ASSERT0(zb->zb_level);
178 bcopy(zb, td->td_resume, sizeof (*td->td_resume));
179 }
180
181 static int
182 traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
183 arc_buf_t *pbuf, blkptr_t *bp, const zbookmark_t *zb)
184 {
185 zbookmark_t czb;
186 int err = 0, lasterr = 0;
187 arc_buf_t *buf = NULL;
188 prefetch_data_t *pd = td->td_pfd;
189 boolean_t hard = td->td_flags & TRAVERSE_HARD;
190 boolean_t pause = B_FALSE;
191
192 switch (resume_skip_check(td, dnp, zb)) {
193 case RESUME_SKIP_ALL:
194 return (0);
195 case RESUME_SKIP_CHILDREN:
196 goto post;
197 case RESUME_SKIP_NONE:
198 break;
199 default:
200 ASSERT(0);
201 }
202
203 if (BP_IS_HOLE(bp)) {
204 err = td->td_func(td->td_spa, NULL, NULL, pbuf, zb, dnp,
205 td->td_arg);
206 return (err);
207 }
208
209 if (bp->blk_birth <= td->td_min_txg)
210 return (0);
211
212 if (pd && !pd->pd_exited &&
213 ((pd->pd_flags & TRAVERSE_PREFETCH_DATA) ||
214 BP_GET_TYPE(bp) == DMU_OT_DNODE || BP_GET_LEVEL(bp) > 0)) {
215 mutex_enter(&pd->pd_mtx);
216 ASSERT(pd->pd_blks_fetched >= 0);
217 while (pd->pd_blks_fetched == 0 && !pd->pd_exited)
218 cv_wait(&pd->pd_cv, &pd->pd_mtx);
219 pd->pd_blks_fetched--;
220 cv_broadcast(&pd->pd_cv);
221 mutex_exit(&pd->pd_mtx);
222 }
223
224 if (td->td_flags & TRAVERSE_PRE) {
225 err = td->td_func(td->td_spa, NULL, bp, pbuf, zb, dnp,
226 td->td_arg);
227 if (err == TRAVERSE_VISIT_NO_CHILDREN)
228 return (0);
229 if (err == ERESTART)
230 pause = B_TRUE; /* handle pausing at a common point */
231 if (err != 0)
232 goto post;
233 }
234
235 if (BP_GET_LEVEL(bp) > 0) {
236 uint32_t flags = ARC_WAIT;
237 int i;
238 blkptr_t *cbp;
239 int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
240
241 err = dsl_read(NULL, td->td_spa, bp, pbuf,
242 arc_getbuf_func, &buf,
243 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
244 if (err)
245 return (err);
246
247 /* recursively visitbp() blocks below this */
248 cbp = buf->b_data;
249 for (i = 0; i < epb; i++, cbp++) {
250 SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
251 zb->zb_level - 1,
252 zb->zb_blkid * epb + i);
253 err = traverse_visitbp(td, dnp, buf, cbp, &czb);
254 if (err) {
255 if (!hard)
256 break;
257 lasterr = err;
258 }
259 }
260 } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
261 uint32_t flags = ARC_WAIT;
262 int i;
263 int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
264
265 err = dsl_read(NULL, td->td_spa, bp, pbuf,
266 arc_getbuf_func, &buf,
267 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
268 if (err)
269 return (err);
270
271 /* recursively visitbp() blocks below this */
272 dnp = buf->b_data;
273 for (i = 0; i < epb; i++, dnp++) {
274 err = traverse_dnode(td, dnp, buf, zb->zb_objset,
275 zb->zb_blkid * epb + i);
276 if (err) {
277 if (!hard)
278 break;
279 lasterr = err;
280 }
281 }
282 } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
283 uint32_t flags = ARC_WAIT;
284 objset_phys_t *osp;
285 dnode_phys_t *dnp;
286
287 err = dsl_read_nolock(NULL, td->td_spa, bp,
288 arc_getbuf_func, &buf,
289 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
290 if (err)
291 return (err);
292
293 osp = buf->b_data;
294 dnp = &osp->os_meta_dnode;
295 err = traverse_dnode(td, dnp, buf, zb->zb_objset,
296 DMU_META_DNODE_OBJECT);
297 if (err && hard) {
298 lasterr = err;
299 err = 0;
300 }
301 if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
302 dnp = &osp->os_userused_dnode;
303 err = traverse_dnode(td, dnp, buf, zb->zb_objset,
304 DMU_USERUSED_OBJECT);
305 }
306 if (err && hard) {
307 lasterr = err;
308 err = 0;
309 }
310 if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
311 dnp = &osp->os_groupused_dnode;
312 err = traverse_dnode(td, dnp, buf, zb->zb_objset,
313 DMU_GROUPUSED_OBJECT);
314 }
315 }
316
317 if (buf)
318 (void) arc_buf_remove_ref(buf, &buf);
319
320 post:
321 if (err == 0 && lasterr == 0 && (td->td_flags & TRAVERSE_POST)) {
322 err = td->td_func(td->td_spa, NULL, bp, pbuf, zb, dnp,
323 td->td_arg);
324 if (err == ERESTART)
325 pause = B_TRUE;
326 }
327
328 if (pause && td->td_resume != NULL) {
329 ASSERT3U(err, ==, ERESTART);
330 ASSERT(!hard);
331 traverse_pause(td, zb);
332 }
333
334 return (err != 0 ? err : lasterr);
335 }
336
337 static int
338 traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
339 arc_buf_t *buf, uint64_t objset, uint64_t object)
340 {
341 int j, err = 0, lasterr = 0;
342 zbookmark_t czb;
343 boolean_t hard = (td->td_flags & TRAVERSE_HARD);
344
345 for (j = 0; j < dnp->dn_nblkptr; j++) {
346 SET_BOOKMARK(&czb, objset, object, dnp->dn_nlevels - 1, j);
347 err = traverse_visitbp(td, dnp, buf,
348 (blkptr_t *)&dnp->dn_blkptr[j], &czb);
349 if (err) {
350 if (!hard)
351 break;
352 lasterr = err;
353 }
354 }
355
356 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
357 SET_BOOKMARK(&czb, objset,
358 object, 0, DMU_SPILL_BLKID);
359 err = traverse_visitbp(td, dnp, buf,
360 (blkptr_t *)&dnp->dn_spill, &czb);
361 if (err) {
362 if (!hard)
363 return (err);
364 lasterr = err;
365 }
366 }
367 return (err != 0 ? err : lasterr);
368 }
369
370 /* ARGSUSED */
371 static int
372 traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
373 arc_buf_t *pbuf, const zbookmark_t *zb, const dnode_phys_t *dnp,
374 void *arg)
375 {
376 prefetch_data_t *pfd = arg;
377 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
378
379 ASSERT(pfd->pd_blks_fetched >= 0);
380 if (pfd->pd_cancel)
381 return (EINTR);
382
383 if (bp == NULL || !((pfd->pd_flags & TRAVERSE_PREFETCH_DATA) ||
384 BP_GET_TYPE(bp) == DMU_OT_DNODE || BP_GET_LEVEL(bp) > 0) ||
385 BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG)
386 return (0);
387
388 mutex_enter(&pfd->pd_mtx);
389 while (!pfd->pd_cancel && pfd->pd_blks_fetched >= pfd->pd_blks_max)
390 cv_wait(&pfd->pd_cv, &pfd->pd_mtx);
391 pfd->pd_blks_fetched++;
392 cv_broadcast(&pfd->pd_cv);
393 mutex_exit(&pfd->pd_mtx);
394
395 (void) dsl_read(NULL, spa, bp, pbuf, NULL, NULL,
396 ZIO_PRIORITY_ASYNC_READ,
397 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
398 &aflags, zb);
399
400 return (0);
401 }
402
403 static void
404 traverse_prefetch_thread(void *arg)
405 {
406 traverse_data_t *td_main = arg;
407 traverse_data_t td = *td_main;
408 zbookmark_t czb;
409
410 td.td_func = traverse_prefetcher;
411 td.td_arg = td_main->td_pfd;
412 td.td_pfd = NULL;
413
414 SET_BOOKMARK(&czb, td.td_objset,
415 ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
416 (void) traverse_visitbp(&td, NULL, NULL, td.td_rootbp, &czb);
417
418 mutex_enter(&td_main->td_pfd->pd_mtx);
419 td_main->td_pfd->pd_exited = B_TRUE;
420 cv_broadcast(&td_main->td_pfd->pd_cv);
421 mutex_exit(&td_main->td_pfd->pd_mtx);
422 }
423
424 /*
425 * NB: dataset must not be changing on-disk (eg, is a snapshot or we are
426 * in syncing context).
427 */
428 static int
429 traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
430 uint64_t txg_start, zbookmark_t *resume, int flags,
431 blkptr_cb_t func, void *arg)
432 {
433 traverse_data_t td;
434 prefetch_data_t pd = { 0 };
435 zbookmark_t czb;
436 int err;
437
438 ASSERT(ds == NULL || objset == ds->ds_object);
439 ASSERT(!(flags & TRAVERSE_PRE) || !(flags & TRAVERSE_POST));
440
441 td.td_spa = spa;
442 td.td_objset = objset;
443 td.td_rootbp = rootbp;
444 td.td_min_txg = txg_start;
445 td.td_resume = resume;
446 td.td_func = func;
447 td.td_arg = arg;
448 td.td_pfd = &pd;
449 td.td_flags = flags;
450
451 pd.pd_blks_max = zfs_pd_blks_max;
452 pd.pd_flags = flags;
453 mutex_init(&pd.pd_mtx, NULL, MUTEX_DEFAULT, NULL);
454 cv_init(&pd.pd_cv, NULL, CV_DEFAULT, NULL);
455
456 /* See comment on ZIL traversal in dsl_scan_visitds. */
457 if (ds != NULL && !dsl_dataset_is_snapshot(ds)) {
458 objset_t *os;
459
460 err = dmu_objset_from_ds(ds, &os);
461 if (err)
462 return (err);
463
464 traverse_zil(&td, &os->os_zil_header);
465 }
466
467 if (!(flags & TRAVERSE_PREFETCH) ||
468 0 == taskq_dispatch(system_taskq, traverse_prefetch_thread,
469 &td, TQ_NOQUEUE))
470 pd.pd_exited = B_TRUE;
471
472 SET_BOOKMARK(&czb, td.td_objset,
473 ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
474 err = traverse_visitbp(&td, NULL, NULL, rootbp, &czb);
475
476 mutex_enter(&pd.pd_mtx);
477 pd.pd_cancel = B_TRUE;
478 cv_broadcast(&pd.pd_cv);
479 while (!pd.pd_exited)
480 cv_wait(&pd.pd_cv, &pd.pd_mtx);
481 mutex_exit(&pd.pd_mtx);
482
483 mutex_destroy(&pd.pd_mtx);
484 cv_destroy(&pd.pd_cv);
485
486 return (err);
487 }
488
489 /*
490 * NB: dataset must not be changing on-disk (eg, is a snapshot or we are
491 * in syncing context).
492 */
493 int
494 traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start, int flags,
495 blkptr_cb_t func, void *arg)
496 {
497 return (traverse_impl(ds->ds_dir->dd_pool->dp_spa, ds, ds->ds_object,
498 &ds->ds_phys->ds_bp, txg_start, NULL, flags, func, arg));
499 }
500
501 int
502 traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr,
503 uint64_t txg_start, zbookmark_t *resume, int flags,
504 blkptr_cb_t func, void *arg)
505 {
506 return (traverse_impl(spa, NULL, ZB_DESTROYED_OBJSET,
507 blkptr, txg_start, resume, flags, func, arg));
508 }
509
510 /*
511 * NB: pool must not be changing on-disk (eg, from zdb or sync context).
512 */
513 int
514 traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
515 blkptr_cb_t func, void *arg)
516 {
517 int err, lasterr = 0;
518 uint64_t obj;
519 dsl_pool_t *dp = spa_get_dsl(spa);
520 objset_t *mos = dp->dp_meta_objset;
521 boolean_t hard = (flags & TRAVERSE_HARD);
522
523 /* visit the MOS */
524 err = traverse_impl(spa, NULL, 0, spa_get_rootblkptr(spa),
525 txg_start, NULL, flags, func, arg);
526 if (err)
527 return (err);
528
529 /* visit each dataset */
530 for (obj = 1; err == 0 || (err != ESRCH && hard);
531 err = dmu_object_next(mos, &obj, FALSE, txg_start)) {
532 dmu_object_info_t doi;
533
534 err = dmu_object_info(mos, obj, &doi);
535 if (err) {
536 if (!hard)
537 return (err);
538 lasterr = err;
539 continue;
540 }
541
542 if (doi.doi_type == DMU_OT_DSL_DATASET) {
543 dsl_dataset_t *ds;
544 uint64_t txg = txg_start;
545
546 rw_enter(&dp->dp_config_rwlock, RW_READER);
547 err = dsl_dataset_hold_obj(dp, obj, FTAG, &ds);
548 rw_exit(&dp->dp_config_rwlock);
549 if (err) {
550 if (!hard)
551 return (err);
552 lasterr = err;
553 continue;
554 }
555 if (ds->ds_phys->ds_prev_snap_txg > txg)
556 txg = ds->ds_phys->ds_prev_snap_txg;
557 err = traverse_dataset(ds, txg, flags, func, arg);
558 dsl_dataset_rele(ds, FTAG);
559 if (err) {
560 if (!hard)
561 return (err);
562 lasterr = err;
563 }
564 }
565 }
566 if (err == ESRCH)
567 err = 0;
568 return (err != 0 ? err : lasterr);
569 }