1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
24 * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
25 * Copyright (c) 2014, Joyent, Inc. All rights reserved.
26 * Copyright 2014 HybridCluster. All rights reserved.
27 * Copyright 2016 RackTop Systems.
28 */
29
30 #include <sys/dmu.h>
31 #include <sys/dmu_impl.h>
32 #include <sys/dmu_tx.h>
33 #include <sys/dbuf.h>
34 #include <sys/dnode.h>
35 #include <sys/zfs_context.h>
36 #include <sys/dmu_objset.h>
37 #include <sys/dmu_traverse.h>
38 #include <sys/dsl_dataset.h>
39 #include <sys/dsl_dir.h>
40 #include <sys/dsl_prop.h>
41 #include <sys/dsl_pool.h>
42 #include <sys/dsl_synctask.h>
43 #include <sys/zfs_ioctl.h>
44 #include <sys/zap.h>
45 #include <sys/zio_checksum.h>
46 #include <sys/zfs_znode.h>
47 #include <zfs_fletcher.h>
48 #include <sys/avl.h>
49 #include <sys/ddt.h>
50 #include <sys/zfs_onexit.h>
51 #include <sys/dmu_send.h>
52 #include <sys/dsl_destroy.h>
53 #include <sys/blkptr.h>
54 #include <sys/dsl_bookmark.h>
55 #include <sys/zfeature.h>
56 #include <sys/bqueue.h>
57
58 /* Set this tunable to TRUE to replace corrupt data with 0x2f5baddb10c */
59 int zfs_send_corrupt_data = B_FALSE;
60 int zfs_send_queue_length = 16 * 1024 * 1024;
61 int zfs_recv_queue_length = 16 * 1024 * 1024;
62 /* Set this tunable to FALSE to disable setting of DRR_FLAG_FREERECORDS */
63 int zfs_send_set_freerecords_bit = B_TRUE;
64
65 static char *dmu_recv_tag = "dmu_recv_tag";
66 const char *recv_clone_name = "%recv";
67
68 #define BP_SPAN(datablkszsec, indblkshift, level) \
69 (((uint64_t)datablkszsec) << (SPA_MINBLOCKSHIFT + \
70 (level) * (indblkshift - SPA_BLKPTRSHIFT)))
71
72 static void byteswap_record(dmu_replay_record_t *drr);
73
74 struct send_thread_arg {
75 bqueue_t q;
76 dsl_dataset_t *ds; /* Dataset to traverse */
77 uint64_t fromtxg; /* Traverse from this txg */
78 int flags; /* flags to pass to traverse_dataset */
79 int error_code;
80 boolean_t cancel;
81 zbookmark_phys_t resume;
82 };
83
84 struct send_block_record {
85 boolean_t eos_marker; /* Marks the end of the stream */
86 blkptr_t bp;
87 zbookmark_phys_t zb;
88 uint8_t indblkshift;
89 uint16_t datablkszsec;
90 bqueue_node_t ln;
91 };
92
93 static int
94 dump_bytes(dmu_sendarg_t *dsp, void *buf, int len)
95 {
96 dsl_dataset_t *ds = dmu_objset_ds(dsp->dsa_os);
97 ssize_t resid; /* have to get resid to get detailed errno */
98 ASSERT0(len % 8);
99
100 dsp->dsa_err = vn_rdwr(UIO_WRITE, dsp->dsa_vp,
101 (caddr_t)buf, len,
102 0, UIO_SYSSPACE, FAPPEND, RLIM64_INFINITY, CRED(), &resid);
103
104 mutex_enter(&ds->ds_sendstream_lock);
105 *dsp->dsa_off += len;
106 mutex_exit(&ds->ds_sendstream_lock);
107
108 return (dsp->dsa_err);
109 }
110
111 /*
112 * For all record types except BEGIN, fill in the checksum (overlaid in
113 * drr_u.drr_checksum.drr_checksum). The checksum verifies everything
114 * up to the start of the checksum itself.
115 */
116 static int
117 dump_record(dmu_sendarg_t *dsp, void *payload, int payload_len)
118 {
119 ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
120 ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
121 fletcher_4_incremental_native(dsp->dsa_drr,
122 offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
123 &dsp->dsa_zc);
124 if (dsp->dsa_drr->drr_type != DRR_BEGIN) {
125 ASSERT(ZIO_CHECKSUM_IS_ZERO(&dsp->dsa_drr->drr_u.
126 drr_checksum.drr_checksum));
127 dsp->dsa_drr->drr_u.drr_checksum.drr_checksum = dsp->dsa_zc;
128 }
129 fletcher_4_incremental_native(&dsp->dsa_drr->
130 drr_u.drr_checksum.drr_checksum,
131 sizeof (zio_cksum_t), &dsp->dsa_zc);
132 if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
133 return (SET_ERROR(EINTR));
134 if (payload_len != 0) {
135 fletcher_4_incremental_native(payload, payload_len,
136 &dsp->dsa_zc);
137 if (dump_bytes(dsp, payload, payload_len) != 0)
138 return (SET_ERROR(EINTR));
139 }
140 return (0);
141 }
142
143 /*
144 * Fill in the drr_free struct, or perform aggregation if the previous record is
145 * also a free record, and the two are adjacent.
146 *
147 * Note that we send free records even for a full send, because we want to be
148 * able to receive a full send as a clone, which requires a list of all the free
149 * and freeobject records that were generated on the source.
150 */
151 static int
152 dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
153 uint64_t length)
154 {
155 struct drr_free *drrf = &(dsp->dsa_drr->drr_u.drr_free);
156
157 /*
158 * When we receive a free record, dbuf_free_range() assumes
159 * that the receiving system doesn't have any dbufs in the range
160 * being freed. This is always true because there is a one-record
161 * constraint: we only send one WRITE record for any given
162 * object,offset. We know that the one-record constraint is
163 * true because we always send data in increasing order by
164 * object,offset.
165 *
166 * If the increasing-order constraint ever changes, we should find
167 * another way to assert that the one-record constraint is still
168 * satisfied.
169 */
170 ASSERT(object > dsp->dsa_last_data_object ||
171 (object == dsp->dsa_last_data_object &&
172 offset > dsp->dsa_last_data_offset));
173
174 if (length != -1ULL && offset + length < offset)
175 length = -1ULL;
176
177 /*
178 * If there is a pending op, but it's not PENDING_FREE, push it out,
179 * since free block aggregation can only be done for blocks of the
180 * same type (i.e., DRR_FREE records can only be aggregated with
181 * other DRR_FREE records. DRR_FREEOBJECTS records can only be
182 * aggregated with other DRR_FREEOBJECTS records.
183 */
184 if (dsp->dsa_pending_op != PENDING_NONE &&
185 dsp->dsa_pending_op != PENDING_FREE) {
186 if (dump_record(dsp, NULL, 0) != 0)
187 return (SET_ERROR(EINTR));
188 dsp->dsa_pending_op = PENDING_NONE;
189 }
190
191 if (dsp->dsa_pending_op == PENDING_FREE) {
192 /*
193 * There should never be a PENDING_FREE if length is -1
194 * (because dump_dnode is the only place where this
195 * function is called with a -1, and only after flushing
196 * any pending record).
197 */
198 ASSERT(length != -1ULL);
199 /*
200 * Check to see whether this free block can be aggregated
201 * with pending one.
202 */
203 if (drrf->drr_object == object && drrf->drr_offset +
204 drrf->drr_length == offset) {
205 drrf->drr_length += length;
206 return (0);
207 } else {
208 /* not a continuation. Push out pending record */
209 if (dump_record(dsp, NULL, 0) != 0)
210 return (SET_ERROR(EINTR));
211 dsp->dsa_pending_op = PENDING_NONE;
212 }
213 }
214 /* create a FREE record and make it pending */
215 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
216 dsp->dsa_drr->drr_type = DRR_FREE;
217 drrf->drr_object = object;
218 drrf->drr_offset = offset;
219 drrf->drr_length = length;
220 drrf->drr_toguid = dsp->dsa_toguid;
221 if (length == -1ULL) {
222 if (dump_record(dsp, NULL, 0) != 0)
223 return (SET_ERROR(EINTR));
224 } else {
225 dsp->dsa_pending_op = PENDING_FREE;
226 }
227
228 return (0);
229 }
230
231 static int
232 dump_write(dmu_sendarg_t *dsp, dmu_object_type_t type,
233 uint64_t object, uint64_t offset, int blksz, const blkptr_t *bp, void *data)
234 {
235 struct drr_write *drrw = &(dsp->dsa_drr->drr_u.drr_write);
236
237 /*
238 * We send data in increasing object, offset order.
239 * See comment in dump_free() for details.
240 */
241 ASSERT(object > dsp->dsa_last_data_object ||
242 (object == dsp->dsa_last_data_object &&
243 offset > dsp->dsa_last_data_offset));
244 dsp->dsa_last_data_object = object;
245 dsp->dsa_last_data_offset = offset + blksz - 1;
246
247 /*
248 * If there is any kind of pending aggregation (currently either
249 * a grouping of free objects or free blocks), push it out to
250 * the stream, since aggregation can't be done across operations
251 * of different types.
252 */
253 if (dsp->dsa_pending_op != PENDING_NONE) {
254 if (dump_record(dsp, NULL, 0) != 0)
255 return (SET_ERROR(EINTR));
256 dsp->dsa_pending_op = PENDING_NONE;
257 }
258 /* write a WRITE record */
259 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
260 dsp->dsa_drr->drr_type = DRR_WRITE;
261 drrw->drr_object = object;
262 drrw->drr_type = type;
263 drrw->drr_offset = offset;
264 drrw->drr_length = blksz;
265 drrw->drr_toguid = dsp->dsa_toguid;
266 if (bp == NULL || BP_IS_EMBEDDED(bp)) {
267 /*
268 * There's no pre-computed checksum for partial-block
269 * writes or embedded BP's, so (like
270 * fletcher4-checkummed blocks) userland will have to
271 * compute a dedup-capable checksum itself.
272 */
273 drrw->drr_checksumtype = ZIO_CHECKSUM_OFF;
274 } else {
275 drrw->drr_checksumtype = BP_GET_CHECKSUM(bp);
276 if (zio_checksum_table[drrw->drr_checksumtype].ci_flags &
277 ZCHECKSUM_FLAG_DEDUP)
278 drrw->drr_checksumflags |= DRR_CHECKSUM_DEDUP;
279 DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp));
280 DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp));
281 DDK_SET_COMPRESS(&drrw->drr_key, BP_GET_COMPRESS(bp));
282 drrw->drr_key.ddk_cksum = bp->blk_cksum;
283 }
284
285 if (dump_record(dsp, data, blksz) != 0)
286 return (SET_ERROR(EINTR));
287 return (0);
288 }
289
290 static int
291 dump_write_embedded(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
292 int blksz, const blkptr_t *bp)
293 {
294 char buf[BPE_PAYLOAD_SIZE];
295 struct drr_write_embedded *drrw =
296 &(dsp->dsa_drr->drr_u.drr_write_embedded);
297
298 if (dsp->dsa_pending_op != PENDING_NONE) {
299 if (dump_record(dsp, NULL, 0) != 0)
300 return (EINTR);
301 dsp->dsa_pending_op = PENDING_NONE;
302 }
303
304 ASSERT(BP_IS_EMBEDDED(bp));
305
306 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
307 dsp->dsa_drr->drr_type = DRR_WRITE_EMBEDDED;
308 drrw->drr_object = object;
309 drrw->drr_offset = offset;
310 drrw->drr_length = blksz;
311 drrw->drr_toguid = dsp->dsa_toguid;
312 drrw->drr_compression = BP_GET_COMPRESS(bp);
313 drrw->drr_etype = BPE_GET_ETYPE(bp);
314 drrw->drr_lsize = BPE_GET_LSIZE(bp);
315 drrw->drr_psize = BPE_GET_PSIZE(bp);
316
317 decode_embedded_bp_compressed(bp, buf);
318
319 if (dump_record(dsp, buf, P2ROUNDUP(drrw->drr_psize, 8)) != 0)
320 return (EINTR);
321 return (0);
322 }
323
324 static int
325 dump_spill(dmu_sendarg_t *dsp, uint64_t object, int blksz, void *data)
326 {
327 struct drr_spill *drrs = &(dsp->dsa_drr->drr_u.drr_spill);
328
329 if (dsp->dsa_pending_op != PENDING_NONE) {
330 if (dump_record(dsp, NULL, 0) != 0)
331 return (SET_ERROR(EINTR));
332 dsp->dsa_pending_op = PENDING_NONE;
333 }
334
335 /* write a SPILL record */
336 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
337 dsp->dsa_drr->drr_type = DRR_SPILL;
338 drrs->drr_object = object;
339 drrs->drr_length = blksz;
340 drrs->drr_toguid = dsp->dsa_toguid;
341
342 if (dump_record(dsp, data, blksz) != 0)
343 return (SET_ERROR(EINTR));
344 return (0);
345 }
346
347 static int
348 dump_freeobjects(dmu_sendarg_t *dsp, uint64_t firstobj, uint64_t numobjs)
349 {
350 struct drr_freeobjects *drrfo = &(dsp->dsa_drr->drr_u.drr_freeobjects);
351
352 /*
353 * If there is a pending op, but it's not PENDING_FREEOBJECTS,
354 * push it out, since free block aggregation can only be done for
355 * blocks of the same type (i.e., DRR_FREE records can only be
356 * aggregated with other DRR_FREE records. DRR_FREEOBJECTS records
357 * can only be aggregated with other DRR_FREEOBJECTS records.
358 */
359 if (dsp->dsa_pending_op != PENDING_NONE &&
360 dsp->dsa_pending_op != PENDING_FREEOBJECTS) {
361 if (dump_record(dsp, NULL, 0) != 0)
362 return (SET_ERROR(EINTR));
363 dsp->dsa_pending_op = PENDING_NONE;
364 }
365 if (dsp->dsa_pending_op == PENDING_FREEOBJECTS) {
366 /*
367 * See whether this free object array can be aggregated
368 * with pending one
369 */
370 if (drrfo->drr_firstobj + drrfo->drr_numobjs == firstobj) {
371 drrfo->drr_numobjs += numobjs;
372 return (0);
373 } else {
374 /* can't be aggregated. Push out pending record */
375 if (dump_record(dsp, NULL, 0) != 0)
376 return (SET_ERROR(EINTR));
377 dsp->dsa_pending_op = PENDING_NONE;
378 }
379 }
380
381 /* write a FREEOBJECTS record */
382 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
383 dsp->dsa_drr->drr_type = DRR_FREEOBJECTS;
384 drrfo->drr_firstobj = firstobj;
385 drrfo->drr_numobjs = numobjs;
386 drrfo->drr_toguid = dsp->dsa_toguid;
387
388 dsp->dsa_pending_op = PENDING_FREEOBJECTS;
389
390 return (0);
391 }
392
393 static int
394 dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp)
395 {
396 struct drr_object *drro = &(dsp->dsa_drr->drr_u.drr_object);
397
398 if (object < dsp->dsa_resume_object) {
399 /*
400 * Note: when resuming, we will visit all the dnodes in
401 * the block of dnodes that we are resuming from. In
402 * this case it's unnecessary to send the dnodes prior to
403 * the one we are resuming from. We should be at most one
404 * block's worth of dnodes behind the resume point.
405 */
406 ASSERT3U(dsp->dsa_resume_object - object, <,
407 1 << (DNODE_BLOCK_SHIFT - DNODE_SHIFT));
408 return (0);
409 }
410
411 if (dnp == NULL || dnp->dn_type == DMU_OT_NONE)
412 return (dump_freeobjects(dsp, object, 1));
413
414 if (dsp->dsa_pending_op != PENDING_NONE) {
415 if (dump_record(dsp, NULL, 0) != 0)
416 return (SET_ERROR(EINTR));
417 dsp->dsa_pending_op = PENDING_NONE;
418 }
419
420 /* write an OBJECT record */
421 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
422 dsp->dsa_drr->drr_type = DRR_OBJECT;
423 drro->drr_object = object;
424 drro->drr_type = dnp->dn_type;
425 drro->drr_bonustype = dnp->dn_bonustype;
426 drro->drr_blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
427 drro->drr_bonuslen = dnp->dn_bonuslen;
428 drro->drr_checksumtype = dnp->dn_checksum;
429 drro->drr_compress = dnp->dn_compress;
430 drro->drr_toguid = dsp->dsa_toguid;
431
432 if (!(dsp->dsa_featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
433 drro->drr_blksz > SPA_OLD_MAXBLOCKSIZE)
434 drro->drr_blksz = SPA_OLD_MAXBLOCKSIZE;
435
436 if (dump_record(dsp, DN_BONUS(dnp),
437 P2ROUNDUP(dnp->dn_bonuslen, 8)) != 0) {
438 return (SET_ERROR(EINTR));
439 }
440
441 /* Free anything past the end of the file. */
442 if (dump_free(dsp, object, (dnp->dn_maxblkid + 1) *
443 (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT), -1ULL) != 0)
444 return (SET_ERROR(EINTR));
445 if (dsp->dsa_err != 0)
446 return (SET_ERROR(EINTR));
447 return (0);
448 }
449
450 static boolean_t
451 backup_do_embed(dmu_sendarg_t *dsp, const blkptr_t *bp)
452 {
453 if (!BP_IS_EMBEDDED(bp))
454 return (B_FALSE);
455
456 /*
457 * Compression function must be legacy, or explicitly enabled.
458 */
459 if ((BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_LEGACY_FUNCTIONS &&
460 !(dsp->dsa_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4)))
461 return (B_FALSE);
462
463 /*
464 * Embed type must be explicitly enabled.
465 */
466 switch (BPE_GET_ETYPE(bp)) {
467 case BP_EMBEDDED_TYPE_DATA:
468 if (dsp->dsa_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)
469 return (B_TRUE);
470 break;
471 default:
472 return (B_FALSE);
473 }
474 return (B_FALSE);
475 }
476
477 /*
478 * This is the callback function to traverse_dataset that acts as the worker
479 * thread for dmu_send_impl.
480 */
481 /*ARGSUSED*/
482 static int
483 send_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
484 const zbookmark_phys_t *zb, const struct dnode_phys *dnp, void *arg)
485 {
486 struct send_thread_arg *sta = arg;
487 struct send_block_record *record;
488 uint64_t record_size;
489 int err = 0;
490
491 ASSERT(zb->zb_object == DMU_META_DNODE_OBJECT ||
492 zb->zb_object >= sta->resume.zb_object);
493
494 if (sta->cancel)
495 return (SET_ERROR(EINTR));
496
497 if (bp == NULL) {
498 ASSERT3U(zb->zb_level, ==, ZB_DNODE_LEVEL);
499 return (0);
500 } else if (zb->zb_level < 0) {
501 return (0);
502 }
503
504 record = kmem_zalloc(sizeof (struct send_block_record), KM_SLEEP);
505 record->eos_marker = B_FALSE;
506 record->bp = *bp;
507 record->zb = *zb;
508 record->indblkshift = dnp->dn_indblkshift;
509 record->datablkszsec = dnp->dn_datablkszsec;
510 record_size = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
511 bqueue_enqueue(&sta->q, record, record_size);
512
513 return (err);
514 }
515
516 /*
517 * This function kicks off the traverse_dataset. It also handles setting the
518 * error code of the thread in case something goes wrong, and pushes the End of
519 * Stream record when the traverse_dataset call has finished. If there is no
520 * dataset to traverse, the thread immediately pushes End of Stream marker.
521 */
522 static void
523 send_traverse_thread(void *arg)
524 {
525 struct send_thread_arg *st_arg = arg;
526 int err;
527 struct send_block_record *data;
528
529 if (st_arg->ds != NULL) {
530 err = traverse_dataset_resume(st_arg->ds,
531 st_arg->fromtxg, &st_arg->resume,
532 st_arg->flags, send_cb, st_arg);
533
534 if (err != EINTR)
535 st_arg->error_code = err;
536 }
537 data = kmem_zalloc(sizeof (*data), KM_SLEEP);
538 data->eos_marker = B_TRUE;
539 bqueue_enqueue(&st_arg->q, data, 1);
540 }
541
542 /*
543 * This function actually handles figuring out what kind of record needs to be
544 * dumped, reading the data (which has hopefully been prefetched), and calling
545 * the appropriate helper function.
546 */
547 static int
548 do_dump(dmu_sendarg_t *dsa, struct send_block_record *data)
549 {
550 dsl_dataset_t *ds = dmu_objset_ds(dsa->dsa_os);
551 const blkptr_t *bp = &data->bp;
552 const zbookmark_phys_t *zb = &data->zb;
553 uint8_t indblkshift = data->indblkshift;
554 uint16_t dblkszsec = data->datablkszsec;
555 spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
556 dmu_object_type_t type = bp ? BP_GET_TYPE(bp) : DMU_OT_NONE;
557 int err = 0;
558
559 ASSERT3U(zb->zb_level, >=, 0);
560
561 ASSERT(zb->zb_object == DMU_META_DNODE_OBJECT ||
562 zb->zb_object >= dsa->dsa_resume_object);
563
564 if (zb->zb_object != DMU_META_DNODE_OBJECT &&
565 DMU_OBJECT_IS_SPECIAL(zb->zb_object)) {
566 return (0);
567 } else if (BP_IS_HOLE(bp) &&
568 zb->zb_object == DMU_META_DNODE_OBJECT) {
569 uint64_t span = BP_SPAN(dblkszsec, indblkshift, zb->zb_level);
570 uint64_t dnobj = (zb->zb_blkid * span) >> DNODE_SHIFT;
571 err = dump_freeobjects(dsa, dnobj, span >> DNODE_SHIFT);
572 } else if (BP_IS_HOLE(bp)) {
573 uint64_t span = BP_SPAN(dblkszsec, indblkshift, zb->zb_level);
574 uint64_t offset = zb->zb_blkid * span;
575 err = dump_free(dsa, zb->zb_object, offset, span);
576 } else if (zb->zb_level > 0 || type == DMU_OT_OBJSET) {
577 return (0);
578 } else if (type == DMU_OT_DNODE) {
579 int blksz = BP_GET_LSIZE(bp);
580 arc_flags_t aflags = ARC_FLAG_WAIT;
581 arc_buf_t *abuf;
582
583 ASSERT0(zb->zb_level);
584
585 if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
586 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
587 &aflags, zb) != 0)
588 return (SET_ERROR(EIO));
589
590 dnode_phys_t *blk = abuf->b_data;
591 uint64_t dnobj = zb->zb_blkid * (blksz >> DNODE_SHIFT);
592 for (int i = 0; i < blksz >> DNODE_SHIFT; i++) {
593 err = dump_dnode(dsa, dnobj + i, blk + i);
594 if (err != 0)
595 break;
596 }
597 (void) arc_buf_remove_ref(abuf, &abuf);
598 } else if (type == DMU_OT_SA) {
599 arc_flags_t aflags = ARC_FLAG_WAIT;
600 arc_buf_t *abuf;
601 int blksz = BP_GET_LSIZE(bp);
602
603 if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
604 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
605 &aflags, zb) != 0)
606 return (SET_ERROR(EIO));
607
608 err = dump_spill(dsa, zb->zb_object, blksz, abuf->b_data);
609 (void) arc_buf_remove_ref(abuf, &abuf);
610 } else if (backup_do_embed(dsa, bp)) {
611 /* it's an embedded level-0 block of a regular object */
612 int blksz = dblkszsec << SPA_MINBLOCKSHIFT;
613 ASSERT0(zb->zb_level);
614 err = dump_write_embedded(dsa, zb->zb_object,
615 zb->zb_blkid * blksz, blksz, bp);
616 } else {
617 /* it's a level-0 block of a regular object */
618 arc_flags_t aflags = ARC_FLAG_WAIT;
619 arc_buf_t *abuf;
620 int blksz = dblkszsec << SPA_MINBLOCKSHIFT;
621 uint64_t offset;
622
623 ASSERT0(zb->zb_level);
624 ASSERT(zb->zb_object > dsa->dsa_resume_object ||
625 (zb->zb_object == dsa->dsa_resume_object &&
626 zb->zb_blkid * blksz >= dsa->dsa_resume_offset));
627
628 if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
629 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
630 &aflags, zb) != 0) {
631 if (zfs_send_corrupt_data) {
632 /* Send a block filled with 0x"zfs badd bloc" */
633 abuf = arc_buf_alloc(spa, blksz, &abuf,
634 ARC_BUFC_DATA);
635 uint64_t *ptr;
636 for (ptr = abuf->b_data;
637 (char *)ptr < (char *)abuf->b_data + blksz;
638 ptr++)
639 *ptr = 0x2f5baddb10cULL;
640 } else {
641 return (SET_ERROR(EIO));
642 }
643 }
644
645 offset = zb->zb_blkid * blksz;
646
647 if (!(dsa->dsa_featureflags &
648 DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
649 blksz > SPA_OLD_MAXBLOCKSIZE) {
650 char *buf = abuf->b_data;
651 while (blksz > 0 && err == 0) {
652 int n = MIN(blksz, SPA_OLD_MAXBLOCKSIZE);
653 err = dump_write(dsa, type, zb->zb_object,
654 offset, n, NULL, buf);
655 offset += n;
656 buf += n;
657 blksz -= n;
658 }
659 } else {
660 err = dump_write(dsa, type, zb->zb_object,
661 offset, blksz, bp, abuf->b_data);
662 }
663 (void) arc_buf_remove_ref(abuf, &abuf);
664 }
665
666 ASSERT(err == 0 || err == EINTR);
667 return (err);
668 }
669
670 /*
671 * Pop the new data off the queue, and free the old data.
672 */
673 static struct send_block_record *
674 get_next_record(bqueue_t *bq, struct send_block_record *data)
675 {
676 struct send_block_record *tmp = bqueue_dequeue(bq);
677 kmem_free(data, sizeof (*data));
678 return (tmp);
679 }
680
681 /*
682 * Actually do the bulk of the work in a zfs send.
683 *
684 * Note: Releases dp using the specified tag.
685 */
686 static int
687 dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds,
688 zfs_bookmark_phys_t *ancestor_zb,
689 boolean_t is_clone, boolean_t embedok, boolean_t large_block_ok, int outfd,
690 uint64_t resumeobj, uint64_t resumeoff,
691 vnode_t *vp, offset_t *off)
692 {
693 objset_t *os;
694 dmu_replay_record_t *drr;
695 dmu_sendarg_t *dsp;
696 int err;
697 uint64_t fromtxg = 0;
698 uint64_t featureflags = 0;
699 struct send_thread_arg to_arg = { 0 };
700
701 err = dmu_objset_from_ds(to_ds, &os);
702 if (err != 0) {
703 dsl_pool_rele(dp, tag);
704 return (err);
705 }
706
707 drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
708 drr->drr_type = DRR_BEGIN;
709 drr->drr_u.drr_begin.drr_magic = DMU_BACKUP_MAGIC;
710 DMU_SET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo,
711 DMU_SUBSTREAM);
712
713 #ifdef _KERNEL
714 if (dmu_objset_type(os) == DMU_OST_ZFS) {
715 uint64_t version;
716 if (zfs_get_zplprop(os, ZFS_PROP_VERSION, &version) != 0) {
717 kmem_free(drr, sizeof (dmu_replay_record_t));
718 dsl_pool_rele(dp, tag);
719 return (SET_ERROR(EINVAL));
720 }
721 if (version >= ZPL_VERSION_SA) {
722 featureflags |= DMU_BACKUP_FEATURE_SA_SPILL;
723 }
724 }
725 #endif
726
727 if (large_block_ok && to_ds->ds_feature_inuse[SPA_FEATURE_LARGE_BLOCKS])
728 featureflags |= DMU_BACKUP_FEATURE_LARGE_BLOCKS;
729 if (embedok &&
730 spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) {
731 featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA;
732 if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
733 featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA_LZ4;
734 }
735
736 if (resumeobj != 0 || resumeoff != 0) {
737 featureflags |= DMU_BACKUP_FEATURE_RESUMING;
738 }
739
740 DMU_SET_FEATUREFLAGS(drr->drr_u.drr_begin.drr_versioninfo,
741 featureflags);
742
743 drr->drr_u.drr_begin.drr_creation_time =
744 dsl_dataset_phys(to_ds)->ds_creation_time;
745 drr->drr_u.drr_begin.drr_type = dmu_objset_type(os);
746 if (is_clone)
747 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CLONE;
748 drr->drr_u.drr_begin.drr_toguid = dsl_dataset_phys(to_ds)->ds_guid;
749 if (dsl_dataset_phys(to_ds)->ds_flags & DS_FLAG_CI_DATASET)
750 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CI_DATA;
751 if (zfs_send_set_freerecords_bit)
752 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_FREERECORDS;
753
754 if (ancestor_zb != NULL) {
755 drr->drr_u.drr_begin.drr_fromguid =
756 ancestor_zb->zbm_guid;
757 fromtxg = ancestor_zb->zbm_creation_txg;
758 }
759 dsl_dataset_name(to_ds, drr->drr_u.drr_begin.drr_toname);
760 if (!to_ds->ds_is_snapshot) {
761 (void) strlcat(drr->drr_u.drr_begin.drr_toname, "@--head--",
762 sizeof (drr->drr_u.drr_begin.drr_toname));
763 }
764
765 dsp = kmem_zalloc(sizeof (dmu_sendarg_t), KM_SLEEP);
766
767 dsp->dsa_drr = drr;
768 dsp->dsa_vp = vp;
769 dsp->dsa_outfd = outfd;
770 dsp->dsa_proc = curproc;
771 dsp->dsa_os = os;
772 dsp->dsa_off = off;
773 dsp->dsa_toguid = dsl_dataset_phys(to_ds)->ds_guid;
774 dsp->dsa_pending_op = PENDING_NONE;
775 dsp->dsa_featureflags = featureflags;
776 dsp->dsa_resume_object = resumeobj;
777 dsp->dsa_resume_offset = resumeoff;
778
779 mutex_enter(&to_ds->ds_sendstream_lock);
780 list_insert_head(&to_ds->ds_sendstreams, dsp);
781 mutex_exit(&to_ds->ds_sendstream_lock);
782
783 dsl_dataset_long_hold(to_ds, FTAG);
784 dsl_pool_rele(dp, tag);
785
786 void *payload = NULL;
787 size_t payload_len = 0;
788 if (resumeobj != 0 || resumeoff != 0) {
789 dmu_object_info_t to_doi;
790 err = dmu_object_info(os, resumeobj, &to_doi);
791 if (err != 0)
792 goto out;
793 SET_BOOKMARK(&to_arg.resume, to_ds->ds_object, resumeobj, 0,
794 resumeoff / to_doi.doi_data_block_size);
795
796 nvlist_t *nvl = fnvlist_alloc();
797 fnvlist_add_uint64(nvl, "resume_object", resumeobj);
798 fnvlist_add_uint64(nvl, "resume_offset", resumeoff);
799 payload = fnvlist_pack(nvl, &payload_len);
800 drr->drr_payloadlen = payload_len;
801 fnvlist_free(nvl);
802 }
803
804 err = dump_record(dsp, payload, payload_len);
805 fnvlist_pack_free(payload, payload_len);
806 if (err != 0) {
807 err = dsp->dsa_err;
808 goto out;
809 }
810
811 err = bqueue_init(&to_arg.q, zfs_send_queue_length,
812 offsetof(struct send_block_record, ln));
813 to_arg.error_code = 0;
814 to_arg.cancel = B_FALSE;
815 to_arg.ds = to_ds;
816 to_arg.fromtxg = fromtxg;
817 to_arg.flags = TRAVERSE_PRE | TRAVERSE_PREFETCH;
818 (void) thread_create(NULL, 0, send_traverse_thread, &to_arg, 0, curproc,
819 TS_RUN, minclsyspri);
820
821 struct send_block_record *to_data;
822 to_data = bqueue_dequeue(&to_arg.q);
823
824 while (!to_data->eos_marker && err == 0) {
825 err = do_dump(dsp, to_data);
826 to_data = get_next_record(&to_arg.q, to_data);
827 if (issig(JUSTLOOKING) && issig(FORREAL))
828 err = EINTR;
829 }
830
831 if (err != 0) {
832 to_arg.cancel = B_TRUE;
833 while (!to_data->eos_marker) {
834 to_data = get_next_record(&to_arg.q, to_data);
835 }
836 }
837 kmem_free(to_data, sizeof (*to_data));
838
839 bqueue_destroy(&to_arg.q);
840
841 if (err == 0 && to_arg.error_code != 0)
842 err = to_arg.error_code;
843
844 if (err != 0)
845 goto out;
846
847 if (dsp->dsa_pending_op != PENDING_NONE)
848 if (dump_record(dsp, NULL, 0) != 0)
849 err = SET_ERROR(EINTR);
850
851 if (err != 0) {
852 if (err == EINTR && dsp->dsa_err != 0)
853 err = dsp->dsa_err;
854 goto out;
855 }
856
857 bzero(drr, sizeof (dmu_replay_record_t));
858 drr->drr_type = DRR_END;
859 drr->drr_u.drr_end.drr_checksum = dsp->dsa_zc;
860 drr->drr_u.drr_end.drr_toguid = dsp->dsa_toguid;
861
862 if (dump_record(dsp, NULL, 0) != 0)
863 err = dsp->dsa_err;
864
865 out:
866 mutex_enter(&to_ds->ds_sendstream_lock);
867 list_remove(&to_ds->ds_sendstreams, dsp);
868 mutex_exit(&to_ds->ds_sendstream_lock);
869
870 kmem_free(drr, sizeof (dmu_replay_record_t));
871 kmem_free(dsp, sizeof (dmu_sendarg_t));
872
873 dsl_dataset_long_rele(to_ds, FTAG);
874
875 return (err);
876 }
877
878 int
879 dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
880 boolean_t embedok, boolean_t large_block_ok,
881 int outfd, vnode_t *vp, offset_t *off)
882 {
883 dsl_pool_t *dp;
884 dsl_dataset_t *ds;
885 dsl_dataset_t *fromds = NULL;
886 int err;
887
888 err = dsl_pool_hold(pool, FTAG, &dp);
889 if (err != 0)
890 return (err);
891
892 err = dsl_dataset_hold_obj(dp, tosnap, FTAG, &ds);
893 if (err != 0) {
894 dsl_pool_rele(dp, FTAG);
895 return (err);
896 }
897
898 if (fromsnap != 0) {
899 zfs_bookmark_phys_t zb;
900 boolean_t is_clone;
901
902 err = dsl_dataset_hold_obj(dp, fromsnap, FTAG, &fromds);
903 if (err != 0) {
904 dsl_dataset_rele(ds, FTAG);
905 dsl_pool_rele(dp, FTAG);
906 return (err);
907 }
908 if (!dsl_dataset_is_before(ds, fromds, 0))
909 err = SET_ERROR(EXDEV);
910 zb.zbm_creation_time =
911 dsl_dataset_phys(fromds)->ds_creation_time;
912 zb.zbm_creation_txg = dsl_dataset_phys(fromds)->ds_creation_txg;
913 zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
914 is_clone = (fromds->ds_dir != ds->ds_dir);
915 dsl_dataset_rele(fromds, FTAG);
916 err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
917 embedok, large_block_ok, outfd, 0, 0, vp, off);
918 } else {
919 err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
920 embedok, large_block_ok, outfd, 0, 0, vp, off);
921 }
922 dsl_dataset_rele(ds, FTAG);
923 return (err);
924 }
925
926 int
927 dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
928 boolean_t large_block_ok, int outfd, uint64_t resumeobj, uint64_t resumeoff,
929 vnode_t *vp, offset_t *off)
930 {
931 dsl_pool_t *dp;
932 dsl_dataset_t *ds;
933 int err;
934 boolean_t owned = B_FALSE;
935
936 if (fromsnap != NULL && strpbrk(fromsnap, "@#") == NULL)
937 return (SET_ERROR(EINVAL));
938
939 err = dsl_pool_hold(tosnap, FTAG, &dp);
940 if (err != 0)
941 return (err);
942
943 if (strchr(tosnap, '@') == NULL && spa_writeable(dp->dp_spa)) {
944 /*
945 * We are sending a filesystem or volume. Ensure
946 * that it doesn't change by owning the dataset.
947 */
948 err = dsl_dataset_own(dp, tosnap, FTAG, &ds);
949 owned = B_TRUE;
950 } else {
951 err = dsl_dataset_hold(dp, tosnap, FTAG, &ds);
952 }
953 if (err != 0) {
954 dsl_pool_rele(dp, FTAG);
955 return (err);
956 }
957
958 if (fromsnap != NULL) {
959 zfs_bookmark_phys_t zb;
960 boolean_t is_clone = B_FALSE;
961 int fsnamelen = strchr(tosnap, '@') - tosnap;
962
963 /*
964 * If the fromsnap is in a different filesystem, then
965 * mark the send stream as a clone.
966 */
967 if (strncmp(tosnap, fromsnap, fsnamelen) != 0 ||
968 (fromsnap[fsnamelen] != '@' &&
969 fromsnap[fsnamelen] != '#')) {
970 is_clone = B_TRUE;
971 }
972
973 if (strchr(fromsnap, '@')) {
974 dsl_dataset_t *fromds;
975 err = dsl_dataset_hold(dp, fromsnap, FTAG, &fromds);
976 if (err == 0) {
977 if (!dsl_dataset_is_before(ds, fromds, 0))
978 err = SET_ERROR(EXDEV);
979 zb.zbm_creation_time =
980 dsl_dataset_phys(fromds)->ds_creation_time;
981 zb.zbm_creation_txg =
982 dsl_dataset_phys(fromds)->ds_creation_txg;
983 zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
984 is_clone = (ds->ds_dir != fromds->ds_dir);
985 dsl_dataset_rele(fromds, FTAG);
986 }
987 } else {
988 err = dsl_bookmark_lookup(dp, fromsnap, ds, &zb);
989 }
990 if (err != 0) {
991 dsl_dataset_rele(ds, FTAG);
992 dsl_pool_rele(dp, FTAG);
993 return (err);
994 }
995 err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
996 embedok, large_block_ok,
997 outfd, resumeobj, resumeoff, vp, off);
998 } else {
999 err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
1000 embedok, large_block_ok,
1001 outfd, resumeobj, resumeoff, vp, off);
1002 }
1003 if (owned)
1004 dsl_dataset_disown(ds, FTAG);
1005 else
1006 dsl_dataset_rele(ds, FTAG);
1007 return (err);
1008 }
1009
1010 static int
1011 dmu_adjust_send_estimate_for_indirects(dsl_dataset_t *ds, uint64_t size,
1012 uint64_t *sizep)
1013 {
1014 int err;
1015 /*
1016 * Assume that space (both on-disk and in-stream) is dominated by
1017 * data. We will adjust for indirect blocks and the copies property,
1018 * but ignore per-object space used (eg, dnodes and DRR_OBJECT records).
1019 */
1020
1021 /*
1022 * Subtract out approximate space used by indirect blocks.
1023 * Assume most space is used by data blocks (non-indirect, non-dnode).
1024 * Assume all blocks are recordsize. Assume ditto blocks and
1025 * internal fragmentation counter out compression.
1026 *
1027 * Therefore, space used by indirect blocks is sizeof(blkptr_t) per
1028 * block, which we observe in practice.
1029 */
1030 uint64_t recordsize;
1031 err = dsl_prop_get_int_ds(ds, "recordsize", &recordsize);
1032 if (err != 0)
1033 return (err);
1034 size -= size / recordsize * sizeof (blkptr_t);
1035
1036 /* Add in the space for the record associated with each block. */
1037 size += size / recordsize * sizeof (dmu_replay_record_t);
1038
1039 *sizep = size;
1040
1041 return (0);
1042 }
1043
1044 int
1045 dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
1046 {
1047 dsl_pool_t *dp = ds->ds_dir->dd_pool;
1048 int err;
1049 uint64_t size;
1050
1051 ASSERT(dsl_pool_config_held(dp));
1052
1053 /* tosnap must be a snapshot */
1054 if (!ds->ds_is_snapshot)
1055 return (SET_ERROR(EINVAL));
1056
1057 /* fromsnap, if provided, must be a snapshot */
1058 if (fromds != NULL && !fromds->ds_is_snapshot)
1059 return (SET_ERROR(EINVAL));
1060
1061 /*
1062 * fromsnap must be an earlier snapshot from the same fs as tosnap,
1063 * or the origin's fs.
1064 */
1065 if (fromds != NULL && !dsl_dataset_is_before(ds, fromds, 0))
1066 return (SET_ERROR(EXDEV));
1067
1068 /* Get uncompressed size estimate of changed data. */
1069 if (fromds == NULL) {
1070 size = dsl_dataset_phys(ds)->ds_uncompressed_bytes;
1071 } else {
1072 uint64_t used, comp;
1073 err = dsl_dataset_space_written(fromds, ds,
1074 &used, &comp, &size);
1075 if (err != 0)
1076 return (err);
1077 }
1078
1079 err = dmu_adjust_send_estimate_for_indirects(ds, size, sizep);
1080 return (err);
1081 }
1082
1083 /*
1084 * Simple callback used to traverse the blocks of a snapshot and sum their
1085 * uncompressed size
1086 */
1087 /* ARGSUSED */
1088 static int
1089 dmu_calculate_send_traversal(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
1090 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
1091 {
1092 uint64_t *spaceptr = arg;
1093 if (bp != NULL && !BP_IS_HOLE(bp)) {
1094 *spaceptr += BP_GET_UCSIZE(bp);
1095 }
1096 return (0);
1097 }
1098
1099 /*
1100 * Given a desination snapshot and a TXG, calculate the approximate size of a
1101 * send stream sent from that TXG. from_txg may be zero, indicating that the
1102 * whole snapshot will be sent.
1103 */
1104 int
1105 dmu_send_estimate_from_txg(dsl_dataset_t *ds, uint64_t from_txg,
1106 uint64_t *sizep)
1107 {
1108 dsl_pool_t *dp = ds->ds_dir->dd_pool;
1109 int err;
1110 uint64_t size = 0;
1111
1112 ASSERT(dsl_pool_config_held(dp));
1113
1114 /* tosnap must be a snapshot */
1115 if (!dsl_dataset_is_snapshot(ds))
1116 return (SET_ERROR(EINVAL));
1117
1118 /* verify that from_txg is before the provided snapshot was taken */
1119 if (from_txg >= dsl_dataset_phys(ds)->ds_creation_txg) {
1120 return (SET_ERROR(EXDEV));
1121 }
1122
1123 /*
1124 * traverse the blocks of the snapshot with birth times after
1125 * from_txg, summing their uncompressed size
1126 */
1127 err = traverse_dataset(ds, from_txg, TRAVERSE_POST,
1128 dmu_calculate_send_traversal, &size);
1129 if (err)
1130 return (err);
1131
1132 err = dmu_adjust_send_estimate_for_indirects(ds, size, sizep);
1133 return (err);
1134 }
1135
1136 typedef struct dmu_recv_begin_arg {
1137 const char *drba_origin;
1138 dmu_recv_cookie_t *drba_cookie;
1139 cred_t *drba_cred;
1140 uint64_t drba_snapobj;
1141 } dmu_recv_begin_arg_t;
1142
1143 static int
1144 recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
1145 uint64_t fromguid)
1146 {
1147 uint64_t val;
1148 int error;
1149 dsl_pool_t *dp = ds->ds_dir->dd_pool;
1150
1151 /* temporary clone name must not exist */
1152 error = zap_lookup(dp->dp_meta_objset,
1153 dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, recv_clone_name,
1154 8, 1, &val);
1155 if (error != ENOENT)
1156 return (error == 0 ? EBUSY : error);
1157
1158 /* new snapshot name must not exist */
1159 error = zap_lookup(dp->dp_meta_objset,
1160 dsl_dataset_phys(ds)->ds_snapnames_zapobj,
1161 drba->drba_cookie->drc_tosnap, 8, 1, &val);
1162 if (error != ENOENT)
1163 return (error == 0 ? EEXIST : error);
1164
1165 /*
1166 * Check snapshot limit before receiving. We'll recheck again at the
1167 * end, but might as well abort before receiving if we're already over
1168 * the limit.
1169 *
1170 * Note that we do not check the file system limit with
1171 * dsl_dir_fscount_check because the temporary %clones don't count
1172 * against that limit.
1173 */
1174 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, ZFS_PROP_SNAPSHOT_LIMIT,
1175 NULL, drba->drba_cred);
1176 if (error != 0)
1177 return (error);
1178
1179 if (fromguid != 0) {
1180 dsl_dataset_t *snap;
1181 uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
1182
1183 /* Find snapshot in this dir that matches fromguid. */
1184 while (obj != 0) {
1185 error = dsl_dataset_hold_obj(dp, obj, FTAG,
1186 &snap);
1187 if (error != 0)
1188 return (SET_ERROR(ENODEV));
1189 if (snap->ds_dir != ds->ds_dir) {
1190 dsl_dataset_rele(snap, FTAG);
1191 return (SET_ERROR(ENODEV));
1192 }
1193 if (dsl_dataset_phys(snap)->ds_guid == fromguid)
1194 break;
1195 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
1196 dsl_dataset_rele(snap, FTAG);
1197 }
1198 if (obj == 0)
1199 return (SET_ERROR(ENODEV));
1200
1201 if (drba->drba_cookie->drc_force) {
1202 drba->drba_snapobj = obj;
1203 } else {
1204 /*
1205 * If we are not forcing, there must be no
1206 * changes since fromsnap.
1207 */
1208 if (dsl_dataset_modified_since_snap(ds, snap)) {
1209 dsl_dataset_rele(snap, FTAG);
1210 return (SET_ERROR(ETXTBSY));
1211 }
1212 drba->drba_snapobj = ds->ds_prev->ds_object;
1213 }
1214
1215 dsl_dataset_rele(snap, FTAG);
1216 } else {
1217 /* if full, then must be forced */
1218 if (!drba->drba_cookie->drc_force)
1219 return (SET_ERROR(EEXIST));
1220 /* start from $ORIGIN@$ORIGIN, if supported */
1221 drba->drba_snapobj = dp->dp_origin_snap != NULL ?
1222 dp->dp_origin_snap->ds_object : 0;
1223 }
1224
1225 return (0);
1226
1227 }
1228
1229 static int
1230 dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
1231 {
1232 dmu_recv_begin_arg_t *drba = arg;
1233 dsl_pool_t *dp = dmu_tx_pool(tx);
1234 struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
1235 uint64_t fromguid = drrb->drr_fromguid;
1236 int flags = drrb->drr_flags;
1237 int error;
1238 uint64_t featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
1239 dsl_dataset_t *ds;
1240 const char *tofs = drba->drba_cookie->drc_tofs;
1241
1242 /* already checked */
1243 ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
1244 ASSERT(!(featureflags & DMU_BACKUP_FEATURE_RESUMING));
1245
1246 if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
1247 DMU_COMPOUNDSTREAM ||
1248 drrb->drr_type >= DMU_OST_NUMTYPES ||
1249 ((flags & DRR_FLAG_CLONE) && drba->drba_origin == NULL))
1250 return (SET_ERROR(EINVAL));
1251
1252 /* Verify pool version supports SA if SA_SPILL feature set */
1253 if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
1254 spa_version(dp->dp_spa) < SPA_VERSION_SA)
1255 return (SET_ERROR(ENOTSUP));
1256
1257 if (drba->drba_cookie->drc_resumable &&
1258 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EXTENSIBLE_DATASET))
1259 return (SET_ERROR(ENOTSUP));
1260
1261 /*
1262 * The receiving code doesn't know how to translate a WRITE_EMBEDDED
1263 * record to a plan WRITE record, so the pool must have the
1264 * EMBEDDED_DATA feature enabled if the stream has WRITE_EMBEDDED
1265 * records. Same with WRITE_EMBEDDED records that use LZ4 compression.
1266 */
1267 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) &&
1268 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA))
1269 return (SET_ERROR(ENOTSUP));
1270 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4) &&
1271 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
1272 return (SET_ERROR(ENOTSUP));
1273
1274 /*
1275 * The receiving code doesn't know how to translate large blocks
1276 * to smaller ones, so the pool must have the LARGE_BLOCKS
1277 * feature enabled if the stream has LARGE_BLOCKS.
1278 */
1279 if ((featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
1280 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS))
1281 return (SET_ERROR(ENOTSUP));
1282
1283 error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
1284 if (error == 0) {
1285 /* target fs already exists; recv into temp clone */
1286
1287 /* Can't recv a clone into an existing fs */
1288 if (flags & DRR_FLAG_CLONE || drba->drba_origin) {
1289 dsl_dataset_rele(ds, FTAG);
1290 return (SET_ERROR(EINVAL));
1291 }
1292
1293 error = recv_begin_check_existing_impl(drba, ds, fromguid);
1294 dsl_dataset_rele(ds, FTAG);
1295 } else if (error == ENOENT) {
1296 /* target fs does not exist; must be a full backup or clone */
1297 char buf[MAXNAMELEN];
1298
1299 /*
1300 * If it's a non-clone incremental, we are missing the
1301 * target fs, so fail the recv.
1302 */
1303 if (fromguid != 0 && !(flags & DRR_FLAG_CLONE ||
1304 drba->drba_origin))
1305 return (SET_ERROR(ENOENT));
1306
1307 /*
1308 * If we're receiving a full send as a clone, and it doesn't
1309 * contain all the necessary free records and freeobject
1310 * records, reject it.
1311 */
1312 if (fromguid == 0 && drba->drba_origin &&
1313 !(flags & DRR_FLAG_FREERECORDS))
1314 return (SET_ERROR(EINVAL));
1315
1316 /* Open the parent of tofs */
1317 ASSERT3U(strlen(tofs), <, MAXNAMELEN);
1318 (void) strlcpy(buf, tofs, strrchr(tofs, '/') - tofs + 1);
1319 error = dsl_dataset_hold(dp, buf, FTAG, &ds);
1320 if (error != 0)
1321 return (error);
1322
1323 /*
1324 * Check filesystem and snapshot limits before receiving. We'll
1325 * recheck snapshot limits again at the end (we create the
1326 * filesystems and increment those counts during begin_sync).
1327 */
1328 error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
1329 ZFS_PROP_FILESYSTEM_LIMIT, NULL, drba->drba_cred);
1330 if (error != 0) {
1331 dsl_dataset_rele(ds, FTAG);
1332 return (error);
1333 }
1334
1335 error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
1336 ZFS_PROP_SNAPSHOT_LIMIT, NULL, drba->drba_cred);
1337 if (error != 0) {
1338 dsl_dataset_rele(ds, FTAG);
1339 return (error);
1340 }
1341
1342 if (drba->drba_origin != NULL) {
1343 dsl_dataset_t *origin;
1344 error = dsl_dataset_hold(dp, drba->drba_origin,
1345 FTAG, &origin);
1346 if (error != 0) {
1347 dsl_dataset_rele(ds, FTAG);
1348 return (error);
1349 }
1350 if (!origin->ds_is_snapshot) {
1351 dsl_dataset_rele(origin, FTAG);
1352 dsl_dataset_rele(ds, FTAG);
1353 return (SET_ERROR(EINVAL));
1354 }
1355 if (dsl_dataset_phys(origin)->ds_guid != fromguid &&
1356 fromguid != 0) {
1357 dsl_dataset_rele(origin, FTAG);
1358 dsl_dataset_rele(ds, FTAG);
1359 return (SET_ERROR(ENODEV));
1360 }
1361 dsl_dataset_rele(origin, FTAG);
1362 }
1363 dsl_dataset_rele(ds, FTAG);
1364 error = 0;
1365 }
1366 return (error);
1367 }
1368
1369 static void
1370 dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
1371 {
1372 dmu_recv_begin_arg_t *drba = arg;
1373 dsl_pool_t *dp = dmu_tx_pool(tx);
1374 objset_t *mos = dp->dp_meta_objset;
1375 struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
1376 const char *tofs = drba->drba_cookie->drc_tofs;
1377 dsl_dataset_t *ds, *newds;
1378 uint64_t dsobj;
1379 int error;
1380 uint64_t crflags = 0;
1381
1382 if (drrb->drr_flags & DRR_FLAG_CI_DATA)
1383 crflags |= DS_FLAG_CI_DATASET;
1384
1385 error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
1386 if (error == 0) {
1387 /* create temporary clone */
1388 dsl_dataset_t *snap = NULL;
1389 if (drba->drba_snapobj != 0) {
1390 VERIFY0(dsl_dataset_hold_obj(dp,
1391 drba->drba_snapobj, FTAG, &snap));
1392 }
1393 dsobj = dsl_dataset_create_sync(ds->ds_dir, recv_clone_name,
1394 snap, crflags, drba->drba_cred, tx);
1395 if (drba->drba_snapobj != 0)
1396 dsl_dataset_rele(snap, FTAG);
1397 dsl_dataset_rele(ds, FTAG);
1398 } else {
1399 dsl_dir_t *dd;
1400 const char *tail;
1401 dsl_dataset_t *origin = NULL;
1402
1403 VERIFY0(dsl_dir_hold(dp, tofs, FTAG, &dd, &tail));
1404
1405 if (drba->drba_origin != NULL) {
1406 VERIFY0(dsl_dataset_hold(dp, drba->drba_origin,
1407 FTAG, &origin));
1408 }
1409
1410 /* Create new dataset. */
1411 dsobj = dsl_dataset_create_sync(dd,
1412 strrchr(tofs, '/') + 1,
1413 origin, crflags, drba->drba_cred, tx);
1414 if (origin != NULL)
1415 dsl_dataset_rele(origin, FTAG);
1416 dsl_dir_rele(dd, FTAG);
1417 drba->drba_cookie->drc_newfs = B_TRUE;
1418 }
1419 VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &newds));
1420
1421 if (drba->drba_cookie->drc_resumable) {
1422 dsl_dataset_zapify(newds, tx);
1423 if (drrb->drr_fromguid != 0) {
1424 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_FROMGUID,
1425 8, 1, &drrb->drr_fromguid, tx));
1426 }
1427 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TOGUID,
1428 8, 1, &drrb->drr_toguid, tx));
1429 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TONAME,
1430 1, strlen(drrb->drr_toname) + 1, drrb->drr_toname, tx));
1431 uint64_t one = 1;
1432 uint64_t zero = 0;
1433 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OBJECT,
1434 8, 1, &one, tx));
1435 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OFFSET,
1436 8, 1, &zero, tx));
1437 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_BYTES,
1438 8, 1, &zero, tx));
1439 if (DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
1440 DMU_BACKUP_FEATURE_EMBED_DATA) {
1441 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_EMBEDOK,
1442 8, 1, &one, tx));
1443 }
1444 }
1445
1446 dmu_buf_will_dirty(newds->ds_dbuf, tx);
1447 dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT;
1448
1449 /*
1450 * If we actually created a non-clone, we need to create the
1451 * objset in our new dataset.
1452 */
1453 if (BP_IS_HOLE(dsl_dataset_get_blkptr(newds))) {
1454 (void) dmu_objset_create_impl(dp->dp_spa,
1455 newds, dsl_dataset_get_blkptr(newds), drrb->drr_type, tx);
1456 }
1457
1458 drba->drba_cookie->drc_ds = newds;
1459
1460 spa_history_log_internal_ds(newds, "receive", tx, "");
1461 }
1462
1463 static int
1464 dmu_recv_resume_begin_check(void *arg, dmu_tx_t *tx)
1465 {
1466 dmu_recv_begin_arg_t *drba = arg;
1467 dsl_pool_t *dp = dmu_tx_pool(tx);
1468 struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
1469 int error;
1470 uint64_t featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
1471 dsl_dataset_t *ds;
1472 const char *tofs = drba->drba_cookie->drc_tofs;
1473
1474 /* already checked */
1475 ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
1476 ASSERT(featureflags & DMU_BACKUP_FEATURE_RESUMING);
1477
1478 if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
1479 DMU_COMPOUNDSTREAM ||
1480 drrb->drr_type >= DMU_OST_NUMTYPES)
1481 return (SET_ERROR(EINVAL));
1482
1483 /* Verify pool version supports SA if SA_SPILL feature set */
1484 if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
1485 spa_version(dp->dp_spa) < SPA_VERSION_SA)
1486 return (SET_ERROR(ENOTSUP));
1487
1488 /*
1489 * The receiving code doesn't know how to translate a WRITE_EMBEDDED
1490 * record to a plain WRITE record, so the pool must have the
1491 * EMBEDDED_DATA feature enabled if the stream has WRITE_EMBEDDED
1492 * records. Same with WRITE_EMBEDDED records that use LZ4 compression.
1493 */
1494 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) &&
1495 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA))
1496 return (SET_ERROR(ENOTSUP));
1497 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4) &&
1498 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
1499 return (SET_ERROR(ENOTSUP));
1500
1501 char recvname[ZFS_MAXNAMELEN];
1502
1503 (void) snprintf(recvname, sizeof (recvname), "%s/%s",
1504 tofs, recv_clone_name);
1505
1506 if (dsl_dataset_hold(dp, recvname, FTAG, &ds) != 0) {
1507 /* %recv does not exist; continue in tofs */
1508 error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
1509 if (error != 0)
1510 return (error);
1511 }
1512
1513 /* check that ds is marked inconsistent */
1514 if (!DS_IS_INCONSISTENT(ds)) {
1515 dsl_dataset_rele(ds, FTAG);
1516 return (SET_ERROR(EINVAL));
1517 }
1518
1519 /* check that there is resuming data, and that the toguid matches */
1520 if (!dsl_dataset_is_zapified(ds)) {
1521 dsl_dataset_rele(ds, FTAG);
1522 return (SET_ERROR(EINVAL));
1523 }
1524 uint64_t val;
1525 error = zap_lookup(dp->dp_meta_objset, ds->ds_object,
1526 DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val);
1527 if (error != 0 || drrb->drr_toguid != val) {
1528 dsl_dataset_rele(ds, FTAG);
1529 return (SET_ERROR(EINVAL));
1530 }
1531
1532 /*
1533 * Check if the receive is still running. If so, it will be owned.
1534 * Note that nothing else can own the dataset (e.g. after the receive
1535 * fails) because it will be marked inconsistent.
1536 */
1537 if (dsl_dataset_has_owner(ds)) {
1538 dsl_dataset_rele(ds, FTAG);
1539 return (SET_ERROR(EBUSY));
1540 }
1541
1542 /* There should not be any snapshots of this fs yet. */
1543 if (ds->ds_prev != NULL && ds->ds_prev->ds_dir == ds->ds_dir) {
1544 dsl_dataset_rele(ds, FTAG);
1545 return (SET_ERROR(EINVAL));
1546 }
1547
1548 /*
1549 * Note: resume point will be checked when we process the first WRITE
1550 * record.
1551 */
1552
1553 /* check that the origin matches */
1554 val = 0;
1555 (void) zap_lookup(dp->dp_meta_objset, ds->ds_object,
1556 DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val);
1557 if (drrb->drr_fromguid != val) {
1558 dsl_dataset_rele(ds, FTAG);
1559 return (SET_ERROR(EINVAL));
1560 }
1561
1562 dsl_dataset_rele(ds, FTAG);
1563 return (0);
1564 }
1565
1566 static void
1567 dmu_recv_resume_begin_sync(void *arg, dmu_tx_t *tx)
1568 {
1569 dmu_recv_begin_arg_t *drba = arg;
1570 dsl_pool_t *dp = dmu_tx_pool(tx);
1571 const char *tofs = drba->drba_cookie->drc_tofs;
1572 dsl_dataset_t *ds;
1573 uint64_t dsobj;
1574 char recvname[ZFS_MAXNAMELEN];
1575
1576 (void) snprintf(recvname, sizeof (recvname), "%s/%s",
1577 tofs, recv_clone_name);
1578
1579 if (dsl_dataset_hold(dp, recvname, FTAG, &ds) != 0) {
1580 /* %recv does not exist; continue in tofs */
1581 VERIFY0(dsl_dataset_hold(dp, tofs, FTAG, &ds));
1582 drba->drba_cookie->drc_newfs = B_TRUE;
1583 }
1584
1585 /* clear the inconsistent flag so that we can own it */
1586 ASSERT(DS_IS_INCONSISTENT(ds));
1587 dmu_buf_will_dirty(ds->ds_dbuf, tx);
1588 dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
1589 dsobj = ds->ds_object;
1590 dsl_dataset_rele(ds, FTAG);
1591
1592 VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &ds));
1593
1594 dmu_buf_will_dirty(ds->ds_dbuf, tx);
1595 dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_INCONSISTENT;
1596
1597 ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)));
1598
1599 drba->drba_cookie->drc_ds = ds;
1600
1601 spa_history_log_internal_ds(ds, "resume receive", tx, "");
1602 }
1603
1604 /*
1605 * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin()
1606 * succeeds; otherwise we will leak the holds on the datasets.
1607 */
1608 int
1609 dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin,
1610 boolean_t force, boolean_t resumable, char *origin, dmu_recv_cookie_t *drc)
1611 {
1612 dmu_recv_begin_arg_t drba = { 0 };
1613
1614 bzero(drc, sizeof (dmu_recv_cookie_t));
1615 drc->drc_drr_begin = drr_begin;
1616 drc->drc_drrb = &drr_begin->drr_u.drr_begin;
1617 drc->drc_tosnap = tosnap;
1618 drc->drc_tofs = tofs;
1619 drc->drc_force = force;
1620 drc->drc_resumable = resumable;
1621 drc->drc_cred = CRED();
1622
1623 if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
1624 drc->drc_byteswap = B_TRUE;
1625 fletcher_4_incremental_byteswap(drr_begin,
1626 sizeof (dmu_replay_record_t), &drc->drc_cksum);
1627 byteswap_record(drr_begin);
1628 } else if (drc->drc_drrb->drr_magic == DMU_BACKUP_MAGIC) {
1629 fletcher_4_incremental_native(drr_begin,
1630 sizeof (dmu_replay_record_t), &drc->drc_cksum);
1631 } else {
1632 return (SET_ERROR(EINVAL));
1633 }
1634
1635 drba.drba_origin = origin;
1636 drba.drba_cookie = drc;
1637 drba.drba_cred = CRED();
1638
1639 if (DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo) &
1640 DMU_BACKUP_FEATURE_RESUMING) {
1641 return (dsl_sync_task(tofs,
1642 dmu_recv_resume_begin_check, dmu_recv_resume_begin_sync,
1643 &drba, 5, ZFS_SPACE_CHECK_NORMAL));
1644 } else {
1645 return (dsl_sync_task(tofs,
1646 dmu_recv_begin_check, dmu_recv_begin_sync,
1647 &drba, 5, ZFS_SPACE_CHECK_NORMAL));
1648 }
1649 }
1650
1651 struct receive_record_arg {
1652 dmu_replay_record_t header;
1653 void *payload; /* Pointer to a buffer containing the payload */
1654 /*
1655 * If the record is a write, pointer to the arc_buf_t containing the
1656 * payload.
1657 */
1658 arc_buf_t *write_buf;
1659 int payload_size;
1660 uint64_t bytes_read; /* bytes read from stream when record created */
1661 boolean_t eos_marker; /* Marks the end of the stream */
1662 bqueue_node_t node;
1663 };
1664
1665 struct receive_writer_arg {
1666 objset_t *os;
1667 boolean_t byteswap;
1668 bqueue_t q;
1669
1670 /*
1671 * These three args are used to signal to the main thread that we're
1672 * done.
1673 */
1674 kmutex_t mutex;
1675 kcondvar_t cv;
1676 boolean_t done;
1677
1678 int err;
1679 /* A map from guid to dataset to help handle dedup'd streams. */
1680 avl_tree_t *guid_to_ds_map;
1681 boolean_t resumable;
1682 uint64_t last_object, last_offset;
1683 uint64_t bytes_read; /* bytes read when current record created */
1684 };
1685
1686 struct objlist {
1687 list_t list; /* List of struct receive_objnode. */
1688 /*
1689 * Last object looked up. Used to assert that objects are being looked
1690 * up in ascending order.
1691 */
1692 uint64_t last_lookup;
1693 };
1694
1695 struct receive_objnode {
1696 list_node_t node;
1697 uint64_t object;
1698 };
1699
1700 struct receive_arg {
1701 objset_t *os;
1702 vnode_t *vp; /* The vnode to read the stream from */
1703 uint64_t voff; /* The current offset in the stream */
1704 uint64_t bytes_read;
1705 /*
1706 * A record that has had its payload read in, but hasn't yet been handed
1707 * off to the worker thread.
1708 */
1709 struct receive_record_arg *rrd;
1710 /* A record that has had its header read in, but not its payload. */
1711 struct receive_record_arg *next_rrd;
1712 zio_cksum_t cksum;
1713 zio_cksum_t prev_cksum;
1714 int err;
1715 boolean_t byteswap;
1716 /* Sorted list of objects not to issue prefetches for. */
1717 struct objlist ignore_objlist;
1718 };
1719
1720 typedef struct guid_map_entry {
1721 uint64_t guid;
1722 dsl_dataset_t *gme_ds;
1723 avl_node_t avlnode;
1724 } guid_map_entry_t;
1725
1726 static int
1727 guid_compare(const void *arg1, const void *arg2)
1728 {
1729 const guid_map_entry_t *gmep1 = arg1;
1730 const guid_map_entry_t *gmep2 = arg2;
1731
1732 if (gmep1->guid < gmep2->guid)
1733 return (-1);
1734 else if (gmep1->guid > gmep2->guid)
1735 return (1);
1736 return (0);
1737 }
1738
1739 static void
1740 free_guid_map_onexit(void *arg)
1741 {
1742 avl_tree_t *ca = arg;
1743 void *cookie = NULL;
1744 guid_map_entry_t *gmep;
1745
1746 while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) {
1747 dsl_dataset_long_rele(gmep->gme_ds, gmep);
1748 dsl_dataset_rele(gmep->gme_ds, gmep);
1749 kmem_free(gmep, sizeof (guid_map_entry_t));
1750 }
1751 avl_destroy(ca);
1752 kmem_free(ca, sizeof (avl_tree_t));
1753 }
1754
1755 static int
1756 receive_read(struct receive_arg *ra, int len, void *buf)
1757 {
1758 int done = 0;
1759
1760 /* some things will require 8-byte alignment, so everything must */
1761 ASSERT0(len % 8);
1762
1763 while (done < len) {
1764 ssize_t resid;
1765
1766 ra->err = vn_rdwr(UIO_READ, ra->vp,
1767 (char *)buf + done, len - done,
1768 ra->voff, UIO_SYSSPACE, FAPPEND,
1769 RLIM64_INFINITY, CRED(), &resid);
1770
1771 if (resid == len - done) {
1772 /*
1773 * Note: ECKSUM indicates that the receive
1774 * was interrupted and can potentially be resumed.
1775 */
1776 ra->err = SET_ERROR(ECKSUM);
1777 }
1778 ra->voff += len - done - resid;
1779 done = len - resid;
1780 if (ra->err != 0)
1781 return (ra->err);
1782 }
1783
1784 ra->bytes_read += len;
1785
1786 ASSERT3U(done, ==, len);
1787 return (0);
1788 }
1789
1790 static void
1791 byteswap_record(dmu_replay_record_t *drr)
1792 {
1793 #define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X))
1794 #define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X))
1795 drr->drr_type = BSWAP_32(drr->drr_type);
1796 drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen);
1797
1798 switch (drr->drr_type) {
1799 case DRR_BEGIN:
1800 DO64(drr_begin.drr_magic);
1801 DO64(drr_begin.drr_versioninfo);
1802 DO64(drr_begin.drr_creation_time);
1803 DO32(drr_begin.drr_type);
1804 DO32(drr_begin.drr_flags);
1805 DO64(drr_begin.drr_toguid);
1806 DO64(drr_begin.drr_fromguid);
1807 break;
1808 case DRR_OBJECT:
1809 DO64(drr_object.drr_object);
1810 DO32(drr_object.drr_type);
1811 DO32(drr_object.drr_bonustype);
1812 DO32(drr_object.drr_blksz);
1813 DO32(drr_object.drr_bonuslen);
1814 DO64(drr_object.drr_toguid);
1815 break;
1816 case DRR_FREEOBJECTS:
1817 DO64(drr_freeobjects.drr_firstobj);
1818 DO64(drr_freeobjects.drr_numobjs);
1819 DO64(drr_freeobjects.drr_toguid);
1820 break;
1821 case DRR_WRITE:
1822 DO64(drr_write.drr_object);
1823 DO32(drr_write.drr_type);
1824 DO64(drr_write.drr_offset);
1825 DO64(drr_write.drr_length);
1826 DO64(drr_write.drr_toguid);
1827 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum);
1828 DO64(drr_write.drr_key.ddk_prop);
1829 break;
1830 case DRR_WRITE_BYREF:
1831 DO64(drr_write_byref.drr_object);
1832 DO64(drr_write_byref.drr_offset);
1833 DO64(drr_write_byref.drr_length);
1834 DO64(drr_write_byref.drr_toguid);
1835 DO64(drr_write_byref.drr_refguid);
1836 DO64(drr_write_byref.drr_refobject);
1837 DO64(drr_write_byref.drr_refoffset);
1838 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write_byref.
1839 drr_key.ddk_cksum);
1840 DO64(drr_write_byref.drr_key.ddk_prop);
1841 break;
1842 case DRR_WRITE_EMBEDDED:
1843 DO64(drr_write_embedded.drr_object);
1844 DO64(drr_write_embedded.drr_offset);
1845 DO64(drr_write_embedded.drr_length);
1846 DO64(drr_write_embedded.drr_toguid);
1847 DO32(drr_write_embedded.drr_lsize);
1848 DO32(drr_write_embedded.drr_psize);
1849 break;
1850 case DRR_FREE:
1851 DO64(drr_free.drr_object);
1852 DO64(drr_free.drr_offset);
1853 DO64(drr_free.drr_length);
1854 DO64(drr_free.drr_toguid);
1855 break;
1856 case DRR_SPILL:
1857 DO64(drr_spill.drr_object);
1858 DO64(drr_spill.drr_length);
1859 DO64(drr_spill.drr_toguid);
1860 break;
1861 case DRR_END:
1862 DO64(drr_end.drr_toguid);
1863 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum);
1864 break;
1865 }
1866
1867 if (drr->drr_type != DRR_BEGIN) {
1868 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum);
1869 }
1870
1871 #undef DO64
1872 #undef DO32
1873 }
1874
1875 static inline uint8_t
1876 deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
1877 {
1878 if (bonus_type == DMU_OT_SA) {
1879 return (1);
1880 } else {
1881 return (1 +
1882 ((DN_MAX_BONUSLEN - bonus_size) >> SPA_BLKPTRSHIFT));
1883 }
1884 }
1885
1886 static void
1887 save_resume_state(struct receive_writer_arg *rwa,
1888 uint64_t object, uint64_t offset, dmu_tx_t *tx)
1889 {
1890 int txgoff = dmu_tx_get_txg(tx) & TXG_MASK;
1891
1892 if (!rwa->resumable)
1893 return;
1894
1895 /*
1896 * We use ds_resume_bytes[] != 0 to indicate that we need to
1897 * update this on disk, so it must not be 0.
1898 */
1899 ASSERT(rwa->bytes_read != 0);
1900
1901 /*
1902 * We only resume from write records, which have a valid
1903 * (non-meta-dnode) object number.
1904 */
1905 ASSERT(object != 0);
1906
1907 /*
1908 * For resuming to work correctly, we must receive records in order,
1909 * sorted by object,offset. This is checked by the callers, but
1910 * assert it here for good measure.
1911 */
1912 ASSERT3U(object, >=, rwa->os->os_dsl_dataset->ds_resume_object[txgoff]);
1913 ASSERT(object != rwa->os->os_dsl_dataset->ds_resume_object[txgoff] ||
1914 offset >= rwa->os->os_dsl_dataset->ds_resume_offset[txgoff]);
1915 ASSERT3U(rwa->bytes_read, >=,
1916 rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff]);
1917
1918 rwa->os->os_dsl_dataset->ds_resume_object[txgoff] = object;
1919 rwa->os->os_dsl_dataset->ds_resume_offset[txgoff] = offset;
1920 rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff] = rwa->bytes_read;
1921 }
1922
1923 static int
1924 receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
1925 void *data)
1926 {
1927 dmu_object_info_t doi;
1928 dmu_tx_t *tx;
1929 uint64_t object;
1930 int err;
1931
1932 if (drro->drr_type == DMU_OT_NONE ||
1933 !DMU_OT_IS_VALID(drro->drr_type) ||
1934 !DMU_OT_IS_VALID(drro->drr_bonustype) ||
1935 drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS ||
1936 drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
1937 P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
1938 drro->drr_blksz < SPA_MINBLOCKSIZE ||
1939 drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(rwa->os)) ||
1940 drro->drr_bonuslen > DN_MAX_BONUSLEN) {
1941 return (SET_ERROR(EINVAL));
1942 }
1943
1944 err = dmu_object_info(rwa->os, drro->drr_object, &doi);
1945
1946 if (err != 0 && err != ENOENT)
1947 return (SET_ERROR(EINVAL));
1948 object = err == 0 ? drro->drr_object : DMU_NEW_OBJECT;
1949
1950 /*
1951 * If we are losing blkptrs or changing the block size this must
1952 * be a new file instance. We must clear out the previous file
1953 * contents before we can change this type of metadata in the dnode.
1954 */
1955 if (err == 0) {
1956 int nblkptr;
1957
1958 nblkptr = deduce_nblkptr(drro->drr_bonustype,
1959 drro->drr_bonuslen);
1960
1961 if (drro->drr_blksz != doi.doi_data_block_size ||
1962 nblkptr < doi.doi_nblkptr) {
1963 err = dmu_free_long_range(rwa->os, drro->drr_object,
1964 0, DMU_OBJECT_END);
1965 if (err != 0)
1966 return (SET_ERROR(EINVAL));
1967 }
1968 }
1969
1970 tx = dmu_tx_create(rwa->os);
1971 dmu_tx_hold_bonus(tx, object);
1972 err = dmu_tx_assign(tx, TXG_WAIT);
1973 if (err != 0) {
1974 dmu_tx_abort(tx);
1975 return (err);
1976 }
1977
1978 if (object == DMU_NEW_OBJECT) {
1979 /* currently free, want to be allocated */
1980 err = dmu_object_claim(rwa->os, drro->drr_object,
1981 drro->drr_type, drro->drr_blksz,
1982 drro->drr_bonustype, drro->drr_bonuslen, tx);
1983 } else if (drro->drr_type != doi.doi_type ||
1984 drro->drr_blksz != doi.doi_data_block_size ||
1985 drro->drr_bonustype != doi.doi_bonus_type ||
1986 drro->drr_bonuslen != doi.doi_bonus_size) {
1987 /* currently allocated, but with different properties */
1988 err = dmu_object_reclaim(rwa->os, drro->drr_object,
1989 drro->drr_type, drro->drr_blksz,
1990 drro->drr_bonustype, drro->drr_bonuslen, tx);
1991 }
1992 if (err != 0) {
1993 dmu_tx_commit(tx);
1994 return (SET_ERROR(EINVAL));
1995 }
1996
1997 dmu_object_set_checksum(rwa->os, drro->drr_object,
1998 drro->drr_checksumtype, tx);
1999 dmu_object_set_compress(rwa->os, drro->drr_object,
2000 drro->drr_compress, tx);
2001
2002 if (data != NULL) {
2003 dmu_buf_t *db;
2004
2005 VERIFY0(dmu_bonus_hold(rwa->os, drro->drr_object, FTAG, &db));
2006 dmu_buf_will_dirty(db, tx);
2007
2008 ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
2009 bcopy(data, db->db_data, drro->drr_bonuslen);
2010 if (rwa->byteswap) {
2011 dmu_object_byteswap_t byteswap =
2012 DMU_OT_BYTESWAP(drro->drr_bonustype);
2013 dmu_ot_byteswap[byteswap].ob_func(db->db_data,
2014 drro->drr_bonuslen);
2015 }
2016 dmu_buf_rele(db, FTAG);
2017 }
2018 dmu_tx_commit(tx);
2019
2020 return (0);
2021 }
2022
2023 /* ARGSUSED */
2024 static int
2025 receive_freeobjects(struct receive_writer_arg *rwa,
2026 struct drr_freeobjects *drrfo)
2027 {
2028 uint64_t obj;
2029 int next_err = 0;
2030
2031 if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj)
2032 return (SET_ERROR(EINVAL));
2033
2034 for (obj = drrfo->drr_firstobj;
2035 obj < drrfo->drr_firstobj + drrfo->drr_numobjs && next_err == 0;
2036 next_err = dmu_object_next(rwa->os, &obj, FALSE, 0)) {
2037 int err;
2038
2039 if (dmu_object_info(rwa->os, obj, NULL) != 0)
2040 continue;
2041
2042 err = dmu_free_long_object(rwa->os, obj);
2043 if (err != 0)
2044 return (err);
2045 }
2046 if (next_err != ESRCH)
2047 return (next_err);
2048 return (0);
2049 }
2050
2051 static int
2052 receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
2053 arc_buf_t *abuf)
2054 {
2055 dmu_tx_t *tx;
2056 int err;
2057
2058 if (drrw->drr_offset + drrw->drr_length < drrw->drr_offset ||
2059 !DMU_OT_IS_VALID(drrw->drr_type))
2060 return (SET_ERROR(EINVAL));
2061
2062 /*
2063 * For resuming to work, records must be in increasing order
2064 * by (object, offset).
2065 */
2066 if (drrw->drr_object < rwa->last_object ||
2067 (drrw->drr_object == rwa->last_object &&
2068 drrw->drr_offset < rwa->last_offset)) {
2069 return (SET_ERROR(EINVAL));
2070 }
2071 rwa->last_object = drrw->drr_object;
2072 rwa->last_offset = drrw->drr_offset;
2073
2074 if (dmu_object_info(rwa->os, drrw->drr_object, NULL) != 0)
2075 return (SET_ERROR(EINVAL));
2076
2077 tx = dmu_tx_create(rwa->os);
2078
2079 dmu_tx_hold_write(tx, drrw->drr_object,
2080 drrw->drr_offset, drrw->drr_length);
2081 err = dmu_tx_assign(tx, TXG_WAIT);
2082 if (err != 0) {
2083 dmu_tx_abort(tx);
2084 return (err);
2085 }
2086 if (rwa->byteswap) {
2087 dmu_object_byteswap_t byteswap =
2088 DMU_OT_BYTESWAP(drrw->drr_type);
2089 dmu_ot_byteswap[byteswap].ob_func(abuf->b_data,
2090 drrw->drr_length);
2091 }
2092
2093 dmu_buf_t *bonus;
2094 if (dmu_bonus_hold(rwa->os, drrw->drr_object, FTAG, &bonus) != 0)
2095 return (SET_ERROR(EINVAL));
2096 dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx);
2097
2098 /*
2099 * Note: If the receive fails, we want the resume stream to start
2100 * with the same record that we last successfully received (as opposed
2101 * to the next record), so that we can verify that we are
2102 * resuming from the correct location.
2103 */
2104 save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx);
2105 dmu_tx_commit(tx);
2106 dmu_buf_rele(bonus, FTAG);
2107
2108 return (0);
2109 }
2110
2111 /*
2112 * Handle a DRR_WRITE_BYREF record. This record is used in dedup'ed
2113 * streams to refer to a copy of the data that is already on the
2114 * system because it came in earlier in the stream. This function
2115 * finds the earlier copy of the data, and uses that copy instead of
2116 * data from the stream to fulfill this write.
2117 */
2118 static int
2119 receive_write_byref(struct receive_writer_arg *rwa,
2120 struct drr_write_byref *drrwbr)
2121 {
2122 dmu_tx_t *tx;
2123 int err;
2124 guid_map_entry_t gmesrch;
2125 guid_map_entry_t *gmep;
2126 avl_index_t where;
2127 objset_t *ref_os = NULL;
2128 dmu_buf_t *dbp;
2129
2130 if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset)
2131 return (SET_ERROR(EINVAL));
2132
2133 /*
2134 * If the GUID of the referenced dataset is different from the
2135 * GUID of the target dataset, find the referenced dataset.
2136 */
2137 if (drrwbr->drr_toguid != drrwbr->drr_refguid) {
2138 gmesrch.guid = drrwbr->drr_refguid;
2139 if ((gmep = avl_find(rwa->guid_to_ds_map, &gmesrch,
2140 &where)) == NULL) {
2141 return (SET_ERROR(EINVAL));
2142 }
2143 if (dmu_objset_from_ds(gmep->gme_ds, &ref_os))
2144 return (SET_ERROR(EINVAL));
2145 } else {
2146 ref_os = rwa->os;
2147 }
2148
2149 err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
2150 drrwbr->drr_refoffset, FTAG, &dbp, DMU_READ_PREFETCH);
2151 if (err != 0)
2152 return (err);
2153
2154 tx = dmu_tx_create(rwa->os);
2155
2156 dmu_tx_hold_write(tx, drrwbr->drr_object,
2157 drrwbr->drr_offset, drrwbr->drr_length);
2158 err = dmu_tx_assign(tx, TXG_WAIT);
2159 if (err != 0) {
2160 dmu_tx_abort(tx);
2161 return (err);
2162 }
2163 dmu_write(rwa->os, drrwbr->drr_object,
2164 drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
2165 dmu_buf_rele(dbp, FTAG);
2166
2167 /* See comment in restore_write. */
2168 save_resume_state(rwa, drrwbr->drr_object, drrwbr->drr_offset, tx);
2169 dmu_tx_commit(tx);
2170 return (0);
2171 }
2172
2173 static int
2174 receive_write_embedded(struct receive_writer_arg *rwa,
2175 struct drr_write_embedded *drrwe, void *data)
2176 {
2177 dmu_tx_t *tx;
2178 int err;
2179
2180 if (drrwe->drr_offset + drrwe->drr_length < drrwe->drr_offset)
2181 return (EINVAL);
2182
2183 if (drrwe->drr_psize > BPE_PAYLOAD_SIZE)
2184 return (EINVAL);
2185
2186 if (drrwe->drr_etype >= NUM_BP_EMBEDDED_TYPES)
2187 return (EINVAL);
2188 if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
2189 return (EINVAL);
2190
2191 tx = dmu_tx_create(rwa->os);
2192
2193 dmu_tx_hold_write(tx, drrwe->drr_object,
2194 drrwe->drr_offset, drrwe->drr_length);
2195 err = dmu_tx_assign(tx, TXG_WAIT);
2196 if (err != 0) {
2197 dmu_tx_abort(tx);
2198 return (err);
2199 }
2200
2201 dmu_write_embedded(rwa->os, drrwe->drr_object,
2202 drrwe->drr_offset, data, drrwe->drr_etype,
2203 drrwe->drr_compression, drrwe->drr_lsize, drrwe->drr_psize,
2204 rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx);
2205
2206 /* See comment in restore_write. */
2207 save_resume_state(rwa, drrwe->drr_object, drrwe->drr_offset, tx);
2208 dmu_tx_commit(tx);
2209 return (0);
2210 }
2211
2212 static int
2213 receive_spill(struct receive_writer_arg *rwa, struct drr_spill *drrs,
2214 void *data)
2215 {
2216 dmu_tx_t *tx;
2217 dmu_buf_t *db, *db_spill;
2218 int err;
2219
2220 if (drrs->drr_length < SPA_MINBLOCKSIZE ||
2221 drrs->drr_length > spa_maxblocksize(dmu_objset_spa(rwa->os)))
2222 return (SET_ERROR(EINVAL));
2223
2224 if (dmu_object_info(rwa->os, drrs->drr_object, NULL) != 0)
2225 return (SET_ERROR(EINVAL));
2226
2227 VERIFY0(dmu_bonus_hold(rwa->os, drrs->drr_object, FTAG, &db));
2228 if ((err = dmu_spill_hold_by_bonus(db, FTAG, &db_spill)) != 0) {
2229 dmu_buf_rele(db, FTAG);
2230 return (err);
2231 }
2232
2233 tx = dmu_tx_create(rwa->os);
2234
2235 dmu_tx_hold_spill(tx, db->db_object);
2236
2237 err = dmu_tx_assign(tx, TXG_WAIT);
2238 if (err != 0) {
2239 dmu_buf_rele(db, FTAG);
2240 dmu_buf_rele(db_spill, FTAG);
2241 dmu_tx_abort(tx);
2242 return (err);
2243 }
2244 dmu_buf_will_dirty(db_spill, tx);
2245
2246 if (db_spill->db_size < drrs->drr_length)
2247 VERIFY(0 == dbuf_spill_set_blksz(db_spill,
2248 drrs->drr_length, tx));
2249 bcopy(data, db_spill->db_data, drrs->drr_length);
2250
2251 dmu_buf_rele(db, FTAG);
2252 dmu_buf_rele(db_spill, FTAG);
2253
2254 dmu_tx_commit(tx);
2255 return (0);
2256 }
2257
2258 /* ARGSUSED */
2259 static int
2260 receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf)
2261 {
2262 int err;
2263
2264 if (drrf->drr_length != -1ULL &&
2265 drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
2266 return (SET_ERROR(EINVAL));
2267
2268 if (dmu_object_info(rwa->os, drrf->drr_object, NULL) != 0)
2269 return (SET_ERROR(EINVAL));
2270
2271 err = dmu_free_long_range(rwa->os, drrf->drr_object,
2272 drrf->drr_offset, drrf->drr_length);
2273
2274 return (err);
2275 }
2276
2277 /* used to destroy the drc_ds on error */
2278 static void
2279 dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
2280 {
2281 if (drc->drc_resumable) {
2282 /* wait for our resume state to be written to disk */
2283 txg_wait_synced(drc->drc_ds->ds_dir->dd_pool, 0);
2284 dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
2285 } else {
2286 char name[MAXNAMELEN];
2287 dsl_dataset_name(drc->drc_ds, name);
2288 dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
2289 (void) dsl_destroy_head(name);
2290 }
2291 }
2292
2293 static void
2294 receive_cksum(struct receive_arg *ra, int len, void *buf)
2295 {
2296 if (ra->byteswap) {
2297 fletcher_4_incremental_byteswap(buf, len, &ra->cksum);
2298 } else {
2299 fletcher_4_incremental_native(buf, len, &ra->cksum);
2300 }
2301 }
2302
2303 /*
2304 * Read the payload into a buffer of size len, and update the current record's
2305 * payload field.
2306 * Allocate ra->next_rrd and read the next record's header into
2307 * ra->next_rrd->header.
2308 * Verify checksum of payload and next record.
2309 */
2310 static int
2311 receive_read_payload_and_next_header(struct receive_arg *ra, int len, void *buf)
2312 {
2313 int err;
2314
2315 if (len != 0) {
2316 ASSERT3U(len, <=, SPA_MAXBLOCKSIZE);
2317 err = receive_read(ra, len, buf);
2318 if (err != 0)
2319 return (err);
2320 receive_cksum(ra, len, buf);
2321
2322 /* note: rrd is NULL when reading the begin record's payload */
2323 if (ra->rrd != NULL) {
2324 ra->rrd->payload = buf;
2325 ra->rrd->payload_size = len;
2326 ra->rrd->bytes_read = ra->bytes_read;
2327 }
2328 }
2329
2330 ra->prev_cksum = ra->cksum;
2331
2332 ra->next_rrd = kmem_zalloc(sizeof (*ra->next_rrd), KM_SLEEP);
2333 err = receive_read(ra, sizeof (ra->next_rrd->header),
2334 &ra->next_rrd->header);
2335 ra->next_rrd->bytes_read = ra->bytes_read;
2336 if (err != 0) {
2337 kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
2338 ra->next_rrd = NULL;
2339 return (err);
2340 }
2341 if (ra->next_rrd->header.drr_type == DRR_BEGIN) {
2342 kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
2343 ra->next_rrd = NULL;
2344 return (SET_ERROR(EINVAL));
2345 }
2346
2347 /*
2348 * Note: checksum is of everything up to but not including the
2349 * checksum itself.
2350 */
2351 ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
2352 ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
2353 receive_cksum(ra,
2354 offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
2355 &ra->next_rrd->header);
2356
2357 zio_cksum_t cksum_orig =
2358 ra->next_rrd->header.drr_u.drr_checksum.drr_checksum;
2359 zio_cksum_t *cksump =
2360 &ra->next_rrd->header.drr_u.drr_checksum.drr_checksum;
2361
2362 if (ra->byteswap)
2363 byteswap_record(&ra->next_rrd->header);
2364
2365 if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) &&
2366 !ZIO_CHECKSUM_EQUAL(ra->cksum, *cksump)) {
2367 kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
2368 ra->next_rrd = NULL;
2369 return (SET_ERROR(ECKSUM));
2370 }
2371
2372 receive_cksum(ra, sizeof (cksum_orig), &cksum_orig);
2373
2374 return (0);
2375 }
2376
2377 static void
2378 objlist_create(struct objlist *list)
2379 {
2380 list_create(&list->list, sizeof (struct receive_objnode),
2381 offsetof(struct receive_objnode, node));
2382 list->last_lookup = 0;
2383 }
2384
2385 static void
2386 objlist_destroy(struct objlist *list)
2387 {
2388 for (struct receive_objnode *n = list_remove_head(&list->list);
2389 n != NULL; n = list_remove_head(&list->list)) {
2390 kmem_free(n, sizeof (*n));
2391 }
2392 list_destroy(&list->list);
2393 }
2394
2395 /*
2396 * This function looks through the objlist to see if the specified object number
2397 * is contained in the objlist. In the process, it will remove all object
2398 * numbers in the list that are smaller than the specified object number. Thus,
2399 * any lookup of an object number smaller than a previously looked up object
2400 * number will always return false; therefore, all lookups should be done in
2401 * ascending order.
2402 */
2403 static boolean_t
2404 objlist_exists(struct objlist *list, uint64_t object)
2405 {
2406 struct receive_objnode *node = list_head(&list->list);
2407 ASSERT3U(object, >=, list->last_lookup);
2408 list->last_lookup = object;
2409 while (node != NULL && node->object < object) {
2410 VERIFY3P(node, ==, list_remove_head(&list->list));
2411 kmem_free(node, sizeof (*node));
2412 node = list_head(&list->list);
2413 }
2414 return (node != NULL && node->object == object);
2415 }
2416
2417 /*
2418 * The objlist is a list of object numbers stored in ascending order. However,
2419 * the insertion of new object numbers does not seek out the correct location to
2420 * store a new object number; instead, it appends it to the list for simplicity.
2421 * Thus, any users must take care to only insert new object numbers in ascending
2422 * order.
2423 */
2424 static void
2425 objlist_insert(struct objlist *list, uint64_t object)
2426 {
2427 struct receive_objnode *node = kmem_zalloc(sizeof (*node), KM_SLEEP);
2428 node->object = object;
2429 #ifdef ZFS_DEBUG
2430 struct receive_objnode *last_object = list_tail(&list->list);
2431 uint64_t last_objnum = (last_object != NULL ? last_object->object : 0);
2432 ASSERT3U(node->object, >, last_objnum);
2433 #endif
2434 list_insert_tail(&list->list, node);
2435 }
2436
2437 /*
2438 * Issue the prefetch reads for any necessary indirect blocks.
2439 *
2440 * We use the object ignore list to tell us whether or not to issue prefetches
2441 * for a given object. We do this for both correctness (in case the blocksize
2442 * of an object has changed) and performance (if the object doesn't exist, don't
2443 * needlessly try to issue prefetches). We also trim the list as we go through
2444 * the stream to prevent it from growing to an unbounded size.
2445 *
2446 * The object numbers within will always be in sorted order, and any write
2447 * records we see will also be in sorted order, but they're not sorted with
2448 * respect to each other (i.e. we can get several object records before
2449 * receiving each object's write records). As a result, once we've reached a
2450 * given object number, we can safely remove any reference to lower object
2451 * numbers in the ignore list. In practice, we receive up to 32 object records
2452 * before receiving write records, so the list can have up to 32 nodes in it.
2453 */
2454 /* ARGSUSED */
2455 static void
2456 receive_read_prefetch(struct receive_arg *ra,
2457 uint64_t object, uint64_t offset, uint64_t length)
2458 {
2459 if (!objlist_exists(&ra->ignore_objlist, object)) {
2460 dmu_prefetch(ra->os, object, 1, offset, length,
2461 ZIO_PRIORITY_SYNC_READ);
2462 }
2463 }
2464
2465 /*
2466 * Read records off the stream, issuing any necessary prefetches.
2467 */
2468 static int
2469 receive_read_record(struct receive_arg *ra)
2470 {
2471 int err;
2472
2473 switch (ra->rrd->header.drr_type) {
2474 case DRR_OBJECT:
2475 {
2476 struct drr_object *drro = &ra->rrd->header.drr_u.drr_object;
2477 uint32_t size = P2ROUNDUP(drro->drr_bonuslen, 8);
2478 void *buf = kmem_zalloc(size, KM_SLEEP);
2479 dmu_object_info_t doi;
2480 err = receive_read_payload_and_next_header(ra, size, buf);
2481 if (err != 0) {
2482 kmem_free(buf, size);
2483 return (err);
2484 }
2485 err = dmu_object_info(ra->os, drro->drr_object, &doi);
2486 /*
2487 * See receive_read_prefetch for an explanation why we're
2488 * storing this object in the ignore_obj_list.
2489 */
2490 if (err == ENOENT ||
2491 (err == 0 && doi.doi_data_block_size != drro->drr_blksz)) {
2492 objlist_insert(&ra->ignore_objlist, drro->drr_object);
2493 err = 0;
2494 }
2495 return (err);
2496 }
2497 case DRR_FREEOBJECTS:
2498 {
2499 err = receive_read_payload_and_next_header(ra, 0, NULL);
2500 return (err);
2501 }
2502 case DRR_WRITE:
2503 {
2504 struct drr_write *drrw = &ra->rrd->header.drr_u.drr_write;
2505 arc_buf_t *abuf = arc_loan_buf(dmu_objset_spa(ra->os),
2506 drrw->drr_length);
2507
2508 err = receive_read_payload_and_next_header(ra,
2509 drrw->drr_length, abuf->b_data);
2510 if (err != 0) {
2511 dmu_return_arcbuf(abuf);
2512 return (err);
2513 }
2514 ra->rrd->write_buf = abuf;
2515 receive_read_prefetch(ra, drrw->drr_object, drrw->drr_offset,
2516 drrw->drr_length);
2517 return (err);
2518 }
2519 case DRR_WRITE_BYREF:
2520 {
2521 struct drr_write_byref *drrwb =
2522 &ra->rrd->header.drr_u.drr_write_byref;
2523 err = receive_read_payload_and_next_header(ra, 0, NULL);
2524 receive_read_prefetch(ra, drrwb->drr_object, drrwb->drr_offset,
2525 drrwb->drr_length);
2526 return (err);
2527 }
2528 case DRR_WRITE_EMBEDDED:
2529 {
2530 struct drr_write_embedded *drrwe =
2531 &ra->rrd->header.drr_u.drr_write_embedded;
2532 uint32_t size = P2ROUNDUP(drrwe->drr_psize, 8);
2533 void *buf = kmem_zalloc(size, KM_SLEEP);
2534
2535 err = receive_read_payload_and_next_header(ra, size, buf);
2536 if (err != 0) {
2537 kmem_free(buf, size);
2538 return (err);
2539 }
2540
2541 receive_read_prefetch(ra, drrwe->drr_object, drrwe->drr_offset,
2542 drrwe->drr_length);
2543 return (err);
2544 }
2545 case DRR_FREE:
2546 {
2547 /*
2548 * It might be beneficial to prefetch indirect blocks here, but
2549 * we don't really have the data to decide for sure.
2550 */
2551 err = receive_read_payload_and_next_header(ra, 0, NULL);
2552 return (err);
2553 }
2554 case DRR_END:
2555 {
2556 struct drr_end *drre = &ra->rrd->header.drr_u.drr_end;
2557 if (!ZIO_CHECKSUM_EQUAL(ra->prev_cksum, drre->drr_checksum))
2558 return (SET_ERROR(ECKSUM));
2559 return (0);
2560 }
2561 case DRR_SPILL:
2562 {
2563 struct drr_spill *drrs = &ra->rrd->header.drr_u.drr_spill;
2564 void *buf = kmem_zalloc(drrs->drr_length, KM_SLEEP);
2565 err = receive_read_payload_and_next_header(ra, drrs->drr_length,
2566 buf);
2567 if (err != 0)
2568 kmem_free(buf, drrs->drr_length);
2569 return (err);
2570 }
2571 default:
2572 return (SET_ERROR(EINVAL));
2573 }
2574 }
2575
2576 /*
2577 * Commit the records to the pool.
2578 */
2579 static int
2580 receive_process_record(struct receive_writer_arg *rwa,
2581 struct receive_record_arg *rrd)
2582 {
2583 int err;
2584
2585 /* Processing in order, therefore bytes_read should be increasing. */
2586 ASSERT3U(rrd->bytes_read, >=, rwa->bytes_read);
2587 rwa->bytes_read = rrd->bytes_read;
2588
2589 switch (rrd->header.drr_type) {
2590 case DRR_OBJECT:
2591 {
2592 struct drr_object *drro = &rrd->header.drr_u.drr_object;
2593 err = receive_object(rwa, drro, rrd->payload);
2594 kmem_free(rrd->payload, rrd->payload_size);
2595 rrd->payload = NULL;
2596 return (err);
2597 }
2598 case DRR_FREEOBJECTS:
2599 {
2600 struct drr_freeobjects *drrfo =
2601 &rrd->header.drr_u.drr_freeobjects;
2602 return (receive_freeobjects(rwa, drrfo));
2603 }
2604 case DRR_WRITE:
2605 {
2606 struct drr_write *drrw = &rrd->header.drr_u.drr_write;
2607 err = receive_write(rwa, drrw, rrd->write_buf);
2608 /* if receive_write() is successful, it consumes the arc_buf */
2609 if (err != 0)
2610 dmu_return_arcbuf(rrd->write_buf);
2611 rrd->write_buf = NULL;
2612 rrd->payload = NULL;
2613 return (err);
2614 }
2615 case DRR_WRITE_BYREF:
2616 {
2617 struct drr_write_byref *drrwbr =
2618 &rrd->header.drr_u.drr_write_byref;
2619 return (receive_write_byref(rwa, drrwbr));
2620 }
2621 case DRR_WRITE_EMBEDDED:
2622 {
2623 struct drr_write_embedded *drrwe =
2624 &rrd->header.drr_u.drr_write_embedded;
2625 err = receive_write_embedded(rwa, drrwe, rrd->payload);
2626 kmem_free(rrd->payload, rrd->payload_size);
2627 rrd->payload = NULL;
2628 return (err);
2629 }
2630 case DRR_FREE:
2631 {
2632 struct drr_free *drrf = &rrd->header.drr_u.drr_free;
2633 return (receive_free(rwa, drrf));
2634 }
2635 case DRR_SPILL:
2636 {
2637 struct drr_spill *drrs = &rrd->header.drr_u.drr_spill;
2638 err = receive_spill(rwa, drrs, rrd->payload);
2639 kmem_free(rrd->payload, rrd->payload_size);
2640 rrd->payload = NULL;
2641 return (err);
2642 }
2643 default:
2644 return (SET_ERROR(EINVAL));
2645 }
2646 }
2647
2648 /*
2649 * dmu_recv_stream's worker thread; pull records off the queue, and then call
2650 * receive_process_record When we're done, signal the main thread and exit.
2651 */
2652 static void
2653 receive_writer_thread(void *arg)
2654 {
2655 struct receive_writer_arg *rwa = arg;
2656 struct receive_record_arg *rrd;
2657 for (rrd = bqueue_dequeue(&rwa->q); !rrd->eos_marker;
2658 rrd = bqueue_dequeue(&rwa->q)) {
2659 /*
2660 * If there's an error, the main thread will stop putting things
2661 * on the queue, but we need to clear everything in it before we
2662 * can exit.
2663 */
2664 if (rwa->err == 0) {
2665 rwa->err = receive_process_record(rwa, rrd);
2666 } else if (rrd->write_buf != NULL) {
2667 dmu_return_arcbuf(rrd->write_buf);
2668 rrd->write_buf = NULL;
2669 rrd->payload = NULL;
2670 } else if (rrd->payload != NULL) {
2671 kmem_free(rrd->payload, rrd->payload_size);
2672 rrd->payload = NULL;
2673 }
2674 kmem_free(rrd, sizeof (*rrd));
2675 }
2676 kmem_free(rrd, sizeof (*rrd));
2677 mutex_enter(&rwa->mutex);
2678 rwa->done = B_TRUE;
2679 cv_signal(&rwa->cv);
2680 mutex_exit(&rwa->mutex);
2681 }
2682
2683 static int
2684 resume_check(struct receive_arg *ra, nvlist_t *begin_nvl)
2685 {
2686 uint64_t val;
2687 objset_t *mos = dmu_objset_pool(ra->os)->dp_meta_objset;
2688 uint64_t dsobj = dmu_objset_id(ra->os);
2689 uint64_t resume_obj, resume_off;
2690
2691 if (nvlist_lookup_uint64(begin_nvl,
2692 "resume_object", &resume_obj) != 0 ||
2693 nvlist_lookup_uint64(begin_nvl,
2694 "resume_offset", &resume_off) != 0) {
2695 return (SET_ERROR(EINVAL));
2696 }
2697 VERIFY0(zap_lookup(mos, dsobj,
2698 DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val));
2699 if (resume_obj != val)
2700 return (SET_ERROR(EINVAL));
2701 VERIFY0(zap_lookup(mos, dsobj,
2702 DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val));
2703 if (resume_off != val)
2704 return (SET_ERROR(EINVAL));
2705
2706 return (0);
2707 }
2708
2709 /*
2710 * Read in the stream's records, one by one, and apply them to the pool. There
2711 * are two threads involved; the thread that calls this function will spin up a
2712 * worker thread, read the records off the stream one by one, and issue
2713 * prefetches for any necessary indirect blocks. It will then push the records
2714 * onto an internal blocking queue. The worker thread will pull the records off
2715 * the queue, and actually write the data into the DMU. This way, the worker
2716 * thread doesn't have to wait for reads to complete, since everything it needs
2717 * (the indirect blocks) will be prefetched.
2718 *
2719 * NB: callers *must* call dmu_recv_end() if this succeeds.
2720 */
2721 int
2722 dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
2723 int cleanup_fd, uint64_t *action_handlep)
2724 {
2725 int err = 0;
2726 struct receive_arg ra = { 0 };
2727 struct receive_writer_arg rwa = { 0 };
2728 int featureflags;
2729 nvlist_t *begin_nvl = NULL;
2730
2731 ra.byteswap = drc->drc_byteswap;
2732 ra.cksum = drc->drc_cksum;
2733 ra.vp = vp;
2734 ra.voff = *voffp;
2735
2736 if (dsl_dataset_is_zapified(drc->drc_ds)) {
2737 (void) zap_lookup(drc->drc_ds->ds_dir->dd_pool->dp_meta_objset,
2738 drc->drc_ds->ds_object, DS_FIELD_RESUME_BYTES,
2739 sizeof (ra.bytes_read), 1, &ra.bytes_read);
2740 }
2741
2742 objlist_create(&ra.ignore_objlist);
2743
2744 /* these were verified in dmu_recv_begin */
2745 ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==,
2746 DMU_SUBSTREAM);
2747 ASSERT3U(drc->drc_drrb->drr_type, <, DMU_OST_NUMTYPES);
2748
2749 /*
2750 * Open the objset we are modifying.
2751 */
2752 VERIFY0(dmu_objset_from_ds(drc->drc_ds, &ra.os));
2753
2754 ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT);
2755
2756 featureflags = DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo);
2757
2758 /* if this stream is dedup'ed, set up the avl tree for guid mapping */
2759 if (featureflags & DMU_BACKUP_FEATURE_DEDUP) {
2760 minor_t minor;
2761
2762 if (cleanup_fd == -1) {
2763 ra.err = SET_ERROR(EBADF);
2764 goto out;
2765 }
2766 ra.err = zfs_onexit_fd_hold(cleanup_fd, &minor);
2767 if (ra.err != 0) {
2768 cleanup_fd = -1;
2769 goto out;
2770 }
2771
2772 if (*action_handlep == 0) {
2773 rwa.guid_to_ds_map =
2774 kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
2775 avl_create(rwa.guid_to_ds_map, guid_compare,
2776 sizeof (guid_map_entry_t),
2777 offsetof(guid_map_entry_t, avlnode));
2778 err = zfs_onexit_add_cb(minor,
2779 free_guid_map_onexit, rwa.guid_to_ds_map,
2780 action_handlep);
2781 if (ra.err != 0)
2782 goto out;
2783 } else {
2784 err = zfs_onexit_cb_data(minor, *action_handlep,
2785 (void **)&rwa.guid_to_ds_map);
2786 if (ra.err != 0)
2787 goto out;
2788 }
2789
2790 drc->drc_guid_to_ds_map = rwa.guid_to_ds_map;
2791 }
2792
2793 uint32_t payloadlen = drc->drc_drr_begin->drr_payloadlen;
2794 void *payload = NULL;
2795 if (payloadlen != 0)
2796 payload = kmem_alloc(payloadlen, KM_SLEEP);
2797
2798 err = receive_read_payload_and_next_header(&ra, payloadlen, payload);
2799 if (err != 0) {
2800 if (payloadlen != 0)
2801 kmem_free(payload, payloadlen);
2802 goto out;
2803 }
2804 if (payloadlen != 0) {
2805 err = nvlist_unpack(payload, payloadlen, &begin_nvl, KM_SLEEP);
2806 kmem_free(payload, payloadlen);
2807 if (err != 0)
2808 goto out;
2809 }
2810
2811 if (featureflags & DMU_BACKUP_FEATURE_RESUMING) {
2812 err = resume_check(&ra, begin_nvl);
2813 if (err != 0)
2814 goto out;
2815 }
2816
2817 (void) bqueue_init(&rwa.q, zfs_recv_queue_length,
2818 offsetof(struct receive_record_arg, node));
2819 cv_init(&rwa.cv, NULL, CV_DEFAULT, NULL);
2820 mutex_init(&rwa.mutex, NULL, MUTEX_DEFAULT, NULL);
2821 rwa.os = ra.os;
2822 rwa.byteswap = drc->drc_byteswap;
2823 rwa.resumable = drc->drc_resumable;
2824
2825 (void) thread_create(NULL, 0, receive_writer_thread, &rwa, 0, curproc,
2826 TS_RUN, minclsyspri);
2827 /*
2828 * We're reading rwa.err without locks, which is safe since we are the
2829 * only reader, and the worker thread is the only writer. It's ok if we
2830 * miss a write for an iteration or two of the loop, since the writer
2831 * thread will keep freeing records we send it until we send it an eos
2832 * marker.
2833 *
2834 * We can leave this loop in 3 ways: First, if rwa.err is
2835 * non-zero. In that case, the writer thread will free the rrd we just
2836 * pushed. Second, if we're interrupted; in that case, either it's the
2837 * first loop and ra.rrd was never allocated, or it's later, and ra.rrd
2838 * has been handed off to the writer thread who will free it. Finally,
2839 * if receive_read_record fails or we're at the end of the stream, then
2840 * we free ra.rrd and exit.
2841 */
2842 while (rwa.err == 0) {
2843 if (issig(JUSTLOOKING) && issig(FORREAL)) {
2844 err = SET_ERROR(EINTR);
2845 break;
2846 }
2847
2848 ASSERT3P(ra.rrd, ==, NULL);
2849 ra.rrd = ra.next_rrd;
2850 ra.next_rrd = NULL;
2851 /* Allocates and loads header into ra.next_rrd */
2852 err = receive_read_record(&ra);
2853
2854 if (ra.rrd->header.drr_type == DRR_END || err != 0) {
2855 kmem_free(ra.rrd, sizeof (*ra.rrd));
2856 ra.rrd = NULL;
2857 break;
2858 }
2859
2860 bqueue_enqueue(&rwa.q, ra.rrd,
2861 sizeof (struct receive_record_arg) + ra.rrd->payload_size);
2862 ra.rrd = NULL;
2863 }
2864 if (ra.next_rrd == NULL)
2865 ra.next_rrd = kmem_zalloc(sizeof (*ra.next_rrd), KM_SLEEP);
2866 ra.next_rrd->eos_marker = B_TRUE;
2867 bqueue_enqueue(&rwa.q, ra.next_rrd, 1);
2868
2869 mutex_enter(&rwa.mutex);
2870 while (!rwa.done) {
2871 cv_wait(&rwa.cv, &rwa.mutex);
2872 }
2873 mutex_exit(&rwa.mutex);
2874
2875 cv_destroy(&rwa.cv);
2876 mutex_destroy(&rwa.mutex);
2877 bqueue_destroy(&rwa.q);
2878 if (err == 0)
2879 err = rwa.err;
2880
2881 out:
2882 nvlist_free(begin_nvl);
2883 if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
2884 zfs_onexit_fd_rele(cleanup_fd);
2885
2886 if (err != 0) {
2887 /*
2888 * Clean up references. If receive is not resumable,
2889 * destroy what we created, so we don't leave it in
2890 * the inconsistent state.
2891 */
2892 dmu_recv_cleanup_ds(drc);
2893 }
2894
2895 *voffp = ra.voff;
2896 objlist_destroy(&ra.ignore_objlist);
2897 return (err);
2898 }
2899
2900 static int
2901 dmu_recv_end_check(void *arg, dmu_tx_t *tx)
2902 {
2903 dmu_recv_cookie_t *drc = arg;
2904 dsl_pool_t *dp = dmu_tx_pool(tx);
2905 int error;
2906
2907 ASSERT3P(drc->drc_ds->ds_owner, ==, dmu_recv_tag);
2908
2909 if (!drc->drc_newfs) {
2910 dsl_dataset_t *origin_head;
2911
2912 error = dsl_dataset_hold(dp, drc->drc_tofs, FTAG, &origin_head);
2913 if (error != 0)
2914 return (error);
2915 if (drc->drc_force) {
2916 /*
2917 * We will destroy any snapshots in tofs (i.e. before
2918 * origin_head) that are after the origin (which is
2919 * the snap before drc_ds, because drc_ds can not
2920 * have any snaps of its own).
2921 */
2922 uint64_t obj;
2923
2924 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
2925 while (obj !=
2926 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
2927 dsl_dataset_t *snap;
2928 error = dsl_dataset_hold_obj(dp, obj, FTAG,
2929 &snap);
2930 if (error != 0)
2931 break;
2932 if (snap->ds_dir != origin_head->ds_dir)
2933 error = SET_ERROR(EINVAL);
2934 if (error == 0) {
2935 error = dsl_destroy_snapshot_check_impl(
2936 snap, B_FALSE);
2937 }
2938 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
2939 dsl_dataset_rele(snap, FTAG);
2940 if (error != 0)
2941 break;
2942 }
2943 if (error != 0) {
2944 dsl_dataset_rele(origin_head, FTAG);
2945 return (error);
2946 }
2947 }
2948 error = dsl_dataset_clone_swap_check_impl(drc->drc_ds,
2949 origin_head, drc->drc_force, drc->drc_owner, tx);
2950 if (error != 0) {
2951 dsl_dataset_rele(origin_head, FTAG);
2952 return (error);
2953 }
2954 error = dsl_dataset_snapshot_check_impl(origin_head,
2955 drc->drc_tosnap, tx, B_TRUE, 1, drc->drc_cred);
2956 dsl_dataset_rele(origin_head, FTAG);
2957 if (error != 0)
2958 return (error);
2959
2960 error = dsl_destroy_head_check_impl(drc->drc_ds, 1);
2961 } else {
2962 error = dsl_dataset_snapshot_check_impl(drc->drc_ds,
2963 drc->drc_tosnap, tx, B_TRUE, 1, drc->drc_cred);
2964 }
2965 return (error);
2966 }
2967
2968 static void
2969 dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
2970 {
2971 dmu_recv_cookie_t *drc = arg;
2972 dsl_pool_t *dp = dmu_tx_pool(tx);
2973
2974 spa_history_log_internal_ds(drc->drc_ds, "finish receiving",
2975 tx, "snap=%s", drc->drc_tosnap);
2976
2977 if (!drc->drc_newfs) {
2978 dsl_dataset_t *origin_head;
2979
2980 VERIFY0(dsl_dataset_hold(dp, drc->drc_tofs, FTAG,
2981 &origin_head));
2982
2983 if (drc->drc_force) {
2984 /*
2985 * Destroy any snapshots of drc_tofs (origin_head)
2986 * after the origin (the snap before drc_ds).
2987 */
2988 uint64_t obj;
2989
2990 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
2991 while (obj !=
2992 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
2993 dsl_dataset_t *snap;
2994 VERIFY0(dsl_dataset_hold_obj(dp, obj, FTAG,
2995 &snap));
2996 ASSERT3P(snap->ds_dir, ==, origin_head->ds_dir);
2997 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
2998 dsl_destroy_snapshot_sync_impl(snap,
2999 B_FALSE, tx);
3000 dsl_dataset_rele(snap, FTAG);
3001 }
3002 }
3003 VERIFY3P(drc->drc_ds->ds_prev, ==,
3004 origin_head->ds_prev);
3005
3006 dsl_dataset_clone_swap_sync_impl(drc->drc_ds,
3007 origin_head, tx);
3008 dsl_dataset_snapshot_sync_impl(origin_head,
3009 drc->drc_tosnap, tx);
3010
3011 /* set snapshot's creation time and guid */
3012 dmu_buf_will_dirty(origin_head->ds_prev->ds_dbuf, tx);
3013 dsl_dataset_phys(origin_head->ds_prev)->ds_creation_time =
3014 drc->drc_drrb->drr_creation_time;
3015 dsl_dataset_phys(origin_head->ds_prev)->ds_guid =
3016 drc->drc_drrb->drr_toguid;
3017 dsl_dataset_phys(origin_head->ds_prev)->ds_flags &=
3018 ~DS_FLAG_INCONSISTENT;
3019
3020 dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
3021 dsl_dataset_phys(origin_head)->ds_flags &=
3022 ~DS_FLAG_INCONSISTENT;
3023
3024 dsl_dataset_rele(origin_head, FTAG);
3025 dsl_destroy_head_sync_impl(drc->drc_ds, tx);
3026
3027 if (drc->drc_owner != NULL)
3028 VERIFY3P(origin_head->ds_owner, ==, drc->drc_owner);
3029 } else {
3030 dsl_dataset_t *ds = drc->drc_ds;
3031
3032 dsl_dataset_snapshot_sync_impl(ds, drc->drc_tosnap, tx);
3033
3034 /* set snapshot's creation time and guid */
3035 dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
3036 dsl_dataset_phys(ds->ds_prev)->ds_creation_time =
3037 drc->drc_drrb->drr_creation_time;
3038 dsl_dataset_phys(ds->ds_prev)->ds_guid =
3039 drc->drc_drrb->drr_toguid;
3040 dsl_dataset_phys(ds->ds_prev)->ds_flags &=
3041 ~DS_FLAG_INCONSISTENT;
3042
3043 dmu_buf_will_dirty(ds->ds_dbuf, tx);
3044 dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
3045 if (dsl_dataset_has_resume_receive_state(ds)) {
3046 (void) zap_remove(dp->dp_meta_objset, ds->ds_object,
3047 DS_FIELD_RESUME_FROMGUID, tx);
3048 (void) zap_remove(dp->dp_meta_objset, ds->ds_object,
3049 DS_FIELD_RESUME_OBJECT, tx);
3050 (void) zap_remove(dp->dp_meta_objset, ds->ds_object,
3051 DS_FIELD_RESUME_OFFSET, tx);
3052 (void) zap_remove(dp->dp_meta_objset, ds->ds_object,
3053 DS_FIELD_RESUME_BYTES, tx);
3054 (void) zap_remove(dp->dp_meta_objset, ds->ds_object,
3055 DS_FIELD_RESUME_TOGUID, tx);
3056 (void) zap_remove(dp->dp_meta_objset, ds->ds_object,
3057 DS_FIELD_RESUME_TONAME, tx);
3058 }
3059 }
3060 drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
3061 /*
3062 * Release the hold from dmu_recv_begin. This must be done before
3063 * we return to open context, so that when we free the dataset's dnode,
3064 * we can evict its bonus buffer.
3065 */
3066 dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
3067 drc->drc_ds = NULL;
3068 }
3069
3070 static int
3071 add_ds_to_guidmap(const char *name, avl_tree_t *guid_map, uint64_t snapobj)
3072 {
3073 dsl_pool_t *dp;
3074 dsl_dataset_t *snapds;
3075 guid_map_entry_t *gmep;
3076 int err;
3077
3078 ASSERT(guid_map != NULL);
3079
3080 err = dsl_pool_hold(name, FTAG, &dp);
3081 if (err != 0)
3082 return (err);
3083 gmep = kmem_alloc(sizeof (*gmep), KM_SLEEP);
3084 err = dsl_dataset_hold_obj(dp, snapobj, gmep, &snapds);
3085 if (err == 0) {
3086 gmep->guid = dsl_dataset_phys(snapds)->ds_guid;
3087 gmep->gme_ds = snapds;
3088 avl_add(guid_map, gmep);
3089 dsl_dataset_long_hold(snapds, gmep);
3090 } else {
3091 kmem_free(gmep, sizeof (*gmep));
3092 }
3093
3094 dsl_pool_rele(dp, FTAG);
3095 return (err);
3096 }
3097
3098 static int dmu_recv_end_modified_blocks = 3;
3099
3100 static int
3101 dmu_recv_existing_end(dmu_recv_cookie_t *drc)
3102 {
3103 int error;
3104 char name[MAXNAMELEN];
3105
3106 #ifdef _KERNEL
3107 /*
3108 * We will be destroying the ds; make sure its origin is unmounted if
3109 * necessary.
3110 */
3111 dsl_dataset_name(drc->drc_ds, name);
3112 zfs_destroy_unmount_origin(name);
3113 #endif
3114
3115 error = dsl_sync_task(drc->drc_tofs,
3116 dmu_recv_end_check, dmu_recv_end_sync, drc,
3117 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
3118
3119 if (error != 0)
3120 dmu_recv_cleanup_ds(drc);
3121 return (error);
3122 }
3123
3124 static int
3125 dmu_recv_new_end(dmu_recv_cookie_t *drc)
3126 {
3127 int error;
3128
3129 error = dsl_sync_task(drc->drc_tofs,
3130 dmu_recv_end_check, dmu_recv_end_sync, drc,
3131 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
3132
3133 if (error != 0) {
3134 dmu_recv_cleanup_ds(drc);
3135 } else if (drc->drc_guid_to_ds_map != NULL) {
3136 (void) add_ds_to_guidmap(drc->drc_tofs,
3137 drc->drc_guid_to_ds_map,
3138 drc->drc_newsnapobj);
3139 }
3140 return (error);
3141 }
3142
3143 int
3144 dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
3145 {
3146 drc->drc_owner = owner;
3147
3148 if (drc->drc_newfs)
3149 return (dmu_recv_new_end(drc));
3150 else
3151 return (dmu_recv_existing_end(drc));
3152 }
3153
3154 /*
3155 * Return TRUE if this objset is currently being received into.
3156 */
3157 boolean_t
3158 dmu_objset_is_receiving(objset_t *os)
3159 {
3160 return (os->os_dsl_dataset != NULL &&
3161 os->os_dsl_dataset->ds_owner == dmu_recv_tag);
3162 }