xref: /titanic_51/usr/src/lib/libzfs/common/libzfs_sendrecv.c (revision c586600796766c83eb9485c446886fd9ed2359a9)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <assert.h>
28 #include <ctype.h>
29 #include <errno.h>
30 #include <libintl.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <strings.h>
34 #include <unistd.h>
35 #include <stddef.h>
36 #include <fcntl.h>
37 #include <sys/mount.h>
38 #include <pthread.h>
39 #include <umem.h>
40 
41 #include <libzfs.h>
42 
43 #include "zfs_namecheck.h"
44 #include "zfs_prop.h"
45 #include "zfs_fletcher.h"
46 #include "libzfs_impl.h"
47 #include <sha2.h>
48 #include <sys/zio_checksum.h>
49 #include <sys/ddt.h>
50 
51 /* in libzfs_dataset.c */
52 extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *);
53 
54 static int zfs_receive_impl(libzfs_handle_t *, const char *, recvflags_t,
55     int, const char *, nvlist_t *, avl_tree_t *, char **);
56 
57 static const zio_cksum_t zero_cksum = { 0 };
58 
59 typedef struct dedup_arg {
60 	int	inputfd;
61 	int	outputfd;
62 	libzfs_handle_t  *dedup_hdl;
63 } dedup_arg_t;
64 
65 typedef struct dataref {
66 	uint64_t ref_guid;
67 	uint64_t ref_object;
68 	uint64_t ref_offset;
69 } dataref_t;
70 
71 typedef struct dedup_entry {
72 	struct dedup_entry	*dde_next;
73 	zio_cksum_t dde_chksum;
74 	uint64_t dde_prop;
75 	dataref_t dde_ref;
76 } dedup_entry_t;
77 
78 #define	MAX_DDT_PHYSMEM_PERCENT		20
79 #define	SMALLEST_POSSIBLE_MAX_DDT_MB		128
80 
81 typedef struct dedup_table {
82 	dedup_entry_t	**dedup_hash_array;
83 	umem_cache_t	*ddecache;
84 	uint64_t	max_ddt_size;  /* max dedup table size in bytes */
85 	uint64_t	cur_ddt_size;  /* current dedup table size in bytes */
86 	uint64_t	ddt_count;
87 	int		numhashbits;
88 	boolean_t	ddt_full;
89 } dedup_table_t;
90 
91 static int
92 high_order_bit(uint64_t n)
93 {
94 	int count;
95 
96 	for (count = 0; n != 0; count++)
97 		n >>= 1;
98 	return (count);
99 }
100 
101 static size_t
102 ssread(void *buf, size_t len, FILE *stream)
103 {
104 	size_t outlen;
105 
106 	if ((outlen = fread(buf, len, 1, stream)) == 0)
107 		return (0);
108 
109 	return (outlen);
110 }
111 
112 static void
113 ddt_hash_append(libzfs_handle_t *hdl, dedup_table_t *ddt, dedup_entry_t **ddepp,
114     zio_cksum_t *cs, uint64_t prop, dataref_t *dr)
115 {
116 	dedup_entry_t	*dde;
117 
118 	if (ddt->cur_ddt_size >= ddt->max_ddt_size) {
119 		if (ddt->ddt_full == B_FALSE) {
120 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
121 			    "Dedup table full.  Deduplication will continue "
122 			    "with existing table entries"));
123 			ddt->ddt_full = B_TRUE;
124 		}
125 		return;
126 	}
127 
128 	if ((dde = umem_cache_alloc(ddt->ddecache, UMEM_DEFAULT))
129 	    != NULL) {
130 		assert(*ddepp == NULL);
131 		dde->dde_next = NULL;
132 		dde->dde_chksum = *cs;
133 		dde->dde_prop = prop;
134 		dde->dde_ref = *dr;
135 		*ddepp = dde;
136 		ddt->cur_ddt_size += sizeof (dedup_entry_t);
137 		ddt->ddt_count++;
138 	}
139 }
140 
141 /*
142  * Using the specified dedup table, do a lookup for an entry with
143  * the checksum cs.  If found, return the block's reference info
144  * in *dr. Otherwise, insert a new entry in the dedup table, using
145  * the reference information specified by *dr.
146  *
147  * return value:  true - entry was found
148  *		  false - entry was not found
149  */
150 static boolean_t
151 ddt_update(libzfs_handle_t *hdl, dedup_table_t *ddt, zio_cksum_t *cs,
152     uint64_t prop, dataref_t *dr)
153 {
154 	uint32_t hashcode;
155 	dedup_entry_t **ddepp;
156 
157 	hashcode = BF64_GET(cs->zc_word[0], 0, ddt->numhashbits);
158 
159 	for (ddepp = &(ddt->dedup_hash_array[hashcode]); *ddepp != NULL;
160 	    ddepp = &((*ddepp)->dde_next)) {
161 		if (ZIO_CHECKSUM_EQUAL(((*ddepp)->dde_chksum), *cs) &&
162 		    (*ddepp)->dde_prop == prop) {
163 			*dr = (*ddepp)->dde_ref;
164 			return (B_TRUE);
165 		}
166 	}
167 	ddt_hash_append(hdl, ddt, ddepp, cs, prop, dr);
168 	return (B_FALSE);
169 }
170 
171 static int
172 cksum_and_write(const void *buf, uint64_t len, zio_cksum_t *zc, int outfd)
173 {
174 	fletcher_4_incremental_native(buf, len, zc);
175 	return (write(outfd, buf, len));
176 }
177 
178 /*
179  * This function is started in a separate thread when the dedup option
180  * has been requested.  The main send thread determines the list of
181  * snapshots to be included in the send stream and makes the ioctl calls
182  * for each one.  But instead of having the ioctl send the output to the
183  * the output fd specified by the caller of zfs_send()), the
184  * ioctl is told to direct the output to a pipe, which is read by the
185  * alternate thread running THIS function.  This function does the
186  * dedup'ing by:
187  *  1. building a dedup table (the DDT)
188  *  2. doing checksums on each data block and inserting a record in the DDT
189  *  3. looking for matching checksums, and
190  *  4.  sending a DRR_WRITE_BYREF record instead of a write record whenever
191  *      a duplicate block is found.
192  * The output of this function then goes to the output fd requested
193  * by the caller of zfs_send().
194  */
195 static void *
196 cksummer(void *arg)
197 {
198 	dedup_arg_t *dda = arg;
199 	char *buf = malloc(1<<20);
200 	dmu_replay_record_t thedrr;
201 	dmu_replay_record_t *drr = &thedrr;
202 	struct drr_begin *drrb = &thedrr.drr_u.drr_begin;
203 	struct drr_end *drre = &thedrr.drr_u.drr_end;
204 	struct drr_object *drro = &thedrr.drr_u.drr_object;
205 	struct drr_write *drrw = &thedrr.drr_u.drr_write;
206 	struct drr_spill *drrs = &thedrr.drr_u.drr_spill;
207 	FILE *ofp;
208 	int outfd;
209 	dmu_replay_record_t wbr_drr = {0};
210 	struct drr_write_byref *wbr_drrr = &wbr_drr.drr_u.drr_write_byref;
211 	dedup_table_t ddt;
212 	zio_cksum_t stream_cksum;
213 	uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
214 	uint64_t numbuckets;
215 
216 	ddt.max_ddt_size =
217 	    MAX((physmem * MAX_DDT_PHYSMEM_PERCENT)/100,
218 	    SMALLEST_POSSIBLE_MAX_DDT_MB<<20);
219 
220 	numbuckets = ddt.max_ddt_size/(sizeof (dedup_entry_t));
221 
222 	/*
223 	 * numbuckets must be a power of 2.  Increase number to
224 	 * a power of 2 if necessary.
225 	 */
226 	if (!ISP2(numbuckets))
227 		numbuckets = 1 << high_order_bit(numbuckets);
228 
229 	ddt.dedup_hash_array = calloc(numbuckets, sizeof (dedup_entry_t *));
230 	ddt.ddecache = umem_cache_create("dde", sizeof (dedup_entry_t), 0,
231 	    NULL, NULL, NULL, NULL, NULL, 0);
232 	ddt.cur_ddt_size = numbuckets * sizeof (dedup_entry_t *);
233 	ddt.numhashbits = high_order_bit(numbuckets) - 1;
234 	ddt.ddt_full = B_FALSE;
235 
236 	/* Initialize the write-by-reference block. */
237 	wbr_drr.drr_type = DRR_WRITE_BYREF;
238 	wbr_drr.drr_payloadlen = 0;
239 
240 	outfd = dda->outputfd;
241 	ofp = fdopen(dda->inputfd, "r");
242 	while (ssread(drr, sizeof (dmu_replay_record_t), ofp) != 0) {
243 
244 		switch (drr->drr_type) {
245 		case DRR_BEGIN:
246 		{
247 			int	fflags;
248 			ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0);
249 
250 			/* set the DEDUP feature flag for this stream */
251 			fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
252 			fflags |= (DMU_BACKUP_FEATURE_DEDUP |
253 			    DMU_BACKUP_FEATURE_DEDUPPROPS);
254 			DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags);
255 
256 			if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
257 			    &stream_cksum, outfd) == -1)
258 				goto out;
259 			if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
260 			    DMU_COMPOUNDSTREAM && drr->drr_payloadlen != 0) {
261 				int sz = drr->drr_payloadlen;
262 
263 				if (sz > 1<<20) {
264 					free(buf);
265 					buf = malloc(sz);
266 				}
267 				(void) ssread(buf, sz, ofp);
268 				if (ferror(stdin))
269 					perror("fread");
270 				if (cksum_and_write(buf, sz, &stream_cksum,
271 				    outfd) == -1)
272 					goto out;
273 			}
274 			break;
275 		}
276 
277 		case DRR_END:
278 		{
279 			/* use the recalculated checksum */
280 			ZIO_SET_CHECKSUM(&drre->drr_checksum,
281 			    stream_cksum.zc_word[0], stream_cksum.zc_word[1],
282 			    stream_cksum.zc_word[2], stream_cksum.zc_word[3]);
283 			if ((write(outfd, drr,
284 			    sizeof (dmu_replay_record_t))) == -1)
285 				goto out;
286 			break;
287 		}
288 
289 		case DRR_OBJECT:
290 		{
291 			if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
292 			    &stream_cksum, outfd) == -1)
293 				goto out;
294 			if (drro->drr_bonuslen > 0) {
295 				(void) ssread(buf,
296 				    P2ROUNDUP((uint64_t)drro->drr_bonuslen, 8),
297 				    ofp);
298 				if (cksum_and_write(buf,
299 				    P2ROUNDUP((uint64_t)drro->drr_bonuslen, 8),
300 				    &stream_cksum, outfd) == -1)
301 					goto out;
302 			}
303 			break;
304 		}
305 
306 		case DRR_SPILL:
307 		{
308 			if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
309 			    &stream_cksum, outfd) == -1)
310 				goto out;
311 			(void) ssread(buf, drrs->drr_length, ofp);
312 			if (cksum_and_write(buf, drrs->drr_length,
313 			    &stream_cksum, outfd) == -1)
314 				goto out;
315 			break;
316 		}
317 
318 		case DRR_FREEOBJECTS:
319 		{
320 			if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
321 			    &stream_cksum, outfd) == -1)
322 				goto out;
323 			break;
324 		}
325 
326 		case DRR_WRITE:
327 		{
328 			dataref_t	dataref;
329 
330 			(void) ssread(buf, drrw->drr_length, ofp);
331 
332 			/*
333 			 * Use the existing checksum if it's dedup-capable,
334 			 * else calculate a SHA256 checksum for it.
335 			 */
336 
337 			if (ZIO_CHECKSUM_EQUAL(drrw->drr_key.ddk_cksum,
338 			    zero_cksum) ||
339 			    !DRR_IS_DEDUP_CAPABLE(drrw->drr_checksumflags)) {
340 				SHA256_CTX	ctx;
341 				zio_cksum_t	tmpsha256;
342 
343 				SHA256Init(&ctx);
344 				SHA256Update(&ctx, buf, drrw->drr_length);
345 				SHA256Final(&tmpsha256, &ctx);
346 				drrw->drr_key.ddk_cksum.zc_word[0] =
347 				    BE_64(tmpsha256.zc_word[0]);
348 				drrw->drr_key.ddk_cksum.zc_word[1] =
349 				    BE_64(tmpsha256.zc_word[1]);
350 				drrw->drr_key.ddk_cksum.zc_word[2] =
351 				    BE_64(tmpsha256.zc_word[2]);
352 				drrw->drr_key.ddk_cksum.zc_word[3] =
353 				    BE_64(tmpsha256.zc_word[3]);
354 				drrw->drr_checksumtype = ZIO_CHECKSUM_SHA256;
355 				drrw->drr_checksumflags = DRR_CHECKSUM_DEDUP;
356 			}
357 
358 			dataref.ref_guid = drrw->drr_toguid;
359 			dataref.ref_object = drrw->drr_object;
360 			dataref.ref_offset = drrw->drr_offset;
361 
362 			if (ddt_update(dda->dedup_hdl, &ddt,
363 			    &drrw->drr_key.ddk_cksum, drrw->drr_key.ddk_prop,
364 			    &dataref)) {
365 				/* block already present in stream */
366 				wbr_drrr->drr_object = drrw->drr_object;
367 				wbr_drrr->drr_offset = drrw->drr_offset;
368 				wbr_drrr->drr_length = drrw->drr_length;
369 				wbr_drrr->drr_toguid = drrw->drr_toguid;
370 				wbr_drrr->drr_refguid = dataref.ref_guid;
371 				wbr_drrr->drr_refobject =
372 				    dataref.ref_object;
373 				wbr_drrr->drr_refoffset =
374 				    dataref.ref_offset;
375 
376 				wbr_drrr->drr_checksumtype =
377 				    drrw->drr_checksumtype;
378 				wbr_drrr->drr_checksumflags =
379 				    drrw->drr_checksumtype;
380 				wbr_drrr->drr_key.ddk_cksum =
381 				    drrw->drr_key.ddk_cksum;
382 				wbr_drrr->drr_key.ddk_prop =
383 				    drrw->drr_key.ddk_prop;
384 
385 				if (cksum_and_write(&wbr_drr,
386 				    sizeof (dmu_replay_record_t), &stream_cksum,
387 				    outfd) == -1)
388 					goto out;
389 			} else {
390 				/* block not previously seen */
391 				if (cksum_and_write(drr,
392 				    sizeof (dmu_replay_record_t), &stream_cksum,
393 				    outfd) == -1)
394 					goto out;
395 				if (cksum_and_write(buf,
396 				    drrw->drr_length,
397 				    &stream_cksum, outfd) == -1)
398 					goto out;
399 			}
400 			break;
401 		}
402 
403 		case DRR_FREE:
404 		{
405 			if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
406 			    &stream_cksum, outfd) == -1)
407 				goto out;
408 			break;
409 		}
410 
411 		default:
412 			(void) printf("INVALID record type 0x%x\n",
413 			    drr->drr_type);
414 			/* should never happen, so assert */
415 			assert(B_FALSE);
416 		}
417 	}
418 out:
419 	umem_cache_destroy(ddt.ddecache);
420 	free(ddt.dedup_hash_array);
421 	free(buf);
422 	(void) fclose(ofp);
423 
424 	return (NULL);
425 }
426 
427 /*
428  * Routines for dealing with the AVL tree of fs-nvlists
429  */
430 typedef struct fsavl_node {
431 	avl_node_t fn_node;
432 	nvlist_t *fn_nvfs;
433 	char *fn_snapname;
434 	uint64_t fn_guid;
435 } fsavl_node_t;
436 
437 static int
438 fsavl_compare(const void *arg1, const void *arg2)
439 {
440 	const fsavl_node_t *fn1 = arg1;
441 	const fsavl_node_t *fn2 = arg2;
442 
443 	if (fn1->fn_guid > fn2->fn_guid)
444 		return (+1);
445 	else if (fn1->fn_guid < fn2->fn_guid)
446 		return (-1);
447 	else
448 		return (0);
449 }
450 
451 /*
452  * Given the GUID of a snapshot, find its containing filesystem and
453  * (optionally) name.
454  */
455 static nvlist_t *
456 fsavl_find(avl_tree_t *avl, uint64_t snapguid, char **snapname)
457 {
458 	fsavl_node_t fn_find;
459 	fsavl_node_t *fn;
460 
461 	fn_find.fn_guid = snapguid;
462 
463 	fn = avl_find(avl, &fn_find, NULL);
464 	if (fn) {
465 		if (snapname)
466 			*snapname = fn->fn_snapname;
467 		return (fn->fn_nvfs);
468 	}
469 	return (NULL);
470 }
471 
472 static void
473 fsavl_destroy(avl_tree_t *avl)
474 {
475 	fsavl_node_t *fn;
476 	void *cookie;
477 
478 	if (avl == NULL)
479 		return;
480 
481 	cookie = NULL;
482 	while ((fn = avl_destroy_nodes(avl, &cookie)) != NULL)
483 		free(fn);
484 	avl_destroy(avl);
485 	free(avl);
486 }
487 
488 /*
489  * Given an nvlist, produce an avl tree of snapshots, ordered by guid
490  */
491 static avl_tree_t *
492 fsavl_create(nvlist_t *fss)
493 {
494 	avl_tree_t *fsavl;
495 	nvpair_t *fselem = NULL;
496 
497 	if ((fsavl = malloc(sizeof (avl_tree_t))) == NULL)
498 		return (NULL);
499 
500 	avl_create(fsavl, fsavl_compare, sizeof (fsavl_node_t),
501 	    offsetof(fsavl_node_t, fn_node));
502 
503 	while ((fselem = nvlist_next_nvpair(fss, fselem)) != NULL) {
504 		nvlist_t *nvfs, *snaps;
505 		nvpair_t *snapelem = NULL;
506 
507 		VERIFY(0 == nvpair_value_nvlist(fselem, &nvfs));
508 		VERIFY(0 == nvlist_lookup_nvlist(nvfs, "snaps", &snaps));
509 
510 		while ((snapelem =
511 		    nvlist_next_nvpair(snaps, snapelem)) != NULL) {
512 			fsavl_node_t *fn;
513 			uint64_t guid;
514 
515 			VERIFY(0 == nvpair_value_uint64(snapelem, &guid));
516 			if ((fn = malloc(sizeof (fsavl_node_t))) == NULL) {
517 				fsavl_destroy(fsavl);
518 				return (NULL);
519 			}
520 			fn->fn_nvfs = nvfs;
521 			fn->fn_snapname = nvpair_name(snapelem);
522 			fn->fn_guid = guid;
523 
524 			/*
525 			 * Note: if there are multiple snaps with the
526 			 * same GUID, we ignore all but one.
527 			 */
528 			if (avl_find(fsavl, fn, NULL) == NULL)
529 				avl_add(fsavl, fn);
530 			else
531 				free(fn);
532 		}
533 	}
534 
535 	return (fsavl);
536 }
537 
538 /*
539  * Routines for dealing with the giant nvlist of fs-nvlists, etc.
540  */
541 typedef struct send_data {
542 	uint64_t parent_fromsnap_guid;
543 	nvlist_t *parent_snaps;
544 	nvlist_t *fss;
545 	nvlist_t *snapprops;
546 	const char *fromsnap;
547 	const char *tosnap;
548 	boolean_t recursive;
549 
550 	/*
551 	 * The header nvlist is of the following format:
552 	 * {
553 	 *   "tosnap" -> string
554 	 *   "fromsnap" -> string (if incremental)
555 	 *   "fss" -> {
556 	 *	id -> {
557 	 *
558 	 *	 "name" -> string (full name; for debugging)
559 	 *	 "parentfromsnap" -> number (guid of fromsnap in parent)
560 	 *
561 	 *	 "props" -> { name -> value (only if set here) }
562 	 *	 "snaps" -> { name (lastname) -> number (guid) }
563 	 *	 "snapprops" -> { name (lastname) -> { name -> value } }
564 	 *
565 	 *	 "origin" -> number (guid) (if clone)
566 	 *	 "sent" -> boolean (not on-disk)
567 	 *	}
568 	 *   }
569 	 * }
570 	 *
571 	 */
572 } send_data_t;
573 
574 static void send_iterate_prop(zfs_handle_t *zhp, nvlist_t *nv);
575 
576 static int
577 send_iterate_snap(zfs_handle_t *zhp, void *arg)
578 {
579 	send_data_t *sd = arg;
580 	uint64_t guid = zhp->zfs_dmustats.dds_guid;
581 	char *snapname;
582 	nvlist_t *nv;
583 
584 	snapname = strrchr(zhp->zfs_name, '@')+1;
585 
586 	VERIFY(0 == nvlist_add_uint64(sd->parent_snaps, snapname, guid));
587 	/*
588 	 * NB: if there is no fromsnap here (it's a newly created fs in
589 	 * an incremental replication), we will substitute the tosnap.
590 	 */
591 	if ((sd->fromsnap && strcmp(snapname, sd->fromsnap) == 0) ||
592 	    (sd->parent_fromsnap_guid == 0 && sd->tosnap &&
593 	    strcmp(snapname, sd->tosnap) == 0)) {
594 		sd->parent_fromsnap_guid = guid;
595 	}
596 
597 	VERIFY(0 == nvlist_alloc(&nv, NV_UNIQUE_NAME, 0));
598 	send_iterate_prop(zhp, nv);
599 	VERIFY(0 == nvlist_add_nvlist(sd->snapprops, snapname, nv));
600 	nvlist_free(nv);
601 
602 	zfs_close(zhp);
603 	return (0);
604 }
605 
606 static void
607 send_iterate_prop(zfs_handle_t *zhp, nvlist_t *nv)
608 {
609 	nvpair_t *elem = NULL;
610 
611 	while ((elem = nvlist_next_nvpair(zhp->zfs_props, elem)) != NULL) {
612 		char *propname = nvpair_name(elem);
613 		zfs_prop_t prop = zfs_name_to_prop(propname);
614 		nvlist_t *propnv;
615 
616 		if (!zfs_prop_user(propname)) {
617 			/*
618 			 * Realistically, this should never happen.  However,
619 			 * we want the ability to add DSL properties without
620 			 * needing to make incompatible version changes.  We
621 			 * need to ignore unknown properties to allow older
622 			 * software to still send datasets containing these
623 			 * properties, with the unknown properties elided.
624 			 */
625 			if (prop == ZPROP_INVAL)
626 				continue;
627 
628 			if (zfs_prop_readonly(prop))
629 				continue;
630 		}
631 
632 		verify(nvpair_value_nvlist(elem, &propnv) == 0);
633 		if (prop == ZFS_PROP_QUOTA || prop == ZFS_PROP_RESERVATION ||
634 		    prop == ZFS_PROP_REFQUOTA ||
635 		    prop == ZFS_PROP_REFRESERVATION) {
636 			char *source;
637 			uint64_t value;
638 			verify(nvlist_lookup_uint64(propnv,
639 			    ZPROP_VALUE, &value) == 0);
640 			if (zhp->zfs_type == ZFS_TYPE_SNAPSHOT)
641 				continue;
642 			/*
643 			 * May have no source before SPA_VERSION_RECVD_PROPS,
644 			 * but is still modifiable.
645 			 */
646 			if (nvlist_lookup_string(propnv,
647 			    ZPROP_SOURCE, &source) == 0) {
648 				if ((strcmp(source, zhp->zfs_name) != 0) &&
649 				    (strcmp(source,
650 				    ZPROP_SOURCE_VAL_RECVD) != 0))
651 					continue;
652 			}
653 		} else {
654 			char *source;
655 			if (nvlist_lookup_string(propnv,
656 			    ZPROP_SOURCE, &source) != 0)
657 				continue;
658 			if ((strcmp(source, zhp->zfs_name) != 0) &&
659 			    (strcmp(source, ZPROP_SOURCE_VAL_RECVD) != 0))
660 				continue;
661 		}
662 
663 		if (zfs_prop_user(propname) ||
664 		    zfs_prop_get_type(prop) == PROP_TYPE_STRING) {
665 			char *value;
666 			verify(nvlist_lookup_string(propnv,
667 			    ZPROP_VALUE, &value) == 0);
668 			VERIFY(0 == nvlist_add_string(nv, propname, value));
669 		} else {
670 			uint64_t value;
671 			verify(nvlist_lookup_uint64(propnv,
672 			    ZPROP_VALUE, &value) == 0);
673 			VERIFY(0 == nvlist_add_uint64(nv, propname, value));
674 		}
675 	}
676 }
677 
678 /*
679  * recursively generate nvlists describing datasets.  See comment
680  * for the data structure send_data_t above for description of contents
681  * of the nvlist.
682  */
683 static int
684 send_iterate_fs(zfs_handle_t *zhp, void *arg)
685 {
686 	send_data_t *sd = arg;
687 	nvlist_t *nvfs, *nv;
688 	int rv = 0;
689 	uint64_t parent_fromsnap_guid_save = sd->parent_fromsnap_guid;
690 	uint64_t guid = zhp->zfs_dmustats.dds_guid;
691 	char guidstring[64];
692 
693 	VERIFY(0 == nvlist_alloc(&nvfs, NV_UNIQUE_NAME, 0));
694 	VERIFY(0 == nvlist_add_string(nvfs, "name", zhp->zfs_name));
695 	VERIFY(0 == nvlist_add_uint64(nvfs, "parentfromsnap",
696 	    sd->parent_fromsnap_guid));
697 
698 	if (zhp->zfs_dmustats.dds_origin[0]) {
699 		zfs_handle_t *origin = zfs_open(zhp->zfs_hdl,
700 		    zhp->zfs_dmustats.dds_origin, ZFS_TYPE_SNAPSHOT);
701 		if (origin == NULL)
702 			return (-1);
703 		VERIFY(0 == nvlist_add_uint64(nvfs, "origin",
704 		    origin->zfs_dmustats.dds_guid));
705 	}
706 
707 	/* iterate over props */
708 	VERIFY(0 == nvlist_alloc(&nv, NV_UNIQUE_NAME, 0));
709 	send_iterate_prop(zhp, nv);
710 	VERIFY(0 == nvlist_add_nvlist(nvfs, "props", nv));
711 	nvlist_free(nv);
712 
713 	/* iterate over snaps, and set sd->parent_fromsnap_guid */
714 	sd->parent_fromsnap_guid = 0;
715 	VERIFY(0 == nvlist_alloc(&sd->parent_snaps, NV_UNIQUE_NAME, 0));
716 	VERIFY(0 == nvlist_alloc(&sd->snapprops, NV_UNIQUE_NAME, 0));
717 	(void) zfs_iter_snapshots(zhp, send_iterate_snap, sd);
718 	VERIFY(0 == nvlist_add_nvlist(nvfs, "snaps", sd->parent_snaps));
719 	VERIFY(0 == nvlist_add_nvlist(nvfs, "snapprops", sd->snapprops));
720 	nvlist_free(sd->parent_snaps);
721 	nvlist_free(sd->snapprops);
722 
723 	/* add this fs to nvlist */
724 	(void) snprintf(guidstring, sizeof (guidstring),
725 	    "0x%llx", (longlong_t)guid);
726 	VERIFY(0 == nvlist_add_nvlist(sd->fss, guidstring, nvfs));
727 	nvlist_free(nvfs);
728 
729 	/* iterate over children */
730 	if (sd->recursive)
731 		rv = zfs_iter_filesystems(zhp, send_iterate_fs, sd);
732 
733 	sd->parent_fromsnap_guid = parent_fromsnap_guid_save;
734 
735 	zfs_close(zhp);
736 	return (rv);
737 }
738 
739 static int
740 gather_nvlist(libzfs_handle_t *hdl, const char *fsname, const char *fromsnap,
741     const char *tosnap, boolean_t recursive, nvlist_t **nvlp, avl_tree_t **avlp)
742 {
743 	zfs_handle_t *zhp;
744 	send_data_t sd = { 0 };
745 	int error;
746 
747 	zhp = zfs_open(hdl, fsname, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
748 	if (zhp == NULL)
749 		return (EZFS_BADTYPE);
750 
751 	VERIFY(0 == nvlist_alloc(&sd.fss, NV_UNIQUE_NAME, 0));
752 	sd.fromsnap = fromsnap;
753 	sd.tosnap = tosnap;
754 	sd.recursive = recursive;
755 
756 	if ((error = send_iterate_fs(zhp, &sd)) != 0) {
757 		nvlist_free(sd.fss);
758 		if (avlp != NULL)
759 			*avlp = NULL;
760 		*nvlp = NULL;
761 		return (error);
762 	}
763 
764 	if (avlp != NULL && (*avlp = fsavl_create(sd.fss)) == NULL) {
765 		nvlist_free(sd.fss);
766 		*nvlp = NULL;
767 		return (EZFS_NOMEM);
768 	}
769 
770 	*nvlp = sd.fss;
771 	return (0);
772 }
773 
774 /*
775  * Routines for dealing with the sorted snapshot functionality
776  */
777 typedef struct zfs_node {
778 	zfs_handle_t	*zn_handle;
779 	avl_node_t	zn_avlnode;
780 } zfs_node_t;
781 
782 static int
783 zfs_sort_snaps(zfs_handle_t *zhp, void *data)
784 {
785 	avl_tree_t *avl = data;
786 	zfs_node_t *node = zfs_alloc(zhp->zfs_hdl, sizeof (zfs_node_t));
787 
788 	node->zn_handle = zhp;
789 	avl_add(avl, node);
790 	return (0);
791 }
792 
793 /* ARGSUSED */
794 static int
795 zfs_snapshot_compare(const void *larg, const void *rarg)
796 {
797 	zfs_handle_t *l = ((zfs_node_t *)larg)->zn_handle;
798 	zfs_handle_t *r = ((zfs_node_t *)rarg)->zn_handle;
799 	uint64_t lcreate, rcreate;
800 
801 	/*
802 	 * Sort them according to creation time.  We use the hidden
803 	 * CREATETXG property to get an absolute ordering of snapshots.
804 	 */
805 	lcreate = zfs_prop_get_int(l, ZFS_PROP_CREATETXG);
806 	rcreate = zfs_prop_get_int(r, ZFS_PROP_CREATETXG);
807 
808 	if (lcreate < rcreate)
809 		return (-1);
810 	else if (lcreate > rcreate)
811 		return (+1);
812 	else
813 		return (0);
814 }
815 
816 int
817 zfs_iter_snapshots_sorted(zfs_handle_t *zhp, zfs_iter_f callback, void *data)
818 {
819 	int ret = 0;
820 	zfs_node_t *node;
821 	avl_tree_t avl;
822 	void *cookie = NULL;
823 
824 	avl_create(&avl, zfs_snapshot_compare,
825 	    sizeof (zfs_node_t), offsetof(zfs_node_t, zn_avlnode));
826 
827 	ret = zfs_iter_snapshots(zhp, zfs_sort_snaps, &avl);
828 
829 	for (node = avl_first(&avl); node != NULL; node = AVL_NEXT(&avl, node))
830 		ret |= callback(node->zn_handle, data);
831 
832 	while ((node = avl_destroy_nodes(&avl, &cookie)) != NULL)
833 		free(node);
834 
835 	avl_destroy(&avl);
836 
837 	return (ret);
838 }
839 
840 /*
841  * Routines specific to "zfs send"
842  */
843 typedef struct send_dump_data {
844 	/* these are all just the short snapname (the part after the @) */
845 	const char *fromsnap;
846 	const char *tosnap;
847 	char prevsnap[ZFS_MAXNAMELEN];
848 	boolean_t seenfrom, seento, replicate, doall, fromorigin;
849 	boolean_t verbose;
850 	int outfd;
851 	boolean_t err;
852 	nvlist_t *fss;
853 	avl_tree_t *fsavl;
854 	snapfilter_cb_t *filter_cb;
855 	void *filter_cb_arg;
856 } send_dump_data_t;
857 
858 /*
859  * Dumps a backup of the given snapshot (incremental from fromsnap if it's not
860  * NULL) to the file descriptor specified by outfd.
861  */
862 static int
863 dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, boolean_t fromorigin,
864     int outfd, boolean_t enoent_ok, boolean_t *got_enoent)
865 {
866 	zfs_cmd_t zc = { 0 };
867 	libzfs_handle_t *hdl = zhp->zfs_hdl;
868 
869 	assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);
870 	assert(fromsnap == NULL || fromsnap[0] == '\0' || !fromorigin);
871 
872 	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
873 	if (fromsnap)
874 		(void) strlcpy(zc.zc_value, fromsnap, sizeof (zc.zc_value));
875 	zc.zc_cookie = outfd;
876 	zc.zc_obj = fromorigin;
877 
878 	*got_enoent = B_FALSE;
879 
880 	if (ioctl(zhp->zfs_hdl->libzfs_fd, ZFS_IOC_SEND, &zc) != 0) {
881 		char errbuf[1024];
882 		(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
883 		    "warning: cannot send '%s'"), zhp->zfs_name);
884 
885 		switch (errno) {
886 
887 		case EXDEV:
888 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
889 			    "not an earlier snapshot from the same fs"));
890 			return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));
891 
892 		case ENOENT:
893 			if (enoent_ok) {
894 				*got_enoent = B_TRUE;
895 				return (0);
896 			}
897 			if (zfs_dataset_exists(hdl, zc.zc_name,
898 			    ZFS_TYPE_SNAPSHOT)) {
899 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
900 				    "incremental source (@%s) does not exist"),
901 				    zc.zc_value);
902 			}
903 			return (zfs_error(hdl, EZFS_NOENT, errbuf));
904 
905 		case EDQUOT:
906 		case EFBIG:
907 		case EIO:
908 		case ENOLINK:
909 		case ENOSPC:
910 		case ENOSTR:
911 		case ENXIO:
912 		case EPIPE:
913 		case ERANGE:
914 		case EFAULT:
915 		case EROFS:
916 			zfs_error_aux(hdl, strerror(errno));
917 			return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));
918 
919 		default:
920 			return (zfs_standard_error(hdl, errno, errbuf));
921 		}
922 	}
923 
924 	return (0);
925 }
926 
927 static int
928 dump_snapshot(zfs_handle_t *zhp, void *arg)
929 {
930 	send_dump_data_t *sdd = arg;
931 	const char *thissnap;
932 	int err;
933 	boolean_t got_enoent;
934 
935 	thissnap = strchr(zhp->zfs_name, '@') + 1;
936 
937 	if (sdd->fromsnap && !sdd->seenfrom &&
938 	    strcmp(sdd->fromsnap, thissnap) == 0) {
939 		sdd->seenfrom = B_TRUE;
940 		(void) strcpy(sdd->prevsnap, thissnap);
941 		zfs_close(zhp);
942 		return (0);
943 	}
944 
945 	if (sdd->seento || !sdd->seenfrom) {
946 		zfs_close(zhp);
947 		return (0);
948 	}
949 
950 	if (strcmp(sdd->tosnap, thissnap) == 0)
951 		sdd->seento = B_TRUE;
952 
953 	/*
954 	 * If a filter function exists, call it to determine whether
955 	 * this snapshot will be sent.
956 	 */
957 	if (sdd->filter_cb != NULL &&
958 	    sdd->filter_cb(zhp, sdd->filter_cb_arg) == B_FALSE) {
959 		/*
960 		 * This snapshot is filtered out.  Don't send it, and don't
961 		 * set prevsnap, so it will be as if this snapshot didn't
962 		 * exist, and the next accepted snapshot will be sent as
963 		 * an incremental from the last accepted one, or as the
964 		 * first (and full) snapshot in the case of a replication,
965 		 * non-incremental send.
966 		 */
967 		zfs_close(zhp);
968 		return (0);
969 	}
970 
971 	/* send it */
972 	if (sdd->verbose) {
973 		(void) fprintf(stderr, "sending from @%s to %s\n",
974 		    sdd->prevsnap, zhp->zfs_name);
975 	}
976 
977 	err = dump_ioctl(zhp, sdd->prevsnap,
978 	    sdd->prevsnap[0] == '\0' && (sdd->fromorigin || sdd->replicate),
979 	    sdd->outfd, B_TRUE, &got_enoent);
980 
981 	if (got_enoent)
982 		err = 0;
983 	else
984 		(void) strcpy(sdd->prevsnap, thissnap);
985 	zfs_close(zhp);
986 	return (err);
987 }
988 
989 static int
990 dump_filesystem(zfs_handle_t *zhp, void *arg)
991 {
992 	int rv = 0;
993 	send_dump_data_t *sdd = arg;
994 	boolean_t missingfrom = B_FALSE;
995 	zfs_cmd_t zc = { 0 };
996 
997 	(void) snprintf(zc.zc_name, sizeof (zc.zc_name), "%s@%s",
998 	    zhp->zfs_name, sdd->tosnap);
999 	if (ioctl(zhp->zfs_hdl->libzfs_fd, ZFS_IOC_OBJSET_STATS, &zc) != 0) {
1000 		(void) fprintf(stderr, "WARNING: "
1001 		    "could not send %s@%s: does not exist\n",
1002 		    zhp->zfs_name, sdd->tosnap);
1003 		sdd->err = B_TRUE;
1004 		return (0);
1005 	}
1006 
1007 	if (sdd->replicate && sdd->fromsnap) {
1008 		/*
1009 		 * If this fs does not have fromsnap, and we're doing
1010 		 * recursive, we need to send a full stream from the
1011 		 * beginning (or an incremental from the origin if this
1012 		 * is a clone).  If we're doing non-recursive, then let
1013 		 * them get the error.
1014 		 */
1015 		(void) snprintf(zc.zc_name, sizeof (zc.zc_name), "%s@%s",
1016 		    zhp->zfs_name, sdd->fromsnap);
1017 		if (ioctl(zhp->zfs_hdl->libzfs_fd,
1018 		    ZFS_IOC_OBJSET_STATS, &zc) != 0) {
1019 			missingfrom = B_TRUE;
1020 		}
1021 	}
1022 
1023 	if (sdd->doall) {
1024 		sdd->seenfrom = sdd->seento = sdd->prevsnap[0] = 0;
1025 		if (sdd->fromsnap == NULL || missingfrom)
1026 			sdd->seenfrom = B_TRUE;
1027 
1028 		rv = zfs_iter_snapshots_sorted(zhp, dump_snapshot, arg);
1029 		if (!sdd->seenfrom) {
1030 			(void) fprintf(stderr,
1031 			    "WARNING: could not send %s@%s:\n"
1032 			    "incremental source (%s@%s) does not exist\n",
1033 			    zhp->zfs_name, sdd->tosnap,
1034 			    zhp->zfs_name, sdd->fromsnap);
1035 			sdd->err = B_TRUE;
1036 		} else if (!sdd->seento) {
1037 			if (sdd->fromsnap) {
1038 				(void) fprintf(stderr,
1039 				    "WARNING: could not send %s@%s:\n"
1040 				    "incremental source (%s@%s) "
1041 				    "is not earlier than it\n",
1042 				    zhp->zfs_name, sdd->tosnap,
1043 				    zhp->zfs_name, sdd->fromsnap);
1044 			} else {
1045 				(void) fprintf(stderr, "WARNING: "
1046 				    "could not send %s@%s: does not exist\n",
1047 				    zhp->zfs_name, sdd->tosnap);
1048 			}
1049 			sdd->err = B_TRUE;
1050 		}
1051 	} else {
1052 		zfs_handle_t *snapzhp;
1053 		char snapname[ZFS_MAXNAMELEN];
1054 
1055 		(void) snprintf(snapname, sizeof (snapname), "%s@%s",
1056 		    zfs_get_name(zhp), sdd->tosnap);
1057 		snapzhp = zfs_open(zhp->zfs_hdl, snapname, ZFS_TYPE_SNAPSHOT);
1058 		if (snapzhp == NULL) {
1059 			rv = -1;
1060 		} else {
1061 			if (sdd->filter_cb == NULL ||
1062 			    sdd->filter_cb(snapzhp, sdd->filter_cb_arg) ==
1063 			    B_TRUE) {
1064 				boolean_t got_enoent;
1065 
1066 				rv = dump_ioctl(snapzhp,
1067 				    missingfrom ? NULL : sdd->fromsnap,
1068 				    sdd->fromorigin || missingfrom,
1069 				    sdd->outfd, B_FALSE, &got_enoent);
1070 			}
1071 			sdd->seento = B_TRUE;
1072 			zfs_close(snapzhp);
1073 		}
1074 	}
1075 
1076 	return (rv);
1077 }
1078 
1079 static int
1080 dump_filesystems(zfs_handle_t *rzhp, void *arg)
1081 {
1082 	send_dump_data_t *sdd = arg;
1083 	nvpair_t *fspair;
1084 	boolean_t needagain, progress;
1085 
1086 	if (!sdd->replicate)
1087 		return (dump_filesystem(rzhp, sdd));
1088 
1089 again:
1090 	needagain = progress = B_FALSE;
1091 	for (fspair = nvlist_next_nvpair(sdd->fss, NULL); fspair;
1092 	    fspair = nvlist_next_nvpair(sdd->fss, fspair)) {
1093 		nvlist_t *fslist;
1094 		char *fsname;
1095 		zfs_handle_t *zhp;
1096 		int err;
1097 		uint64_t origin_guid = 0;
1098 		nvlist_t *origin_nv;
1099 
1100 		VERIFY(nvpair_value_nvlist(fspair, &fslist) == 0);
1101 		if (nvlist_lookup_boolean(fslist, "sent") == 0)
1102 			continue;
1103 
1104 		VERIFY(nvlist_lookup_string(fslist, "name", &fsname) == 0);
1105 		(void) nvlist_lookup_uint64(fslist, "origin", &origin_guid);
1106 
1107 		origin_nv = fsavl_find(sdd->fsavl, origin_guid, NULL);
1108 		if (origin_nv &&
1109 		    nvlist_lookup_boolean(origin_nv, "sent") == ENOENT) {
1110 			/*
1111 			 * origin has not been sent yet;
1112 			 * skip this clone.
1113 			 */
1114 			needagain = B_TRUE;
1115 			continue;
1116 		}
1117 
1118 		zhp = zfs_open(rzhp->zfs_hdl, fsname, ZFS_TYPE_DATASET);
1119 		if (zhp == NULL)
1120 			return (-1);
1121 		err = dump_filesystem(zhp, sdd);
1122 		VERIFY(nvlist_add_boolean(fslist, "sent") == 0);
1123 		progress = B_TRUE;
1124 		zfs_close(zhp);
1125 		if (err)
1126 			return (err);
1127 	}
1128 	if (needagain) {
1129 		assert(progress);
1130 		goto again;
1131 	}
1132 	return (0);
1133 }
1134 
1135 /*
1136  * Generate a send stream for the dataset identified by the argument zhp.
1137  *
1138  * The content of the send stream is the snapshot identified by
1139  * 'tosnap'.  Incremental streams are requested in two ways:
1140  *     - from the snapshot identified by "fromsnap" (if non-null) or
1141  *     - from the origin of the dataset identified by zhp, which must
1142  *	 be a clone.  In this case, "fromsnap" is null and "fromorigin"
1143  *	 is TRUE.
1144  *
1145  * The send stream is recursive (i.e. dumps a hierarchy of snapshots) and
1146  * uses a special header (with a hdrtype field of DMU_COMPOUNDSTREAM)
1147  * if "replicate" is set.  If "doall" is set, dump all the intermediate
1148  * snapshots. The DMU_COMPOUNDSTREAM header is used in the "doall"
1149  * case too. If "props" is set, send properties.
1150  */
1151 int
1152 zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
1153     sendflags_t flags, int outfd, snapfilter_cb_t filter_func,
1154     void *cb_arg)
1155 {
1156 	char errbuf[1024];
1157 	send_dump_data_t sdd = { 0 };
1158 	int err;
1159 	nvlist_t *fss = NULL;
1160 	avl_tree_t *fsavl = NULL;
1161 	char holdtag[128];
1162 	static uint64_t holdseq;
1163 	int spa_version;
1164 	boolean_t holdsnaps = B_FALSE;
1165 	pthread_t tid;
1166 	int pipefd[2];
1167 	dedup_arg_t dda = { 0 };
1168 	int featureflags = 0;
1169 
1170 	if (zhp->zfs_type == ZFS_TYPE_FILESYSTEM) {
1171 		uint64_t version;
1172 		version = zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
1173 		if (version >= ZPL_VERSION_SA) {
1174 			featureflags |= DMU_BACKUP_FEATURE_SA_SPILL;
1175 		}
1176 	}
1177 
1178 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
1179 	    "cannot send '%s'"), zhp->zfs_name);
1180 
1181 	if (fromsnap && fromsnap[0] == '\0') {
1182 		zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
1183 		    "zero-length incremental source"));
1184 		return (zfs_error(zhp->zfs_hdl, EZFS_NOENT, errbuf));
1185 	}
1186 
1187 	if (zfs_spa_version(zhp, &spa_version) == 0 &&
1188 	    spa_version >= SPA_VERSION_USERREFS)
1189 		holdsnaps = B_TRUE;
1190 
1191 	if (flags.dedup) {
1192 		featureflags |= (DMU_BACKUP_FEATURE_DEDUP |
1193 		    DMU_BACKUP_FEATURE_DEDUPPROPS);
1194 		if (err = pipe(pipefd)) {
1195 			zfs_error_aux(zhp->zfs_hdl, strerror(errno));
1196 			return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED,
1197 			    errbuf));
1198 		}
1199 		dda.outputfd = outfd;
1200 		dda.inputfd = pipefd[1];
1201 		dda.dedup_hdl = zhp->zfs_hdl;
1202 		if (err = pthread_create(&tid, NULL, cksummer, &dda)) {
1203 			(void) close(pipefd[0]);
1204 			(void) close(pipefd[1]);
1205 			zfs_error_aux(zhp->zfs_hdl, strerror(errno));
1206 			return (zfs_error(zhp->zfs_hdl,
1207 			    EZFS_THREADCREATEFAILED, errbuf));
1208 		}
1209 	}
1210 
1211 	if (flags.replicate || flags.doall || flags.props) {
1212 		dmu_replay_record_t drr = { 0 };
1213 		char *packbuf = NULL;
1214 		size_t buflen = 0;
1215 		zio_cksum_t zc = { 0 };
1216 
1217 		if (holdsnaps) {
1218 			(void) snprintf(holdtag, sizeof (holdtag),
1219 			    ".send-%d-%llu", getpid(), (u_longlong_t)holdseq);
1220 			++holdseq;
1221 			err = zfs_hold_range(zhp, fromsnap, tosnap,
1222 			    holdtag, flags.replicate, B_TRUE, filter_func,
1223 			    cb_arg);
1224 			if (err)
1225 				goto err_out;
1226 		}
1227 
1228 		if (flags.replicate || flags.props) {
1229 			nvlist_t *hdrnv;
1230 
1231 			VERIFY(0 == nvlist_alloc(&hdrnv, NV_UNIQUE_NAME, 0));
1232 			if (fromsnap) {
1233 				VERIFY(0 == nvlist_add_string(hdrnv,
1234 				    "fromsnap", fromsnap));
1235 			}
1236 			VERIFY(0 == nvlist_add_string(hdrnv, "tosnap", tosnap));
1237 			if (!flags.replicate) {
1238 				VERIFY(0 == nvlist_add_boolean(hdrnv,
1239 				    "not_recursive"));
1240 			}
1241 
1242 			err = gather_nvlist(zhp->zfs_hdl, zhp->zfs_name,
1243 			    fromsnap, tosnap, flags.replicate, &fss, &fsavl);
1244 			if (err) {
1245 				if (holdsnaps) {
1246 					(void) zfs_release_range(zhp, fromsnap,
1247 					    tosnap, holdtag, flags.replicate);
1248 				}
1249 				goto err_out;
1250 			}
1251 			VERIFY(0 == nvlist_add_nvlist(hdrnv, "fss", fss));
1252 			err = nvlist_pack(hdrnv, &packbuf, &buflen,
1253 			    NV_ENCODE_XDR, 0);
1254 			nvlist_free(hdrnv);
1255 			if (err) {
1256 				fsavl_destroy(fsavl);
1257 				nvlist_free(fss);
1258 				if (holdsnaps) {
1259 					(void) zfs_release_range(zhp, fromsnap,
1260 					    tosnap, holdtag, flags.replicate);
1261 				}
1262 				goto stderr_out;
1263 			}
1264 		}
1265 
1266 		/* write first begin record */
1267 		drr.drr_type = DRR_BEGIN;
1268 		drr.drr_u.drr_begin.drr_magic = DMU_BACKUP_MAGIC;
1269 		DMU_SET_STREAM_HDRTYPE(drr.drr_u.drr_begin.drr_versioninfo,
1270 		    DMU_COMPOUNDSTREAM);
1271 		DMU_SET_FEATUREFLAGS(drr.drr_u.drr_begin.drr_versioninfo,
1272 		    featureflags);
1273 		(void) snprintf(drr.drr_u.drr_begin.drr_toname,
1274 		    sizeof (drr.drr_u.drr_begin.drr_toname),
1275 		    "%s@%s", zhp->zfs_name, tosnap);
1276 		drr.drr_payloadlen = buflen;
1277 		err = cksum_and_write(&drr, sizeof (drr), &zc, outfd);
1278 
1279 		/* write header nvlist */
1280 		if (err != -1 && packbuf != NULL) {
1281 			err = cksum_and_write(packbuf, buflen, &zc, outfd);
1282 		}
1283 		free(packbuf);
1284 		if (err == -1) {
1285 			fsavl_destroy(fsavl);
1286 			nvlist_free(fss);
1287 			if (holdsnaps) {
1288 				(void) zfs_release_range(zhp, fromsnap, tosnap,
1289 				    holdtag, flags.replicate);
1290 			}
1291 			err = errno;
1292 			goto stderr_out;
1293 		}
1294 
1295 		/* write end record */
1296 		if (err != -1) {
1297 			bzero(&drr, sizeof (drr));
1298 			drr.drr_type = DRR_END;
1299 			drr.drr_u.drr_end.drr_checksum = zc;
1300 			err = write(outfd, &drr, sizeof (drr));
1301 			if (err == -1) {
1302 				fsavl_destroy(fsavl);
1303 				nvlist_free(fss);
1304 				err = errno;
1305 				if (holdsnaps) {
1306 					(void) zfs_release_range(zhp, fromsnap,
1307 					    tosnap, holdtag, flags.replicate);
1308 				}
1309 				goto stderr_out;
1310 			}
1311 		}
1312 	}
1313 
1314 	/* dump each stream */
1315 	sdd.fromsnap = fromsnap;
1316 	sdd.tosnap = tosnap;
1317 	if (flags.dedup)
1318 		sdd.outfd = pipefd[0];
1319 	else
1320 		sdd.outfd = outfd;
1321 	sdd.replicate = flags.replicate;
1322 	sdd.doall = flags.doall;
1323 	sdd.fromorigin = flags.fromorigin;
1324 	sdd.fss = fss;
1325 	sdd.fsavl = fsavl;
1326 	sdd.verbose = flags.verbose;
1327 	sdd.filter_cb = filter_func;
1328 	sdd.filter_cb_arg = cb_arg;
1329 	err = dump_filesystems(zhp, &sdd);
1330 	fsavl_destroy(fsavl);
1331 	nvlist_free(fss);
1332 
1333 	if (flags.dedup) {
1334 		(void) close(pipefd[0]);
1335 		(void) pthread_join(tid, NULL);
1336 	}
1337 
1338 	if (flags.replicate || flags.doall || flags.props) {
1339 		/*
1340 		 * write final end record.  NB: want to do this even if
1341 		 * there was some error, because it might not be totally
1342 		 * failed.
1343 		 */
1344 		dmu_replay_record_t drr = { 0 };
1345 		drr.drr_type = DRR_END;
1346 		if (holdsnaps) {
1347 			(void) zfs_release_range(zhp, fromsnap, tosnap,
1348 			    holdtag, flags.replicate);
1349 		}
1350 		if (write(outfd, &drr, sizeof (drr)) == -1) {
1351 			return (zfs_standard_error(zhp->zfs_hdl,
1352 			    errno, errbuf));
1353 		}
1354 	}
1355 
1356 	return (err || sdd.err);
1357 
1358 stderr_out:
1359 	err = zfs_standard_error(zhp->zfs_hdl, err, errbuf);
1360 err_out:
1361 	if (flags.dedup) {
1362 		(void) pthread_cancel(tid);
1363 		(void) pthread_join(tid, NULL);
1364 		(void) close(pipefd[0]);
1365 	}
1366 	return (err);
1367 }
1368 
1369 /*
1370  * Routines specific to "zfs recv"
1371  */
1372 
1373 static int
1374 recv_read(libzfs_handle_t *hdl, int fd, void *buf, int ilen,
1375     boolean_t byteswap, zio_cksum_t *zc)
1376 {
1377 	char *cp = buf;
1378 	int rv;
1379 	int len = ilen;
1380 
1381 	do {
1382 		rv = read(fd, cp, len);
1383 		cp += rv;
1384 		len -= rv;
1385 	} while (rv > 0);
1386 
1387 	if (rv < 0 || len != 0) {
1388 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1389 		    "failed to read from stream"));
1390 		return (zfs_error(hdl, EZFS_BADSTREAM, dgettext(TEXT_DOMAIN,
1391 		    "cannot receive")));
1392 	}
1393 
1394 	if (zc) {
1395 		if (byteswap)
1396 			fletcher_4_incremental_byteswap(buf, ilen, zc);
1397 		else
1398 			fletcher_4_incremental_native(buf, ilen, zc);
1399 	}
1400 	return (0);
1401 }
1402 
1403 static int
1404 recv_read_nvlist(libzfs_handle_t *hdl, int fd, int len, nvlist_t **nvp,
1405     boolean_t byteswap, zio_cksum_t *zc)
1406 {
1407 	char *buf;
1408 	int err;
1409 
1410 	buf = zfs_alloc(hdl, len);
1411 	if (buf == NULL)
1412 		return (ENOMEM);
1413 
1414 	err = recv_read(hdl, fd, buf, len, byteswap, zc);
1415 	if (err != 0) {
1416 		free(buf);
1417 		return (err);
1418 	}
1419 
1420 	err = nvlist_unpack(buf, len, nvp, 0);
1421 	free(buf);
1422 	if (err != 0) {
1423 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
1424 		    "stream (malformed nvlist)"));
1425 		return (EINVAL);
1426 	}
1427 	return (0);
1428 }
1429 
1430 static int
1431 recv_rename(libzfs_handle_t *hdl, const char *name, const char *tryname,
1432     int baselen, char *newname, recvflags_t flags)
1433 {
1434 	static int seq;
1435 	zfs_cmd_t zc = { 0 };
1436 	int err;
1437 	prop_changelist_t *clp;
1438 	zfs_handle_t *zhp;
1439 
1440 	zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
1441 	if (zhp == NULL)
1442 		return (-1);
1443 	clp = changelist_gather(zhp, ZFS_PROP_NAME, 0,
1444 	    flags.force ? MS_FORCE : 0);
1445 	zfs_close(zhp);
1446 	if (clp == NULL)
1447 		return (-1);
1448 	err = changelist_prefix(clp);
1449 	if (err)
1450 		return (err);
1451 
1452 	zc.zc_objset_type = DMU_OST_ZFS;
1453 	(void) strlcpy(zc.zc_name, name, sizeof (zc.zc_name));
1454 
1455 	if (tryname) {
1456 		(void) strcpy(newname, tryname);
1457 
1458 		(void) strlcpy(zc.zc_value, tryname, sizeof (zc.zc_value));
1459 
1460 		if (flags.verbose) {
1461 			(void) printf("attempting rename %s to %s\n",
1462 			    zc.zc_name, zc.zc_value);
1463 		}
1464 		err = ioctl(hdl->libzfs_fd, ZFS_IOC_RENAME, &zc);
1465 		if (err == 0)
1466 			changelist_rename(clp, name, tryname);
1467 	} else {
1468 		err = ENOENT;
1469 	}
1470 
1471 	if (err != 0 && strncmp(name+baselen, "recv-", 5) != 0) {
1472 		seq++;
1473 
1474 		(void) strncpy(newname, name, baselen);
1475 		(void) snprintf(newname+baselen, ZFS_MAXNAMELEN-baselen,
1476 		    "recv-%u-%u", getpid(), seq);
1477 		(void) strlcpy(zc.zc_value, newname, sizeof (zc.zc_value));
1478 
1479 		if (flags.verbose) {
1480 			(void) printf("failed - trying rename %s to %s\n",
1481 			    zc.zc_name, zc.zc_value);
1482 		}
1483 		err = ioctl(hdl->libzfs_fd, ZFS_IOC_RENAME, &zc);
1484 		if (err == 0)
1485 			changelist_rename(clp, name, newname);
1486 		if (err && flags.verbose) {
1487 			(void) printf("failed (%u) - "
1488 			    "will try again on next pass\n", errno);
1489 		}
1490 		err = EAGAIN;
1491 	} else if (flags.verbose) {
1492 		if (err == 0)
1493 			(void) printf("success\n");
1494 		else
1495 			(void) printf("failed (%u)\n", errno);
1496 	}
1497 
1498 	(void) changelist_postfix(clp);
1499 	changelist_free(clp);
1500 
1501 	return (err);
1502 }
1503 
1504 static int
1505 recv_destroy(libzfs_handle_t *hdl, const char *name, int baselen,
1506     char *newname, recvflags_t flags)
1507 {
1508 	zfs_cmd_t zc = { 0 };
1509 	int err = 0;
1510 	prop_changelist_t *clp;
1511 	zfs_handle_t *zhp;
1512 	boolean_t defer = B_FALSE;
1513 	int spa_version;
1514 
1515 	zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
1516 	if (zhp == NULL)
1517 		return (-1);
1518 	clp = changelist_gather(zhp, ZFS_PROP_NAME, 0,
1519 	    flags.force ? MS_FORCE : 0);
1520 	if (zfs_get_type(zhp) == ZFS_TYPE_SNAPSHOT &&
1521 	    zfs_spa_version(zhp, &spa_version) == 0 &&
1522 	    spa_version >= SPA_VERSION_USERREFS)
1523 		defer = B_TRUE;
1524 	zfs_close(zhp);
1525 	if (clp == NULL)
1526 		return (-1);
1527 	err = changelist_prefix(clp);
1528 	if (err)
1529 		return (err);
1530 
1531 	zc.zc_objset_type = DMU_OST_ZFS;
1532 	zc.zc_defer_destroy = defer;
1533 	(void) strlcpy(zc.zc_name, name, sizeof (zc.zc_name));
1534 
1535 	if (flags.verbose)
1536 		(void) printf("attempting destroy %s\n", zc.zc_name);
1537 	err = ioctl(hdl->libzfs_fd, ZFS_IOC_DESTROY, &zc);
1538 	if (err == 0) {
1539 		if (flags.verbose)
1540 			(void) printf("success\n");
1541 		changelist_remove(clp, zc.zc_name);
1542 	}
1543 
1544 	(void) changelist_postfix(clp);
1545 	changelist_free(clp);
1546 
1547 	/*
1548 	 * Deferred destroy might destroy the snapshot or only mark it to be
1549 	 * destroyed later, and it returns success in either case.
1550 	 */
1551 	if (err != 0 || (defer && zfs_dataset_exists(hdl, name,
1552 	    ZFS_TYPE_SNAPSHOT))) {
1553 		err = recv_rename(hdl, name, NULL, baselen, newname, flags);
1554 	}
1555 
1556 	return (err);
1557 }
1558 
1559 typedef struct guid_to_name_data {
1560 	uint64_t guid;
1561 	char *name;
1562 } guid_to_name_data_t;
1563 
1564 static int
1565 guid_to_name_cb(zfs_handle_t *zhp, void *arg)
1566 {
1567 	guid_to_name_data_t *gtnd = arg;
1568 	int err;
1569 
1570 	if (zhp->zfs_dmustats.dds_guid == gtnd->guid) {
1571 		(void) strcpy(gtnd->name, zhp->zfs_name);
1572 		zfs_close(zhp);
1573 		return (EEXIST);
1574 	}
1575 	err = zfs_iter_children(zhp, guid_to_name_cb, gtnd);
1576 	zfs_close(zhp);
1577 	return (err);
1578 }
1579 
1580 static int
1581 guid_to_name(libzfs_handle_t *hdl, const char *parent, uint64_t guid,
1582     char *name)
1583 {
1584 	/* exhaustive search all local snapshots */
1585 	guid_to_name_data_t gtnd;
1586 	int err = 0;
1587 	zfs_handle_t *zhp;
1588 	char *cp;
1589 
1590 	gtnd.guid = guid;
1591 	gtnd.name = name;
1592 
1593 	if (strchr(parent, '@') == NULL) {
1594 		zhp = make_dataset_handle(hdl, parent);
1595 		if (zhp != NULL) {
1596 			err = zfs_iter_children(zhp, guid_to_name_cb, &gtnd);
1597 			zfs_close(zhp);
1598 			if (err == EEXIST)
1599 				return (0);
1600 		}
1601 	}
1602 
1603 	cp = strchr(parent, '/');
1604 	if (cp)
1605 		*cp = '\0';
1606 	zhp = make_dataset_handle(hdl, parent);
1607 	if (cp)
1608 		*cp = '/';
1609 
1610 	if (zhp) {
1611 		err = zfs_iter_children(zhp, guid_to_name_cb, &gtnd);
1612 		zfs_close(zhp);
1613 	}
1614 
1615 	return (err == EEXIST ? 0 : ENOENT);
1616 
1617 }
1618 
1619 /*
1620  * Return true if dataset guid1 is created before guid2.
1621  */
1622 static int
1623 created_before(libzfs_handle_t *hdl, avl_tree_t *avl,
1624     uint64_t guid1, uint64_t guid2)
1625 {
1626 	nvlist_t *nvfs;
1627 	char *fsname, *snapname;
1628 	char buf[ZFS_MAXNAMELEN];
1629 	int rv;
1630 	zfs_node_t zn1, zn2;
1631 
1632 	if (guid2 == 0)
1633 		return (0);
1634 	if (guid1 == 0)
1635 		return (1);
1636 
1637 	nvfs = fsavl_find(avl, guid1, &snapname);
1638 	VERIFY(0 == nvlist_lookup_string(nvfs, "name", &fsname));
1639 	(void) snprintf(buf, sizeof (buf), "%s@%s", fsname, snapname);
1640 	zn1.zn_handle = zfs_open(hdl, buf, ZFS_TYPE_SNAPSHOT);
1641 	if (zn1.zn_handle == NULL)
1642 		return (-1);
1643 
1644 	nvfs = fsavl_find(avl, guid2, &snapname);
1645 	VERIFY(0 == nvlist_lookup_string(nvfs, "name", &fsname));
1646 	(void) snprintf(buf, sizeof (buf), "%s@%s", fsname, snapname);
1647 	zn2.zn_handle = zfs_open(hdl, buf, ZFS_TYPE_SNAPSHOT);
1648 	if (zn2.zn_handle == NULL) {
1649 		zfs_close(zn2.zn_handle);
1650 		return (-1);
1651 	}
1652 
1653 	rv = (zfs_snapshot_compare(&zn1, &zn2) == -1);
1654 
1655 	zfs_close(zn1.zn_handle);
1656 	zfs_close(zn2.zn_handle);
1657 
1658 	return (rv);
1659 }
1660 
1661 static int
1662 recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs,
1663     recvflags_t flags, nvlist_t *stream_nv, avl_tree_t *stream_avl,
1664     nvlist_t *renamed)
1665 {
1666 	nvlist_t *local_nv;
1667 	avl_tree_t *local_avl;
1668 	nvpair_t *fselem, *nextfselem;
1669 	char *fromsnap;
1670 	char newname[ZFS_MAXNAMELEN];
1671 	int error;
1672 	boolean_t needagain, progress, recursive;
1673 	char *s1, *s2;
1674 
1675 	VERIFY(0 == nvlist_lookup_string(stream_nv, "fromsnap", &fromsnap));
1676 
1677 	recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
1678 	    ENOENT);
1679 
1680 	if (flags.dryrun)
1681 		return (0);
1682 
1683 again:
1684 	needagain = progress = B_FALSE;
1685 
1686 	if ((error = gather_nvlist(hdl, tofs, fromsnap, NULL,
1687 	    recursive, &local_nv, &local_avl)) != 0)
1688 		return (error);
1689 
1690 	/*
1691 	 * Process deletes and renames
1692 	 */
1693 	for (fselem = nvlist_next_nvpair(local_nv, NULL);
1694 	    fselem; fselem = nextfselem) {
1695 		nvlist_t *nvfs, *snaps;
1696 		nvlist_t *stream_nvfs = NULL;
1697 		nvpair_t *snapelem, *nextsnapelem;
1698 		uint64_t fromguid = 0;
1699 		uint64_t originguid = 0;
1700 		uint64_t stream_originguid = 0;
1701 		uint64_t parent_fromsnap_guid, stream_parent_fromsnap_guid;
1702 		char *fsname, *stream_fsname;
1703 
1704 		nextfselem = nvlist_next_nvpair(local_nv, fselem);
1705 
1706 		VERIFY(0 == nvpair_value_nvlist(fselem, &nvfs));
1707 		VERIFY(0 == nvlist_lookup_nvlist(nvfs, "snaps", &snaps));
1708 		VERIFY(0 == nvlist_lookup_string(nvfs, "name", &fsname));
1709 		VERIFY(0 == nvlist_lookup_uint64(nvfs, "parentfromsnap",
1710 		    &parent_fromsnap_guid));
1711 		(void) nvlist_lookup_uint64(nvfs, "origin", &originguid);
1712 
1713 		/*
1714 		 * First find the stream's fs, so we can check for
1715 		 * a different origin (due to "zfs promote")
1716 		 */
1717 		for (snapelem = nvlist_next_nvpair(snaps, NULL);
1718 		    snapelem; snapelem = nvlist_next_nvpair(snaps, snapelem)) {
1719 			uint64_t thisguid;
1720 
1721 			VERIFY(0 == nvpair_value_uint64(snapelem, &thisguid));
1722 			stream_nvfs = fsavl_find(stream_avl, thisguid, NULL);
1723 
1724 			if (stream_nvfs != NULL)
1725 				break;
1726 		}
1727 
1728 		/* check for promote */
1729 		(void) nvlist_lookup_uint64(stream_nvfs, "origin",
1730 		    &stream_originguid);
1731 		if (stream_nvfs && originguid != stream_originguid) {
1732 			switch (created_before(hdl, local_avl,
1733 			    stream_originguid, originguid)) {
1734 			case 1: {
1735 				/* promote it! */
1736 				zfs_cmd_t zc = { 0 };
1737 				nvlist_t *origin_nvfs;
1738 				char *origin_fsname;
1739 
1740 				if (flags.verbose)
1741 					(void) printf("promoting %s\n", fsname);
1742 
1743 				origin_nvfs = fsavl_find(local_avl, originguid,
1744 				    NULL);
1745 				VERIFY(0 == nvlist_lookup_string(origin_nvfs,
1746 				    "name", &origin_fsname));
1747 				(void) strlcpy(zc.zc_value, origin_fsname,
1748 				    sizeof (zc.zc_value));
1749 				(void) strlcpy(zc.zc_name, fsname,
1750 				    sizeof (zc.zc_name));
1751 				error = zfs_ioctl(hdl, ZFS_IOC_PROMOTE, &zc);
1752 				if (error == 0)
1753 					progress = B_TRUE;
1754 				break;
1755 			}
1756 			default:
1757 				break;
1758 			case -1:
1759 				fsavl_destroy(local_avl);
1760 				nvlist_free(local_nv);
1761 				return (-1);
1762 			}
1763 			/*
1764 			 * We had/have the wrong origin, therefore our
1765 			 * list of snapshots is wrong.  Need to handle
1766 			 * them on the next pass.
1767 			 */
1768 			needagain = B_TRUE;
1769 			continue;
1770 		}
1771 
1772 		for (snapelem = nvlist_next_nvpair(snaps, NULL);
1773 		    snapelem; snapelem = nextsnapelem) {
1774 			uint64_t thisguid;
1775 			char *stream_snapname;
1776 			nvlist_t *found, *props;
1777 
1778 			nextsnapelem = nvlist_next_nvpair(snaps, snapelem);
1779 
1780 			VERIFY(0 == nvpair_value_uint64(snapelem, &thisguid));
1781 			found = fsavl_find(stream_avl, thisguid,
1782 			    &stream_snapname);
1783 
1784 			/* check for delete */
1785 			if (found == NULL) {
1786 				char name[ZFS_MAXNAMELEN];
1787 
1788 				if (!flags.force)
1789 					continue;
1790 
1791 				(void) snprintf(name, sizeof (name), "%s@%s",
1792 				    fsname, nvpair_name(snapelem));
1793 
1794 				error = recv_destroy(hdl, name,
1795 				    strlen(fsname)+1, newname, flags);
1796 				if (error)
1797 					needagain = B_TRUE;
1798 				else
1799 					progress = B_TRUE;
1800 				continue;
1801 			}
1802 
1803 			stream_nvfs = found;
1804 
1805 			if (0 == nvlist_lookup_nvlist(stream_nvfs, "snapprops",
1806 			    &props) && 0 == nvlist_lookup_nvlist(props,
1807 			    stream_snapname, &props)) {
1808 				zfs_cmd_t zc = { 0 };
1809 
1810 				zc.zc_cookie = B_TRUE; /* received */
1811 				(void) snprintf(zc.zc_name, sizeof (zc.zc_name),
1812 				    "%s@%s", fsname, nvpair_name(snapelem));
1813 				if (zcmd_write_src_nvlist(hdl, &zc,
1814 				    props) == 0) {
1815 					(void) zfs_ioctl(hdl,
1816 					    ZFS_IOC_SET_PROP, &zc);
1817 					zcmd_free_nvlists(&zc);
1818 				}
1819 			}
1820 
1821 			/* check for different snapname */
1822 			if (strcmp(nvpair_name(snapelem),
1823 			    stream_snapname) != 0) {
1824 				char name[ZFS_MAXNAMELEN];
1825 				char tryname[ZFS_MAXNAMELEN];
1826 
1827 				(void) snprintf(name, sizeof (name), "%s@%s",
1828 				    fsname, nvpair_name(snapelem));
1829 				(void) snprintf(tryname, sizeof (name), "%s@%s",
1830 				    fsname, stream_snapname);
1831 
1832 				error = recv_rename(hdl, name, tryname,
1833 				    strlen(fsname)+1, newname, flags);
1834 				if (error)
1835 					needagain = B_TRUE;
1836 				else
1837 					progress = B_TRUE;
1838 			}
1839 
1840 			if (strcmp(stream_snapname, fromsnap) == 0)
1841 				fromguid = thisguid;
1842 		}
1843 
1844 		/* check for delete */
1845 		if (stream_nvfs == NULL) {
1846 			if (!flags.force)
1847 				continue;
1848 
1849 			error = recv_destroy(hdl, fsname, strlen(tofs)+1,
1850 			    newname, flags);
1851 			if (error)
1852 				needagain = B_TRUE;
1853 			else
1854 				progress = B_TRUE;
1855 			continue;
1856 		}
1857 
1858 		if (fromguid == 0) {
1859 			if (flags.verbose) {
1860 				(void) printf("local fs %s does not have "
1861 				    "fromsnap (%s in stream); must have "
1862 				    "been deleted locally; ignoring\n",
1863 				    fsname, fromsnap);
1864 			}
1865 			continue;
1866 		}
1867 
1868 		VERIFY(0 == nvlist_lookup_string(stream_nvfs,
1869 		    "name", &stream_fsname));
1870 		VERIFY(0 == nvlist_lookup_uint64(stream_nvfs,
1871 		    "parentfromsnap", &stream_parent_fromsnap_guid));
1872 
1873 		s1 = strrchr(fsname, '/');
1874 		s2 = strrchr(stream_fsname, '/');
1875 
1876 		/*
1877 		 * Check for rename. If the exact receive path is specified, it
1878 		 * does not count as a rename, but we still need to check the
1879 		 * datasets beneath it.
1880 		 */
1881 		if ((stream_parent_fromsnap_guid != 0 &&
1882 		    parent_fromsnap_guid != 0 &&
1883 		    stream_parent_fromsnap_guid != parent_fromsnap_guid) ||
1884 		    ((flags.isprefix || strcmp(tofs, fsname) != 0) &&
1885 		    (s1 != NULL) && (s2 != NULL) && strcmp(s1, s2) != 0)) {
1886 			nvlist_t *parent;
1887 			char tryname[ZFS_MAXNAMELEN];
1888 
1889 			parent = fsavl_find(local_avl,
1890 			    stream_parent_fromsnap_guid, NULL);
1891 			/*
1892 			 * NB: parent might not be found if we used the
1893 			 * tosnap for stream_parent_fromsnap_guid,
1894 			 * because the parent is a newly-created fs;
1895 			 * we'll be able to rename it after we recv the
1896 			 * new fs.
1897 			 */
1898 			if (parent != NULL) {
1899 				char *pname;
1900 
1901 				VERIFY(0 == nvlist_lookup_string(parent, "name",
1902 				    &pname));
1903 				(void) snprintf(tryname, sizeof (tryname),
1904 				    "%s%s", pname, strrchr(stream_fsname, '/'));
1905 			} else {
1906 				tryname[0] = '\0';
1907 				if (flags.verbose) {
1908 					(void) printf("local fs %s new parent "
1909 					    "not found\n", fsname);
1910 				}
1911 			}
1912 
1913 			newname[0] = '\0';
1914 
1915 			error = recv_rename(hdl, fsname, tryname,
1916 			    strlen(tofs)+1, newname, flags);
1917 
1918 			if (renamed != NULL && newname[0] != '\0') {
1919 				VERIFY(0 == nvlist_add_boolean(renamed,
1920 				    newname));
1921 			}
1922 
1923 			if (error)
1924 				needagain = B_TRUE;
1925 			else
1926 				progress = B_TRUE;
1927 		}
1928 	}
1929 
1930 	fsavl_destroy(local_avl);
1931 	nvlist_free(local_nv);
1932 
1933 	if (needagain && progress) {
1934 		/* do another pass to fix up temporary names */
1935 		if (flags.verbose)
1936 			(void) printf("another pass:\n");
1937 		goto again;
1938 	}
1939 
1940 	return (needagain);
1941 }
1942 
1943 static int
1944 zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
1945     recvflags_t flags, dmu_replay_record_t *drr, zio_cksum_t *zc,
1946     char **top_zfs)
1947 {
1948 	nvlist_t *stream_nv = NULL;
1949 	avl_tree_t *stream_avl = NULL;
1950 	char *fromsnap = NULL;
1951 	char *cp;
1952 	char tofs[ZFS_MAXNAMELEN];
1953 	char sendfs[ZFS_MAXNAMELEN];
1954 	char errbuf[1024];
1955 	dmu_replay_record_t drre;
1956 	int error;
1957 	boolean_t anyerr = B_FALSE;
1958 	boolean_t softerr = B_FALSE;
1959 	boolean_t recursive;
1960 
1961 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
1962 	    "cannot receive"));
1963 
1964 	assert(drr->drr_type == DRR_BEGIN);
1965 	assert(drr->drr_u.drr_begin.drr_magic == DMU_BACKUP_MAGIC);
1966 	assert(DMU_GET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo) ==
1967 	    DMU_COMPOUNDSTREAM);
1968 
1969 	/*
1970 	 * Read in the nvlist from the stream.
1971 	 */
1972 	if (drr->drr_payloadlen != 0) {
1973 		error = recv_read_nvlist(hdl, fd, drr->drr_payloadlen,
1974 		    &stream_nv, flags.byteswap, zc);
1975 		if (error) {
1976 			error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
1977 			goto out;
1978 		}
1979 	}
1980 
1981 	recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
1982 	    ENOENT);
1983 
1984 	if (recursive && strchr(destname, '@')) {
1985 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1986 		    "cannot specify snapshot name for multi-snapshot stream"));
1987 		error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
1988 		goto out;
1989 	}
1990 
1991 	/*
1992 	 * Read in the end record and verify checksum.
1993 	 */
1994 	if (0 != (error = recv_read(hdl, fd, &drre, sizeof (drre),
1995 	    flags.byteswap, NULL)))
1996 		goto out;
1997 	if (flags.byteswap) {
1998 		drre.drr_type = BSWAP_32(drre.drr_type);
1999 		drre.drr_u.drr_end.drr_checksum.zc_word[0] =
2000 		    BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[0]);
2001 		drre.drr_u.drr_end.drr_checksum.zc_word[1] =
2002 		    BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[1]);
2003 		drre.drr_u.drr_end.drr_checksum.zc_word[2] =
2004 		    BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[2]);
2005 		drre.drr_u.drr_end.drr_checksum.zc_word[3] =
2006 		    BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[3]);
2007 	}
2008 	if (drre.drr_type != DRR_END) {
2009 		error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
2010 		goto out;
2011 	}
2012 	if (!ZIO_CHECKSUM_EQUAL(drre.drr_u.drr_end.drr_checksum, *zc)) {
2013 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2014 		    "incorrect header checksum"));
2015 		error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
2016 		goto out;
2017 	}
2018 
2019 	(void) nvlist_lookup_string(stream_nv, "fromsnap", &fromsnap);
2020 
2021 	if (drr->drr_payloadlen != 0) {
2022 		nvlist_t *stream_fss;
2023 
2024 		VERIFY(0 == nvlist_lookup_nvlist(stream_nv, "fss",
2025 		    &stream_fss));
2026 		if ((stream_avl = fsavl_create(stream_fss)) == NULL) {
2027 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2028 			    "couldn't allocate avl tree"));
2029 			error = zfs_error(hdl, EZFS_NOMEM, errbuf);
2030 			goto out;
2031 		}
2032 
2033 		if (fromsnap != NULL) {
2034 			nvlist_t *renamed = NULL;
2035 			nvpair_t *pair = NULL;
2036 
2037 			(void) strlcpy(tofs, destname, ZFS_MAXNAMELEN);
2038 			if (flags.isprefix) {
2039 				struct drr_begin *drrb = &drr->drr_u.drr_begin;
2040 				int i;
2041 
2042 				if (flags.istail) {
2043 					cp = strrchr(drrb->drr_toname, '/');
2044 					if (cp == NULL) {
2045 						(void) strlcat(tofs, "/",
2046 						    ZFS_MAXNAMELEN);
2047 						i = 0;
2048 					} else {
2049 						i = (cp - drrb->drr_toname);
2050 					}
2051 				} else {
2052 					i = strcspn(drrb->drr_toname, "/@");
2053 				}
2054 				/* zfs_receive_one() will create_parents() */
2055 				(void) strlcat(tofs, &drrb->drr_toname[i],
2056 				    ZFS_MAXNAMELEN);
2057 				*strchr(tofs, '@') = '\0';
2058 			}
2059 
2060 			if (recursive && !flags.dryrun && !flags.nomount) {
2061 				VERIFY(0 == nvlist_alloc(&renamed,
2062 				    NV_UNIQUE_NAME, 0));
2063 			}
2064 
2065 			softerr = recv_incremental_replication(hdl, tofs, flags,
2066 			    stream_nv, stream_avl, renamed);
2067 
2068 			/* Unmount renamed filesystems before receiving. */
2069 			while ((pair = nvlist_next_nvpair(renamed,
2070 			    pair)) != NULL) {
2071 				zfs_handle_t *zhp;
2072 				prop_changelist_t *clp = NULL;
2073 
2074 				zhp = zfs_open(hdl, nvpair_name(pair),
2075 				    ZFS_TYPE_FILESYSTEM);
2076 				if (zhp != NULL) {
2077 					clp = changelist_gather(zhp,
2078 					    ZFS_PROP_MOUNTPOINT, 0, 0);
2079 					zfs_close(zhp);
2080 					if (clp != NULL) {
2081 						softerr |=
2082 						    changelist_prefix(clp);
2083 						changelist_free(clp);
2084 					}
2085 				}
2086 			}
2087 
2088 			nvlist_free(renamed);
2089 		}
2090 	}
2091 
2092 	/*
2093 	 * Get the fs specified by the first path in the stream (the top level
2094 	 * specified by 'zfs send') and pass it to each invocation of
2095 	 * zfs_receive_one().
2096 	 */
2097 	(void) strlcpy(sendfs, drr->drr_u.drr_begin.drr_toname,
2098 	    ZFS_MAXNAMELEN);
2099 	if ((cp = strchr(sendfs, '@')) != NULL)
2100 		*cp = '\0';
2101 
2102 	/* Finally, receive each contained stream */
2103 	do {
2104 		/*
2105 		 * we should figure out if it has a recoverable
2106 		 * error, in which case do a recv_skip() and drive on.
2107 		 * Note, if we fail due to already having this guid,
2108 		 * zfs_receive_one() will take care of it (ie,
2109 		 * recv_skip() and return 0).
2110 		 */
2111 		error = zfs_receive_impl(hdl, destname, flags, fd,
2112 		    sendfs, stream_nv, stream_avl, top_zfs);
2113 		if (error == ENODATA) {
2114 			error = 0;
2115 			break;
2116 		}
2117 		anyerr |= error;
2118 	} while (error == 0);
2119 
2120 	if (drr->drr_payloadlen != 0 && fromsnap != NULL) {
2121 		/*
2122 		 * Now that we have the fs's they sent us, try the
2123 		 * renames again.
2124 		 */
2125 		softerr = recv_incremental_replication(hdl, tofs, flags,
2126 		    stream_nv, stream_avl, NULL);
2127 	}
2128 
2129 out:
2130 	fsavl_destroy(stream_avl);
2131 	if (stream_nv)
2132 		nvlist_free(stream_nv);
2133 	if (softerr)
2134 		error = -2;
2135 	if (anyerr)
2136 		error = -1;
2137 	return (error);
2138 }
2139 
2140 static void
2141 trunc_prop_errs(int truncated)
2142 {
2143 	ASSERT(truncated != 0);
2144 
2145 	if (truncated == 1)
2146 		(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
2147 		    "1 more property could not be set\n"));
2148 	else
2149 		(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
2150 		    "%d more properties could not be set\n"), truncated);
2151 }
2152 
2153 static int
2154 recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap)
2155 {
2156 	dmu_replay_record_t *drr;
2157 	void *buf = malloc(1<<20);
2158 	char errbuf[1024];
2159 
2160 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
2161 	    "cannot receive:"));
2162 
2163 	/* XXX would be great to use lseek if possible... */
2164 	drr = buf;
2165 
2166 	while (recv_read(hdl, fd, drr, sizeof (dmu_replay_record_t),
2167 	    byteswap, NULL) == 0) {
2168 		if (byteswap)
2169 			drr->drr_type = BSWAP_32(drr->drr_type);
2170 
2171 		switch (drr->drr_type) {
2172 		case DRR_BEGIN:
2173 			/* NB: not to be used on v2 stream packages */
2174 			if (drr->drr_payloadlen != 0) {
2175 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2176 				    "invalid substream header"));
2177 				return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
2178 			}
2179 			break;
2180 
2181 		case DRR_END:
2182 			free(buf);
2183 			return (0);
2184 
2185 		case DRR_OBJECT:
2186 			if (byteswap) {
2187 				drr->drr_u.drr_object.drr_bonuslen =
2188 				    BSWAP_32(drr->drr_u.drr_object.
2189 				    drr_bonuslen);
2190 			}
2191 			(void) recv_read(hdl, fd, buf,
2192 			    P2ROUNDUP(drr->drr_u.drr_object.drr_bonuslen, 8),
2193 			    B_FALSE, NULL);
2194 			break;
2195 
2196 		case DRR_WRITE:
2197 			if (byteswap) {
2198 				drr->drr_u.drr_write.drr_length =
2199 				    BSWAP_64(drr->drr_u.drr_write.drr_length);
2200 			}
2201 			(void) recv_read(hdl, fd, buf,
2202 			    drr->drr_u.drr_write.drr_length, B_FALSE, NULL);
2203 			break;
2204 		case DRR_SPILL:
2205 			if (byteswap) {
2206 				drr->drr_u.drr_write.drr_length =
2207 				    BSWAP_64(drr->drr_u.drr_spill.drr_length);
2208 			}
2209 			(void) recv_read(hdl, fd, buf,
2210 			    drr->drr_u.drr_spill.drr_length, B_FALSE, NULL);
2211 			break;
2212 		case DRR_WRITE_BYREF:
2213 		case DRR_FREEOBJECTS:
2214 		case DRR_FREE:
2215 			break;
2216 
2217 		default:
2218 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2219 			    "invalid record type"));
2220 			return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
2221 		}
2222 	}
2223 
2224 	free(buf);
2225 	return (-1);
2226 }
2227 
2228 /*
2229  * Restores a backup of tosnap from the file descriptor specified by infd.
2230  */
2231 static int
2232 zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
2233     recvflags_t flags, dmu_replay_record_t *drr,
2234     dmu_replay_record_t *drr_noswap, const char *sendfs,
2235     nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs)
2236 {
2237 	zfs_cmd_t zc = { 0 };
2238 	time_t begin_time;
2239 	int ioctl_err, ioctl_errno, err;
2240 	char *cp;
2241 	struct drr_begin *drrb = &drr->drr_u.drr_begin;
2242 	char errbuf[1024];
2243 	char prop_errbuf[1024];
2244 	const char *chopprefix;
2245 	boolean_t newfs = B_FALSE;
2246 	boolean_t stream_wantsnewfs;
2247 	uint64_t parent_snapguid = 0;
2248 	prop_changelist_t *clp = NULL;
2249 	nvlist_t *snapprops_nvlist = NULL;
2250 	zprop_errflags_t prop_errflags;
2251 	boolean_t recursive;
2252 
2253 	begin_time = time(NULL);
2254 
2255 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
2256 	    "cannot receive"));
2257 
2258 	recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
2259 	    ENOENT);
2260 
2261 	if (stream_avl != NULL) {
2262 		char *snapname;
2263 		nvlist_t *fs = fsavl_find(stream_avl, drrb->drr_toguid,
2264 		    &snapname);
2265 		nvlist_t *props;
2266 		int ret;
2267 
2268 		(void) nvlist_lookup_uint64(fs, "parentfromsnap",
2269 		    &parent_snapguid);
2270 		err = nvlist_lookup_nvlist(fs, "props", &props);
2271 		if (err)
2272 			VERIFY(0 == nvlist_alloc(&props, NV_UNIQUE_NAME, 0));
2273 
2274 		if (flags.canmountoff) {
2275 			VERIFY(0 == nvlist_add_uint64(props,
2276 			    zfs_prop_to_name(ZFS_PROP_CANMOUNT), 0));
2277 		}
2278 		ret = zcmd_write_src_nvlist(hdl, &zc, props);
2279 		if (err)
2280 			nvlist_free(props);
2281 
2282 		if (0 == nvlist_lookup_nvlist(fs, "snapprops", &props)) {
2283 			VERIFY(0 == nvlist_lookup_nvlist(props,
2284 			    snapname, &snapprops_nvlist));
2285 		}
2286 
2287 		if (ret != 0)
2288 			return (-1);
2289 	}
2290 
2291 	cp = NULL;
2292 
2293 	/*
2294 	 * Determine how much of the snapshot name stored in the stream
2295 	 * we are going to tack on to the name they specified on the
2296 	 * command line, and how much we are going to chop off.
2297 	 *
2298 	 * If they specified a snapshot, chop the entire name stored in
2299 	 * the stream.
2300 	 */
2301 	if (flags.istail) {
2302 		/*
2303 		 * A filesystem was specified with -e. We want to tack on only
2304 		 * the tail of the sent snapshot path.
2305 		 */
2306 		if (strchr(tosnap, '@')) {
2307 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
2308 			    "argument - snapshot not allowed with -e"));
2309 			return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf));
2310 		}
2311 
2312 		chopprefix = strrchr(sendfs, '/');
2313 
2314 		if (chopprefix == NULL) {
2315 			/*
2316 			 * The tail is the poolname, so we need to
2317 			 * prepend a path separator.
2318 			 */
2319 			int len = strlen(drrb->drr_toname);
2320 			cp = malloc(len + 2);
2321 			cp[0] = '/';
2322 			(void) strcpy(&cp[1], drrb->drr_toname);
2323 			chopprefix = cp;
2324 		} else {
2325 			chopprefix = drrb->drr_toname + (chopprefix - sendfs);
2326 		}
2327 	} else if (flags.isprefix) {
2328 		/*
2329 		 * A filesystem was specified with -d. We want to tack on
2330 		 * everything but the first element of the sent snapshot path
2331 		 * (all but the pool name).
2332 		 */
2333 		if (strchr(tosnap, '@')) {
2334 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
2335 			    "argument - snapshot not allowed with -d"));
2336 			return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf));
2337 		}
2338 
2339 		chopprefix = strchr(drrb->drr_toname, '/');
2340 		if (chopprefix == NULL)
2341 			chopprefix = strchr(drrb->drr_toname, '@');
2342 	} else if (strchr(tosnap, '@') == NULL) {
2343 		/*
2344 		 * If a filesystem was specified without -d or -e, we want to
2345 		 * tack on everything after the fs specified by 'zfs send'.
2346 		 */
2347 		chopprefix = drrb->drr_toname + strlen(sendfs);
2348 	} else {
2349 		/* A snapshot was specified as an exact path (no -d or -e). */
2350 		if (recursive) {
2351 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2352 			    "cannot specify snapshot name for multi-snapshot "
2353 			    "stream"));
2354 			return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
2355 		}
2356 		chopprefix = drrb->drr_toname + strlen(drrb->drr_toname);
2357 	}
2358 
2359 	ASSERT(strstr(drrb->drr_toname, sendfs) == drrb->drr_toname);
2360 	ASSERT(chopprefix > drrb->drr_toname);
2361 	ASSERT(chopprefix <= drrb->drr_toname + strlen(drrb->drr_toname));
2362 	ASSERT(chopprefix[0] == '/' || chopprefix[0] == '@' ||
2363 	    chopprefix[0] == '\0');
2364 
2365 	/*
2366 	 * Determine name of destination snapshot, store in zc_value.
2367 	 */
2368 	(void) strcpy(zc.zc_top_ds, tosnap);
2369 	(void) strcpy(zc.zc_value, tosnap);
2370 	(void) strncat(zc.zc_value, chopprefix, sizeof (zc.zc_value));
2371 	free(cp);
2372 	if (!zfs_name_valid(zc.zc_value, ZFS_TYPE_SNAPSHOT)) {
2373 		zcmd_free_nvlists(&zc);
2374 		return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf));
2375 	}
2376 
2377 	/*
2378 	 * Determine the name of the origin snapshot, store in zc_string.
2379 	 */
2380 	if (drrb->drr_flags & DRR_FLAG_CLONE) {
2381 		if (guid_to_name(hdl, tosnap,
2382 		    drrb->drr_fromguid, zc.zc_string) != 0) {
2383 			zcmd_free_nvlists(&zc);
2384 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2385 			    "local origin for clone %s does not exist"),
2386 			    zc.zc_value);
2387 			return (zfs_error(hdl, EZFS_NOENT, errbuf));
2388 		}
2389 		if (flags.verbose)
2390 			(void) printf("found clone origin %s\n", zc.zc_string);
2391 	}
2392 
2393 	stream_wantsnewfs = (drrb->drr_fromguid == NULL ||
2394 	    (drrb->drr_flags & DRR_FLAG_CLONE));
2395 
2396 	if (stream_wantsnewfs) {
2397 		/*
2398 		 * if the parent fs does not exist, look for it based on
2399 		 * the parent snap GUID
2400 		 */
2401 		(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
2402 		    "cannot receive new filesystem stream"));
2403 
2404 		(void) strcpy(zc.zc_name, zc.zc_value);
2405 		cp = strrchr(zc.zc_name, '/');
2406 		if (cp)
2407 			*cp = '\0';
2408 		if (cp &&
2409 		    !zfs_dataset_exists(hdl, zc.zc_name, ZFS_TYPE_DATASET)) {
2410 			char suffix[ZFS_MAXNAMELEN];
2411 			(void) strcpy(suffix, strrchr(zc.zc_value, '/'));
2412 			if (guid_to_name(hdl, tosnap, parent_snapguid,
2413 			    zc.zc_value) == 0) {
2414 				*strchr(zc.zc_value, '@') = '\0';
2415 				(void) strcat(zc.zc_value, suffix);
2416 			}
2417 		}
2418 	} else {
2419 		/*
2420 		 * if the fs does not exist, look for it based on the
2421 		 * fromsnap GUID
2422 		 */
2423 		(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
2424 		    "cannot receive incremental stream"));
2425 
2426 		(void) strcpy(zc.zc_name, zc.zc_value);
2427 		*strchr(zc.zc_name, '@') = '\0';
2428 
2429 		/*
2430 		 * If the exact receive path was specified and this is the
2431 		 * topmost path in the stream, then if the fs does not exist we
2432 		 * should look no further.
2433 		 */
2434 		if ((flags.isprefix || (*(chopprefix = drrb->drr_toname +
2435 		    strlen(sendfs)) != '\0' && *chopprefix != '@')) &&
2436 		    !zfs_dataset_exists(hdl, zc.zc_name, ZFS_TYPE_DATASET)) {
2437 			char snap[ZFS_MAXNAMELEN];
2438 			(void) strcpy(snap, strchr(zc.zc_value, '@'));
2439 			if (guid_to_name(hdl, tosnap, drrb->drr_fromguid,
2440 			    zc.zc_value) == 0) {
2441 				*strchr(zc.zc_value, '@') = '\0';
2442 				(void) strcat(zc.zc_value, snap);
2443 			}
2444 		}
2445 	}
2446 
2447 	(void) strcpy(zc.zc_name, zc.zc_value);
2448 	*strchr(zc.zc_name, '@') = '\0';
2449 
2450 	if (zfs_dataset_exists(hdl, zc.zc_name, ZFS_TYPE_DATASET)) {
2451 		zfs_handle_t *zhp;
2452 
2453 		/*
2454 		 * Destination fs exists.  Therefore this should either
2455 		 * be an incremental, or the stream specifies a new fs
2456 		 * (full stream or clone) and they want us to blow it
2457 		 * away (and have therefore specified -F and removed any
2458 		 * snapshots).
2459 		 */
2460 		if (stream_wantsnewfs) {
2461 			if (!flags.force) {
2462 				zcmd_free_nvlists(&zc);
2463 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2464 				    "destination '%s' exists\n"
2465 				    "must specify -F to overwrite it"),
2466 				    zc.zc_name);
2467 				return (zfs_error(hdl, EZFS_EXISTS, errbuf));
2468 			}
2469 			if (ioctl(hdl->libzfs_fd, ZFS_IOC_SNAPSHOT_LIST_NEXT,
2470 			    &zc) == 0) {
2471 				zcmd_free_nvlists(&zc);
2472 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2473 				    "destination has snapshots (eg. %s)\n"
2474 				    "must destroy them to overwrite it"),
2475 				    zc.zc_name);
2476 				return (zfs_error(hdl, EZFS_EXISTS, errbuf));
2477 			}
2478 		}
2479 
2480 		if ((zhp = zfs_open(hdl, zc.zc_name,
2481 		    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME)) == NULL) {
2482 			zcmd_free_nvlists(&zc);
2483 			return (-1);
2484 		}
2485 
2486 		if (stream_wantsnewfs &&
2487 		    zhp->zfs_dmustats.dds_origin[0]) {
2488 			zcmd_free_nvlists(&zc);
2489 			zfs_close(zhp);
2490 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2491 			    "destination '%s' is a clone\n"
2492 			    "must destroy it to overwrite it"),
2493 			    zc.zc_name);
2494 			return (zfs_error(hdl, EZFS_EXISTS, errbuf));
2495 		}
2496 
2497 		if (!flags.dryrun && zhp->zfs_type == ZFS_TYPE_FILESYSTEM &&
2498 		    stream_wantsnewfs) {
2499 			/* We can't do online recv in this case */
2500 			clp = changelist_gather(zhp, ZFS_PROP_NAME, 0, 0);
2501 			if (clp == NULL) {
2502 				zfs_close(zhp);
2503 				zcmd_free_nvlists(&zc);
2504 				return (-1);
2505 			}
2506 			if (changelist_prefix(clp) != 0) {
2507 				changelist_free(clp);
2508 				zfs_close(zhp);
2509 				zcmd_free_nvlists(&zc);
2510 				return (-1);
2511 			}
2512 		}
2513 		zfs_close(zhp);
2514 	} else {
2515 		/*
2516 		 * Destination filesystem does not exist.  Therefore we better
2517 		 * be creating a new filesystem (either from a full backup, or
2518 		 * a clone).  It would therefore be invalid if the user
2519 		 * specified only the pool name (i.e. if the destination name
2520 		 * contained no slash character).
2521 		 */
2522 		if (!stream_wantsnewfs ||
2523 		    (cp = strrchr(zc.zc_name, '/')) == NULL) {
2524 			zcmd_free_nvlists(&zc);
2525 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2526 			    "destination '%s' does not exist"), zc.zc_name);
2527 			return (zfs_error(hdl, EZFS_NOENT, errbuf));
2528 		}
2529 
2530 		/*
2531 		 * Trim off the final dataset component so we perform the
2532 		 * recvbackup ioctl to the filesystems's parent.
2533 		 */
2534 		*cp = '\0';
2535 
2536 		if (flags.isprefix && !flags.istail && !flags.dryrun &&
2537 		    create_parents(hdl, zc.zc_value, strlen(tosnap)) != 0) {
2538 			zcmd_free_nvlists(&zc);
2539 			return (zfs_error(hdl, EZFS_BADRESTORE, errbuf));
2540 		}
2541 
2542 		newfs = B_TRUE;
2543 	}
2544 
2545 	zc.zc_begin_record = drr_noswap->drr_u.drr_begin;
2546 	zc.zc_cookie = infd;
2547 	zc.zc_guid = flags.force;
2548 	if (flags.verbose) {
2549 		(void) printf("%s %s stream of %s into %s\n",
2550 		    flags.dryrun ? "would receive" : "receiving",
2551 		    drrb->drr_fromguid ? "incremental" : "full",
2552 		    drrb->drr_toname, zc.zc_value);
2553 		(void) fflush(stdout);
2554 	}
2555 
2556 	if (flags.dryrun) {
2557 		zcmd_free_nvlists(&zc);
2558 		return (recv_skip(hdl, infd, flags.byteswap));
2559 	}
2560 
2561 	zc.zc_nvlist_dst = (uint64_t)(uintptr_t)prop_errbuf;
2562 	zc.zc_nvlist_dst_size = sizeof (prop_errbuf);
2563 
2564 	err = ioctl_err = zfs_ioctl(hdl, ZFS_IOC_RECV, &zc);
2565 	ioctl_errno = errno;
2566 	prop_errflags = (zprop_errflags_t)zc.zc_obj;
2567 
2568 	if (err == 0) {
2569 		nvlist_t *prop_errors;
2570 		VERIFY(0 == nvlist_unpack((void *)(uintptr_t)zc.zc_nvlist_dst,
2571 		    zc.zc_nvlist_dst_size, &prop_errors, 0));
2572 
2573 		nvpair_t *prop_err = NULL;
2574 
2575 		while ((prop_err = nvlist_next_nvpair(prop_errors,
2576 		    prop_err)) != NULL) {
2577 			char tbuf[1024];
2578 			zfs_prop_t prop;
2579 			int intval;
2580 
2581 			prop = zfs_name_to_prop(nvpair_name(prop_err));
2582 			(void) nvpair_value_int32(prop_err, &intval);
2583 			if (strcmp(nvpair_name(prop_err),
2584 			    ZPROP_N_MORE_ERRORS) == 0) {
2585 				trunc_prop_errs(intval);
2586 				break;
2587 			} else {
2588 				(void) snprintf(tbuf, sizeof (tbuf),
2589 				    dgettext(TEXT_DOMAIN,
2590 				    "cannot receive %s property on %s"),
2591 				    nvpair_name(prop_err), zc.zc_name);
2592 				zfs_setprop_error(hdl, prop, intval, tbuf);
2593 			}
2594 		}
2595 		nvlist_free(prop_errors);
2596 	}
2597 
2598 	zc.zc_nvlist_dst = 0;
2599 	zc.zc_nvlist_dst_size = 0;
2600 	zcmd_free_nvlists(&zc);
2601 
2602 	if (err == 0 && snapprops_nvlist) {
2603 		zfs_cmd_t zc2 = { 0 };
2604 
2605 		(void) strcpy(zc2.zc_name, zc.zc_value);
2606 		zc2.zc_cookie = B_TRUE; /* received */
2607 		if (zcmd_write_src_nvlist(hdl, &zc2, snapprops_nvlist) == 0) {
2608 			(void) zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc2);
2609 			zcmd_free_nvlists(&zc2);
2610 		}
2611 	}
2612 
2613 	if (err && (ioctl_errno == ENOENT || ioctl_errno == EEXIST)) {
2614 		/*
2615 		 * It may be that this snapshot already exists,
2616 		 * in which case we want to consume & ignore it
2617 		 * rather than failing.
2618 		 */
2619 		avl_tree_t *local_avl;
2620 		nvlist_t *local_nv, *fs;
2621 		cp = strchr(zc.zc_value, '@');
2622 
2623 		/*
2624 		 * XXX Do this faster by just iterating over snaps in
2625 		 * this fs.  Also if zc_value does not exist, we will
2626 		 * get a strange "does not exist" error message.
2627 		 */
2628 		*cp = '\0';
2629 		if (gather_nvlist(hdl, zc.zc_value, NULL, NULL, B_FALSE,
2630 		    &local_nv, &local_avl) == 0) {
2631 			*cp = '@';
2632 			fs = fsavl_find(local_avl, drrb->drr_toguid, NULL);
2633 			fsavl_destroy(local_avl);
2634 			nvlist_free(local_nv);
2635 
2636 			if (fs != NULL) {
2637 				if (flags.verbose) {
2638 					(void) printf("snap %s already exists; "
2639 					    "ignoring\n", zc.zc_value);
2640 				}
2641 				err = ioctl_err = recv_skip(hdl, infd,
2642 				    flags.byteswap);
2643 			}
2644 		}
2645 		*cp = '@';
2646 	}
2647 
2648 	if (ioctl_err != 0) {
2649 		switch (ioctl_errno) {
2650 		case ENODEV:
2651 			cp = strchr(zc.zc_value, '@');
2652 			*cp = '\0';
2653 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2654 			    "most recent snapshot of %s does not\n"
2655 			    "match incremental source"), zc.zc_value);
2656 			(void) zfs_error(hdl, EZFS_BADRESTORE, errbuf);
2657 			*cp = '@';
2658 			break;
2659 		case ETXTBSY:
2660 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2661 			    "destination %s has been modified\n"
2662 			    "since most recent snapshot"), zc.zc_name);
2663 			(void) zfs_error(hdl, EZFS_BADRESTORE, errbuf);
2664 			break;
2665 		case EEXIST:
2666 			cp = strchr(zc.zc_value, '@');
2667 			if (newfs) {
2668 				/* it's the containing fs that exists */
2669 				*cp = '\0';
2670 			}
2671 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2672 			    "destination already exists"));
2673 			(void) zfs_error_fmt(hdl, EZFS_EXISTS,
2674 			    dgettext(TEXT_DOMAIN, "cannot restore to %s"),
2675 			    zc.zc_value);
2676 			*cp = '@';
2677 			break;
2678 		case EINVAL:
2679 			(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
2680 			break;
2681 		case ECKSUM:
2682 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2683 			    "invalid stream (checksum mismatch)"));
2684 			(void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
2685 			break;
2686 		default:
2687 			(void) zfs_standard_error(hdl, ioctl_errno, errbuf);
2688 		}
2689 	}
2690 
2691 	/*
2692 	 * Mount the target filesystem (if created).  Also mount any
2693 	 * children of the target filesystem if we did a replication
2694 	 * receive (indicated by stream_avl being non-NULL).
2695 	 */
2696 	cp = strchr(zc.zc_value, '@');
2697 	if (cp && (ioctl_err == 0 || !newfs)) {
2698 		zfs_handle_t *h;
2699 
2700 		*cp = '\0';
2701 		h = zfs_open(hdl, zc.zc_value,
2702 		    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
2703 		if (h != NULL) {
2704 			if (h->zfs_type == ZFS_TYPE_VOLUME) {
2705 				*cp = '@';
2706 			} else if (newfs || stream_avl) {
2707 				/*
2708 				 * Track the first/top of hierarchy fs,
2709 				 * for mounting and sharing later.
2710 				 */
2711 				if (top_zfs && *top_zfs == NULL)
2712 					*top_zfs = zfs_strdup(hdl, zc.zc_value);
2713 			}
2714 			zfs_close(h);
2715 		}
2716 		*cp = '@';
2717 	}
2718 
2719 	if (clp) {
2720 		err |= changelist_postfix(clp);
2721 		changelist_free(clp);
2722 	}
2723 
2724 	if (prop_errflags & ZPROP_ERR_NOCLEAR) {
2725 		(void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: "
2726 		    "failed to clear unreceived properties on %s"),
2727 		    zc.zc_name);
2728 		(void) fprintf(stderr, "\n");
2729 	}
2730 	if (prop_errflags & ZPROP_ERR_NORESTORE) {
2731 		(void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: "
2732 		    "failed to restore original properties on %s"),
2733 		    zc.zc_name);
2734 		(void) fprintf(stderr, "\n");
2735 	}
2736 
2737 	if (err || ioctl_err)
2738 		return (-1);
2739 
2740 	if (flags.verbose) {
2741 		char buf1[64];
2742 		char buf2[64];
2743 		uint64_t bytes = zc.zc_cookie;
2744 		time_t delta = time(NULL) - begin_time;
2745 		if (delta == 0)
2746 			delta = 1;
2747 		zfs_nicenum(bytes, buf1, sizeof (buf1));
2748 		zfs_nicenum(bytes/delta, buf2, sizeof (buf1));
2749 
2750 		(void) printf("received %sB stream in %lu seconds (%sB/sec)\n",
2751 		    buf1, delta, buf2);
2752 	}
2753 
2754 	return (0);
2755 }
2756 
2757 static int
2758 zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, recvflags_t flags,
2759     int infd, const char *sendfs, nvlist_t *stream_nv, avl_tree_t *stream_avl,
2760     char **top_zfs)
2761 {
2762 	int err;
2763 	dmu_replay_record_t drr, drr_noswap;
2764 	struct drr_begin *drrb = &drr.drr_u.drr_begin;
2765 	char errbuf[1024];
2766 	zio_cksum_t zcksum = { 0 };
2767 	uint64_t featureflags;
2768 	int hdrtype;
2769 
2770 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
2771 	    "cannot receive"));
2772 
2773 	if (flags.isprefix &&
2774 	    !zfs_dataset_exists(hdl, tosnap, ZFS_TYPE_DATASET)) {
2775 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "specified fs "
2776 		    "(%s) does not exist"), tosnap);
2777 		return (zfs_error(hdl, EZFS_NOENT, errbuf));
2778 	}
2779 
2780 	/* read in the BEGIN record */
2781 	if (0 != (err = recv_read(hdl, infd, &drr, sizeof (drr), B_FALSE,
2782 	    &zcksum)))
2783 		return (err);
2784 
2785 	if (drr.drr_type == DRR_END || drr.drr_type == BSWAP_32(DRR_END)) {
2786 		/* It's the double end record at the end of a package */
2787 		return (ENODATA);
2788 	}
2789 
2790 	/* the kernel needs the non-byteswapped begin record */
2791 	drr_noswap = drr;
2792 
2793 	flags.byteswap = B_FALSE;
2794 	if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
2795 		/*
2796 		 * We computed the checksum in the wrong byteorder in
2797 		 * recv_read() above; do it again correctly.
2798 		 */
2799 		bzero(&zcksum, sizeof (zio_cksum_t));
2800 		fletcher_4_incremental_byteswap(&drr, sizeof (drr), &zcksum);
2801 		flags.byteswap = B_TRUE;
2802 
2803 		drr.drr_type = BSWAP_32(drr.drr_type);
2804 		drr.drr_payloadlen = BSWAP_32(drr.drr_payloadlen);
2805 		drrb->drr_magic = BSWAP_64(drrb->drr_magic);
2806 		drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo);
2807 		drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time);
2808 		drrb->drr_type = BSWAP_32(drrb->drr_type);
2809 		drrb->drr_flags = BSWAP_32(drrb->drr_flags);
2810 		drrb->drr_toguid = BSWAP_64(drrb->drr_toguid);
2811 		drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid);
2812 	}
2813 
2814 	if (drrb->drr_magic != DMU_BACKUP_MAGIC || drr.drr_type != DRR_BEGIN) {
2815 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
2816 		    "stream (bad magic number)"));
2817 		return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
2818 	}
2819 
2820 	featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
2821 	hdrtype = DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo);
2822 
2823 	if (!DMU_STREAM_SUPPORTED(featureflags) ||
2824 	    (hdrtype != DMU_SUBSTREAM && hdrtype != DMU_COMPOUNDSTREAM)) {
2825 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2826 		    "stream has unsupported feature, feature flags = %lx"),
2827 		    featureflags);
2828 		return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
2829 	}
2830 
2831 	if (strchr(drrb->drr_toname, '@') == NULL) {
2832 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
2833 		    "stream (bad snapshot name)"));
2834 		return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
2835 	}
2836 
2837 	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_SUBSTREAM) {
2838 		char nonpackage_sendfs[ZFS_MAXNAMELEN];
2839 		if (sendfs == NULL) {
2840 			/*
2841 			 * We were not called from zfs_receive_package(). Get
2842 			 * the fs specified by 'zfs send'.
2843 			 */
2844 			char *cp;
2845 			(void) strlcpy(nonpackage_sendfs,
2846 			    drr.drr_u.drr_begin.drr_toname, ZFS_MAXNAMELEN);
2847 			if ((cp = strchr(nonpackage_sendfs, '@')) != NULL)
2848 				*cp = '\0';
2849 			sendfs = nonpackage_sendfs;
2850 		}
2851 		return (zfs_receive_one(hdl, infd, tosnap, flags,
2852 		    &drr, &drr_noswap, sendfs, stream_nv, stream_avl,
2853 		    top_zfs));
2854 	} else {
2855 		assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
2856 		    DMU_COMPOUNDSTREAM);
2857 		return (zfs_receive_package(hdl, infd, tosnap, flags,
2858 		    &drr, &zcksum, top_zfs));
2859 	}
2860 }
2861 
2862 /*
2863  * Restores a backup of tosnap from the file descriptor specified by infd.
2864  * Return 0 on total success, -2 if some things couldn't be
2865  * destroyed/renamed/promoted, -1 if some things couldn't be received.
2866  * (-1 will override -2).
2867  */
2868 int
2869 zfs_receive(libzfs_handle_t *hdl, const char *tosnap, recvflags_t flags,
2870     int infd, avl_tree_t *stream_avl)
2871 {
2872 	char *top_zfs = NULL;
2873 	int err;
2874 
2875 	err = zfs_receive_impl(hdl, tosnap, flags, infd, NULL, NULL,
2876 	    stream_avl, &top_zfs);
2877 
2878 	if (err == 0 && !flags.nomount && top_zfs) {
2879 		zfs_handle_t *zhp;
2880 		prop_changelist_t *clp;
2881 
2882 		zhp = zfs_open(hdl, top_zfs, ZFS_TYPE_FILESYSTEM);
2883 		if (zhp != NULL) {
2884 			clp = changelist_gather(zhp, ZFS_PROP_MOUNTPOINT,
2885 			    CL_GATHER_MOUNT_ALWAYS, 0);
2886 			zfs_close(zhp);
2887 			if (clp != NULL) {
2888 				/* mount and share received datasets */
2889 				err = changelist_postfix(clp);
2890 				changelist_free(clp);
2891 			}
2892 		}
2893 		if (zhp == NULL || clp == NULL || err)
2894 			err = -1;
2895 	}
2896 	if (top_zfs)
2897 		free(top_zfs);
2898 
2899 	return (err);
2900 }
2901