228 static int do_dump(dmu_send_cookie_t *dscp, struct send_range *range);
231 range_free(struct send_range *range)  in range_free()  argument
233 	if (range->type == OBJECT) {  in range_free()
235 		    (range->sru.object.dnp->dn_extra_slots + 1);  in range_free()
236 		kmem_free(range->sru.object.dnp, size);  in range_free()
237 		if (range->sru.object.spill_range)  in range_free()
238 			range_free(range->sru.object.spill_range);  in range_free()
239 	} else if (range->type == DATA) {  in range_free()
240 		mutex_enter(&range->sru.data.lock);  in range_free()
241 		while (range->sru.data.io_outstanding)  in range_free()
242 			cv_wait(&range->sru.data.cv, &range->sru.data.lock);  in range_free()
243 		if (range->sru.data.abd != NULL)  in range_free()
244 			abd_free(range->sru.data.abd);  in range_free()
245 		if (range->sru.data.abuf != NULL) {  in range_free()
246 			arc_buf_destroy(range->sru.data.abuf,  in range_free()
247 			    &range->sru.data.abuf);  in range_free()
249 		mutex_exit(&range->sru.data.lock);  in range_free()
251 		cv_destroy(&range->sru.data.cv);  in range_free()
252 		mutex_destroy(&range->sru.data.lock);  in range_free()
254 	kmem_free(range, sizeof (*range));  in range_free()
883 do_dump(dmu_send_cookie_t *dscp, struct send_range *range)  in do_dump()  argument
886 	switch (range->type) {  in do_dump()
888 		err = dump_dnode(dscp, &range->sru.object.bp, range->object,  in do_dump()
889 		    range->sru.object.dnp);  in do_dump()
891 		if (!err && range->sru.object.spill_range)  in do_dump()
892 			err = do_dump(dscp, range->sru.object.spill_range);  in do_dump()
895 		ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);  in do_dump()
899 		uint64_t epb = BP_GET_LSIZE(&range->sru.object_range.bp) >>  in do_dump()
901 		uint64_t firstobj = range->start_blkid * epb;  in do_dump()
902 		err = dump_object_range(dscp, &range->sru.object_range.bp,  in do_dump()
907 		struct srr *srrp = &range->sru.redact;  in do_dump()
908 		err = dump_redact(dscp, range->object, range->start_blkid *  in do_dump()
909 		    srrp->datablksz, (range->end_blkid - range->start_blkid) *  in do_dump()
914 		struct srd *srdp = &range->sru.data;  in do_dump()
920 		ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);  in do_dump()
923 			err = dump_write_embedded(dscp, range->object,  in do_dump()
924 			    range->start_blkid * srdp->datablksz,  in do_dump()
928 		ASSERT(range->object > dscp->dsc_resume_object ||  in do_dump()
929 		    (range->object == dscp->dsc_resume_object &&  in do_dump()
930 		    (range->start_blkid == DMU_SPILL_BLKID ||  in do_dump()
931 		    range->start_blkid * srdp->datablksz >=  in do_dump()
971 			ASSERT3U(range->start_blkid, ==, DMU_SPILL_BLKID);  in do_dump()
972 			err = dump_spill(dscp, bp, range->object, data);  in do_dump()
976 		uint64_t offset = range->start_blkid * srdp->datablksz;  in do_dump()
990 				    range->object, offset, n, n, NULL, B_FALSE,  in do_dump()
1004 			    range->object, offset,  in do_dump()
1011 		struct srh *srhp = &range->sru.hole;  in do_dump()
1012 		if (range->object == DMU_META_DNODE_OBJECT) {  in do_dump()
1014 			uint64_t first_obj = range->start_blkid * span;  in do_dump()
1015 			uint64_t numobj = range->end_blkid * span - first_obj;  in do_dump()
1025 		if (!overflow_multiply(range->start_blkid, srhp->datablksz,  in do_dump()
1031 		if (!overflow_multiply(range->end_blkid, srhp->datablksz, &len))  in do_dump()
1034 		return (dump_free(dscp, range->object, offset, len));  in do_dump()
1037 		panic("Invalid range type in do_dump: %d", range->type);  in do_dump()
1046 	struct send_range *range = kmem_alloc(sizeof (*range), KM_SLEEP);  in range_alloc()  local
1047 	range->type = type;  in range_alloc()
1048 	range->object = object;  in range_alloc()
1049 	range->start_blkid = start_blkid;  in range_alloc()
1050 	range->end_blkid = end_blkid;  in range_alloc()
1051 	range->eos_marker = eos;  in range_alloc()
1053 		range->sru.data.abd = NULL;  in range_alloc()
1054 		range->sru.data.abuf = NULL;  in range_alloc()
1055 		mutex_init(&range->sru.data.lock, NULL, MUTEX_DEFAULT, NULL);  in range_alloc()
1056 		cv_init(&range->sru.data.cv, NULL, CV_DEFAULT, NULL);  in range_alloc()
1057 		range->sru.data.io_outstanding = 0;  in range_alloc()
1058 		range->sru.data.io_err = 0;  in range_alloc()
1059 		range->sru.data.io_compressed = B_FALSE;  in range_alloc()
1061 		range->sru.object.spill_range = NULL;  in range_alloc()
1063 	return (range);  in range_alloc()
1513 	struct send_range *range;  in send_merge_thread()  local
1514 	for (range = find_next_range(front_ranges, queues, &mask);  in send_merge_thread()
1515 	    !range->eos_marker && err == 0 && !smt_arg->cancel;  in send_merge_thread()
1516 	    range = find_next_range(front_ranges, queues, &mask)) {  in send_merge_thread()
1524 			ASSERT3U(range->type, ==, REDACT);  in send_merge_thread()
1525 			range_free(range);  in send_merge_thread()
1528 		bqueue_enqueue(&smt_arg->q, range, sizeof (*range));  in send_merge_thread()
1555 	range->eos_marker = B_TRUE;  in send_merge_thread()
1556 	bqueue_enqueue_flush(&smt_arg->q, range, 1);  in send_merge_thread()
1573 	struct send_range *range = zio->io_private;  in dmu_send_read_done()  local
1575 	mutex_enter(&range->sru.data.lock);  in dmu_send_read_done()
1577 		abd_free(range->sru.data.abd);  in dmu_send_read_done()
1578 		range->sru.data.abd = NULL;  in dmu_send_read_done()
1579 		range->sru.data.io_err = zio->io_error;  in dmu_send_read_done()
1582 	ASSERT(range->sru.data.io_outstanding);  in dmu_send_read_done()
1583 	range->sru.data.io_outstanding = B_FALSE;  in dmu_send_read_done()
1584 	cv_broadcast(&range->sru.data.cv);  in dmu_send_read_done()
1585 	mutex_exit(&range->sru.data.lock);  in dmu_send_read_done()
1589 issue_data_read(struct send_reader_thread_arg *srta, struct send_range *range)  in issue_data_read()  argument
1591 	struct srd *srdp = &range->sru.data;  in issue_data_read()
1595 	ASSERT3U(range->type, ==, DATA);  in issue_data_read()
1596 	ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);  in issue_data_read()
1643 	    .zb_object = range->object,  in issue_data_read()
1645 	    .zb_blkid = range->start_blkid,  in issue_data_read()
1663 		    srdp->datasz, dmu_send_read_done, range,  in issue_data_read()
1678 	struct send_range *range = range_alloc(range_type, dn->dn_object,  in enqueue_range()  local
1688 		range->sru.hole.datablksz = datablksz;  in enqueue_range()
1692 		range->sru.data.datablksz = datablksz;  in enqueue_range()
1693 		range->sru.data.obj_type = dn->dn_type;  in enqueue_range()
1694 		range->sru.data.bp = *bp;  in enqueue_range()
1695 		issue_data_read(srta, range);  in enqueue_range()
1698 		range->sru.redact.datablksz = datablksz;  in enqueue_range()
1703 	bqueue_enqueue(q, range, datablksz);  in enqueue_range()
1720     struct send_range *range)  in piggyback_unmodified_spill()  argument
1722 	ASSERT3U(range->type, ==, OBJECT);  in piggyback_unmodified_spill()
1724 	dnode_phys_t *dnp = range->sru.object.dnp;  in piggyback_unmodified_spill()
1733 	struct send_range *spill_range = range_alloc(DATA, range->object,  in piggyback_unmodified_spill()
1740 	range->sru.object.spill_range = spill_range;  in piggyback_unmodified_spill()
1761 	struct send_range *range = bqueue_dequeue(inq);  in send_reader_thread()  local
1774 	while (!range->eos_marker && !srta->cancel && smta->error == 0 &&  in send_reader_thread()
1777 		switch (range->type) {  in send_reader_thread()
1779 			issue_data_read(srta, range);  in send_reader_thread()
1780 			bqueue_enqueue(outq, range, range->sru.data.datablksz);  in send_reader_thread()
1781 			range = get_next_range_nofree(inq, range);  in send_reader_thread()
1784 			spill = piggyback_unmodified_spill(srta, range);  in send_reader_thread()
1789 			bqueue_enqueue(outq, range, sizeof (*range) + spill);  in send_reader_thread()
1790 			range = get_next_range_nofree(inq, range);  in send_reader_thread()
1817 			if (range->object == last_obj && !last_obj_exists) {  in send_reader_thread()
1825 				err = dnode_hold(os, range->object, FTAG, &dn);  in send_reader_thread()
1830 				last_obj = range->object;  in send_reader_thread()
1843 				range = get_next_range(inq, range);  in send_reader_thread()
1847 			    MIN(dn->dn_maxblkid, range->end_blkid);  in send_reader_thread()
1853 			for (uint64_t blkid = range->start_blkid;  in send_reader_thread()
1901 			range = get_next_range(inq, range);  in send_reader_thread()
1911 	while (!range->eos_marker)  in send_reader_thread()
1912 		range = get_next_range(inq, range);  in send_reader_thread()
1914 	bqueue_enqueue_flush(outq, range, 1);  in send_reader_thread()
2377 	struct send_range *range;  in dmu_send_impl()  local
2569 	range = bqueue_dequeue(&srt_arg->q);  in dmu_send_impl()
2570 	while (err == 0 && !range->eos_marker) {  in dmu_send_impl()
2571 		err = do_dump(&dsc, range);  in dmu_send_impl()
2572 		range = get_next_range(&srt_arg->q, range);  in dmu_send_impl()
2585 		while (!range->eos_marker) {  in dmu_send_impl()
2586 			range = get_next_range(&srt_arg->q, range);  in dmu_send_impl()
2589 	range_free(range);  in dmu_send_impl()