227 static int do_dump(dmu_send_cookie_t *dscp, struct send_range *range);
230 range_free(struct send_range *range)  in range_free()  argument
232 	if (range->type == OBJECT) {  in range_free()
234 		    (range->sru.object.dnp->dn_extra_slots + 1);  in range_free()
235 		kmem_free(range->sru.object.dnp, size);  in range_free()
236 		if (range->sru.object.spill_range)  in range_free()
237 			range_free(range->sru.object.spill_range);  in range_free()
238 	} else if (range->type == DATA) {  in range_free()
239 		mutex_enter(&range->sru.data.lock);  in range_free()
240 		while (range->sru.data.io_outstanding)  in range_free()
241 			cv_wait(&range->sru.data.cv, &range->sru.data.lock);  in range_free()
242 		if (range->sru.data.abd != NULL)  in range_free()
243 			abd_free(range->sru.data.abd);  in range_free()
244 		if (range->sru.data.abuf != NULL) {  in range_free()
245 			arc_buf_destroy(range->sru.data.abuf,  in range_free()
246 			    &range->sru.data.abuf);  in range_free()
248 		mutex_exit(&range->sru.data.lock);  in range_free()
250 		cv_destroy(&range->sru.data.cv);  in range_free()
251 		mutex_destroy(&range->sru.data.lock);  in range_free()
253 	kmem_free(range, sizeof (*range));  in range_free()
882 do_dump(dmu_send_cookie_t *dscp, struct send_range *range)  in do_dump()  argument
885 	switch (range->type) {  in do_dump()
887 		err = dump_dnode(dscp, &range->sru.object.bp, range->object,  in do_dump()
888 		    range->sru.object.dnp);  in do_dump()
890 		if (!err && range->sru.object.spill_range)  in do_dump()
891 			err = do_dump(dscp, range->sru.object.spill_range);  in do_dump()
894 		ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);  in do_dump()
898 		uint64_t epb = BP_GET_LSIZE(&range->sru.object_range.bp) >>  in do_dump()
900 		uint64_t firstobj = range->start_blkid * epb;  in do_dump()
901 		err = dump_object_range(dscp, &range->sru.object_range.bp,  in do_dump()
906 		struct srr *srrp = &range->sru.redact;  in do_dump()
907 		err = dump_redact(dscp, range->object, range->start_blkid *  in do_dump()
908 		    srrp->datablksz, (range->end_blkid - range->start_blkid) *  in do_dump()
913 		struct srd *srdp = &range->sru.data;  in do_dump()
919 		ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);  in do_dump()
922 			err = dump_write_embedded(dscp, range->object,  in do_dump()
923 			    range->start_blkid * srdp->datablksz,  in do_dump()
927 		ASSERT(range->object > dscp->dsc_resume_object ||  in do_dump()
928 		    (range->object == dscp->dsc_resume_object &&  in do_dump()
929 		    (range->start_blkid == DMU_SPILL_BLKID ||  in do_dump()
930 		    range->start_blkid * srdp->datablksz >=  in do_dump()
970 			ASSERT3U(range->start_blkid, ==, DMU_SPILL_BLKID);  in do_dump()
971 			err = dump_spill(dscp, bp, range->object, data);  in do_dump()
975 		uint64_t offset = range->start_blkid * srdp->datablksz;  in do_dump()
989 				    range->object, offset, n, n, NULL, B_FALSE,  in do_dump()
1003 			    range->object, offset,  in do_dump()
1010 		struct srh *srhp = &range->sru.hole;  in do_dump()
1011 		if (range->object == DMU_META_DNODE_OBJECT) {  in do_dump()
1013 			uint64_t first_obj = range->start_blkid * span;  in do_dump()
1014 			uint64_t numobj = range->end_blkid * span - first_obj;  in do_dump()
1024 		if (!overflow_multiply(range->start_blkid, srhp->datablksz,  in do_dump()
1030 		if (!overflow_multiply(range->end_blkid, srhp->datablksz, &len))  in do_dump()
1033 		return (dump_free(dscp, range->object, offset, len));  in do_dump()
1036 		panic("Invalid range type in do_dump: %d", range->type);  in do_dump()
1045 	struct send_range *range = kmem_alloc(sizeof (*range), KM_SLEEP);  in range_alloc()  local
1046 	range->type = type;  in range_alloc()
1047 	range->object = object;  in range_alloc()
1048 	range->start_blkid = start_blkid;  in range_alloc()
1049 	range->end_blkid = end_blkid;  in range_alloc()
1050 	range->eos_marker = eos;  in range_alloc()
1052 		range->sru.data.abd = NULL;  in range_alloc()
1053 		range->sru.data.abuf = NULL;  in range_alloc()
1054 		mutex_init(&range->sru.data.lock, NULL, MUTEX_DEFAULT, NULL);  in range_alloc()
1055 		cv_init(&range->sru.data.cv, NULL, CV_DEFAULT, NULL);  in range_alloc()
1056 		range->sru.data.io_outstanding = 0;  in range_alloc()
1057 		range->sru.data.io_err = 0;  in range_alloc()
1058 		range->sru.data.io_compressed = B_FALSE;  in range_alloc()
1060 		range->sru.object.spill_range = NULL;  in range_alloc()
1062 	return (range);  in range_alloc()
1512 	struct send_range *range;  in send_merge_thread()  local
1513 	for (range = find_next_range(front_ranges, queues, &mask);  in send_merge_thread()
1514 	    !range->eos_marker && err == 0 && !smt_arg->cancel;  in send_merge_thread()
1515 	    range = find_next_range(front_ranges, queues, &mask)) {  in send_merge_thread()
1523 			ASSERT3U(range->type, ==, REDACT);  in send_merge_thread()
1524 			range_free(range);  in send_merge_thread()
1527 		bqueue_enqueue(&smt_arg->q, range, sizeof (*range));  in send_merge_thread()
1554 	range->eos_marker = B_TRUE;  in send_merge_thread()
1555 	bqueue_enqueue_flush(&smt_arg->q, range, 1);  in send_merge_thread()
1572 	struct send_range *range = zio->io_private;  in dmu_send_read_done()  local
1574 	mutex_enter(&range->sru.data.lock);  in dmu_send_read_done()
1576 		abd_free(range->sru.data.abd);  in dmu_send_read_done()
1577 		range->sru.data.abd = NULL;  in dmu_send_read_done()
1578 		range->sru.data.io_err = zio->io_error;  in dmu_send_read_done()
1581 	ASSERT(range->sru.data.io_outstanding);  in dmu_send_read_done()
1582 	range->sru.data.io_outstanding = B_FALSE;  in dmu_send_read_done()
1583 	cv_broadcast(&range->sru.data.cv);  in dmu_send_read_done()
1584 	mutex_exit(&range->sru.data.lock);  in dmu_send_read_done()
1588 issue_data_read(struct send_reader_thread_arg *srta, struct send_range *range)  in issue_data_read()  argument
1590 	struct srd *srdp = &range->sru.data;  in issue_data_read()
1594 	ASSERT3U(range->type, ==, DATA);  in issue_data_read()
1595 	ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);  in issue_data_read()
1642 	    .zb_object = range->object,  in issue_data_read()
1644 	    .zb_blkid = range->start_blkid,  in issue_data_read()
1662 		    srdp->datasz, dmu_send_read_done, range,  in issue_data_read()
1677 	struct send_range *range = range_alloc(range_type, dn->dn_object,  in enqueue_range()  local
1687 		range->sru.hole.datablksz = datablksz;  in enqueue_range()
1691 		range->sru.data.datablksz = datablksz;  in enqueue_range()
1692 		range->sru.data.obj_type = dn->dn_type;  in enqueue_range()
1693 		range->sru.data.bp = *bp;  in enqueue_range()
1694 		issue_data_read(srta, range);  in enqueue_range()
1697 		range->sru.redact.datablksz = datablksz;  in enqueue_range()
1702 	bqueue_enqueue(q, range, datablksz);  in enqueue_range()
1719     struct send_range *range)  in piggyback_unmodified_spill()  argument
1721 	ASSERT3U(range->type, ==, OBJECT);  in piggyback_unmodified_spill()
1723 	dnode_phys_t *dnp = range->sru.object.dnp;  in piggyback_unmodified_spill()
1732 	struct send_range *spill_range = range_alloc(DATA, range->object,  in piggyback_unmodified_spill()
1739 	range->sru.object.spill_range = spill_range;  in piggyback_unmodified_spill()
1760 	struct send_range *range = bqueue_dequeue(inq);  in send_reader_thread()  local
1773 	while (!range->eos_marker && !srta->cancel && smta->error == 0 &&  in send_reader_thread()
1776 		switch (range->type) {  in send_reader_thread()
1778 			issue_data_read(srta, range);  in send_reader_thread()
1779 			bqueue_enqueue(outq, range, range->sru.data.datablksz);  in send_reader_thread()
1780 			range = get_next_range_nofree(inq, range);  in send_reader_thread()
1783 			spill = piggyback_unmodified_spill(srta, range);  in send_reader_thread()
1788 			bqueue_enqueue(outq, range, sizeof (*range) + spill);  in send_reader_thread()
1789 			range = get_next_range_nofree(inq, range);  in send_reader_thread()
1816 			if (range->object == last_obj && !last_obj_exists) {  in send_reader_thread()
1824 				err = dnode_hold(os, range->object, FTAG, &dn);  in send_reader_thread()
1829 				last_obj = range->object;  in send_reader_thread()
1842 				range = get_next_range(inq, range);  in send_reader_thread()
1846 			    MIN(dn->dn_maxblkid, range->end_blkid);  in send_reader_thread()
1852 			for (uint64_t blkid = range->start_blkid;  in send_reader_thread()
1900 			range = get_next_range(inq, range);  in send_reader_thread()
1910 	while (!range->eos_marker)  in send_reader_thread()
1911 		range = get_next_range(inq, range);  in send_reader_thread()
1913 	bqueue_enqueue_flush(outq, range, 1);  in send_reader_thread()
2376 	struct send_range *range;  in dmu_send_impl()  local
2568 	range = bqueue_dequeue(&srt_arg->q);  in dmu_send_impl()
2569 	while (err == 0 && !range->eos_marker) {  in dmu_send_impl()
2570 		err = do_dump(&dsc, range);  in dmu_send_impl()
2571 		range = get_next_range(&srt_arg->q, range);  in dmu_send_impl()
2584 		while (!range->eos_marker) {  in dmu_send_impl()
2585 			range = get_next_range(&srt_arg->q, range);  in dmu_send_impl()
2588 	range_free(range);  in dmu_send_impl()