71 static int traverse_dnode(traverse_data_t *td, const blkptr_t *bp,
73 static void prefetch_dnode_metadata(traverse_data_t *td, const dnode_phys_t *,
80 	traverse_data_t *td = arg;  in traverse_zil_block()  local
87 	    BP_GET_LOGICAL_BIRTH(bp) >= spa_min_claim_txg(td->td_spa))  in traverse_zil_block()
90 	SET_BOOKMARK(&zb, td->td_objset, ZB_ZIL_OBJECT, ZB_ZIL_LEVEL,  in traverse_zil_block()
93 	(void) td->td_func(td->td_spa, zilog, bp, &zb, NULL, td->td_arg);  in traverse_zil_block()
102 	traverse_data_t *td = arg;  in traverse_zil_record()  local
116 		SET_BOOKMARK(&zb, td->td_objset, lr->lr_foid,  in traverse_zil_record()
119 		(void) td->td_func(td->td_spa, zilog, bp, &zb, NULL,  in traverse_zil_record()
120 		    td->td_arg);  in traverse_zil_record()
126 traverse_zil(traverse_data_t *td, zil_header_t *zh)  in traverse_zil()  argument
134 	if (claim_txg == 0 && spa_writeable(td->td_spa))  in traverse_zil()
137 	zilog_t *zilog = zil_alloc(spa_get_dsl(td->td_spa)->dp_meta_objset, zh);  in traverse_zil()
138 	(void) zil_parse(zilog, traverse_zil_block, traverse_zil_record, td,  in traverse_zil()
139 	    claim_txg, !(td->td_flags & TRAVERSE_NO_DECRYPT));  in traverse_zil()
158 resume_skip_check(const traverse_data_t *td, const dnode_phys_t *dnp,  in resume_skip_check()  argument
161 	if (td->td_resume != NULL) {  in resume_skip_check()
166 		if (zbookmark_subtree_completed(dnp, zb, td->td_resume))  in resume_skip_check()
169 		if (memcmp(zb, td->td_resume, sizeof (*zb)) == 0) {  in resume_skip_check()
170 			if (td->td_flags & TRAVERSE_POST)  in resume_skip_check()
181 traverse_prefetch_metadata(traverse_data_t *td, const dnode_phys_t *dnp,  in traverse_prefetch_metadata()  argument
188 	if (!(td->td_flags & TRAVERSE_PREFETCH_METADATA))  in traverse_prefetch_metadata()
194 	if (resume_skip_check(td, dnp, zb) != RESUME_SKIP_NONE)  in traverse_prefetch_metadata()
196 	if (BP_IS_HOLE(bp) || BP_GET_LOGICAL_BIRTH(bp) <= td->td_min_txg)  in traverse_prefetch_metadata()
202 	if ((td->td_flags & TRAVERSE_NO_DECRYPT) && BP_IS_PROTECTED(bp))  in traverse_prefetch_metadata()
205 	(void) arc_read(NULL, td->td_spa, bp, NULL, NULL,  in traverse_prefetch_metadata()
221 traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,  in traverse_visitbp()  argument
226 	prefetch_data_t *pd = td->td_pfd;  in traverse_visitbp()
228 	switch (resume_skip_check(td, dnp, zb)) {  in traverse_visitbp()
263 		    (!td->td_realloc_possible ||  in traverse_visitbp()
265 		    td->td_hole_birth_enabled_txg <= td->td_min_txg)  in traverse_visitbp()
267 	} else if (BP_GET_LOGICAL_BIRTH(bp) <= td->td_min_txg) {  in traverse_visitbp()
283 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);  in traverse_visitbp()
289 	if (td->td_flags & TRAVERSE_PRE) {  in traverse_visitbp()
290 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp,  in traverse_visitbp()
291 		    td->td_arg);  in traverse_visitbp()
307 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,  in traverse_visitbp()
342 					if (traverse_prefetch_metadata(td, dnp,  in traverse_visitbp()
357 			err = traverse_visitbp(td, dnp,  in traverse_visitbp()
376 		if ((td->td_flags & TRAVERSE_NO_DECRYPT) && BP_IS_PROTECTED(bp))  in traverse_visitbp()
379 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,  in traverse_visitbp()
387 			prefetch_dnode_metadata(td, &child_dnp[i],  in traverse_visitbp()
393 			err = traverse_dnode(td, bp, &child_dnp[i],  in traverse_visitbp()
403 		if ((td->td_flags & TRAVERSE_NO_DECRYPT) && BP_IS_PROTECTED(bp))  in traverse_visitbp()
406 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,  in traverse_visitbp()
412 		prefetch_dnode_metadata(td, &osp->os_meta_dnode, zb->zb_objset,  in traverse_visitbp()
421 			td->td_realloc_possible = B_FALSE;  in traverse_visitbp()
425 				prefetch_dnode_metadata(td,  in traverse_visitbp()
428 			prefetch_dnode_metadata(td, &osp->os_groupused_dnode,  in traverse_visitbp()
430 			prefetch_dnode_metadata(td, &osp->os_userused_dnode,  in traverse_visitbp()
434 		err = traverse_dnode(td, bp, &osp->os_meta_dnode, zb->zb_objset,  in traverse_visitbp()
438 				err = traverse_dnode(td, bp,  in traverse_visitbp()
442 				err = traverse_dnode(td, bp,  in traverse_visitbp()
446 				err = traverse_dnode(td, bp,  in traverse_visitbp()
456 	if (err == 0 && (td->td_flags & TRAVERSE_POST))  in traverse_visitbp()
457 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);  in traverse_visitbp()
459 	if ((td->td_flags & TRAVERSE_HARD) && (err == EIO || err == ECKSUM)) {  in traverse_visitbp()
470 	if (td->td_resume != NULL && err != 0 && !td->td_paused) {  in traverse_visitbp()
471 		td->td_resume->zb_objset = zb->zb_objset;  in traverse_visitbp()
472 		td->td_resume->zb_object = zb->zb_object;  in traverse_visitbp()
473 		td->td_resume->zb_level = 0;  in traverse_visitbp()
484 		td->td_resume->zb_blkid = zb->zb_blkid;  in traverse_visitbp()
486 			td->td_resume->zb_blkid <<= zb->zb_level *  in traverse_visitbp()
489 		td->td_paused = B_TRUE;  in traverse_visitbp()
496 prefetch_dnode_metadata(traverse_data_t *td, const dnode_phys_t *dnp,  in prefetch_dnode_metadata()  argument
504 		traverse_prefetch_metadata(td, dnp, &dnp->dn_blkptr[j], &czb);  in prefetch_dnode_metadata()
509 		traverse_prefetch_metadata(td, dnp, DN_SPILL_BLKPTR(dnp), &czb);  in prefetch_dnode_metadata()
514 traverse_dnode(traverse_data_t *td, const blkptr_t *bp, const dnode_phys_t *dnp,  in traverse_dnode()  argument
520 	if (object != DMU_META_DNODE_OBJECT && td->td_resume != NULL &&  in traverse_dnode()
521 	    object < td->td_resume->zb_object)  in traverse_dnode()
524 	if (td->td_flags & TRAVERSE_PRE) {  in traverse_dnode()
527 		err = td->td_func(td->td_spa, NULL, bp, &czb, dnp,  in traverse_dnode()
528 		    td->td_arg);  in traverse_dnode()
537 		err = traverse_visitbp(td, dnp, &dnp->dn_blkptr[j], &czb);  in traverse_dnode()
544 		err = traverse_visitbp(td, dnp, DN_SPILL_BLKPTR(dnp), &czb);  in traverse_dnode()
547 	if (err == 0 && (td->td_flags & TRAVERSE_POST)) {  in traverse_dnode()
550 		err = td->td_func(td->td_spa, NULL, bp, &czb, dnp,  in traverse_dnode()
551 		    td->td_arg);  in traverse_dnode()
599 	traverse_data_t td = *td_main;  in traverse_prefetch_thread()  local
603 	td.td_func = traverse_prefetcher;  in traverse_prefetch_thread()
604 	td.td_arg = td_main->td_pfd;  in traverse_prefetch_thread()
605 	td.td_pfd = NULL;  in traverse_prefetch_thread()
606 	td.td_resume = &td_main->td_pfd->pd_resume;  in traverse_prefetch_thread()
608 	SET_BOOKMARK(&czb, td.td_objset,  in traverse_prefetch_thread()
610 	(void) traverse_visitbp(&td, NULL, td.td_rootbp, &czb);  in traverse_prefetch_thread()
628 	traverse_data_t *td;  in traverse_impl()  local
636 	td = kmem_alloc(sizeof (traverse_data_t), KM_SLEEP);  in traverse_impl()
640 	td->td_spa = spa;  in traverse_impl()
641 	td->td_objset = objset;  in traverse_impl()
642 	td->td_rootbp = rootbp;  in traverse_impl()
643 	td->td_min_txg = txg_start;  in traverse_impl()
644 	td->td_resume = resume;  in traverse_impl()
645 	td->td_func = func;  in traverse_impl()
646 	td->td_arg = arg;  in traverse_impl()
647 	td->td_pfd = pd;  in traverse_impl()
648 	td->td_flags = flags;  in traverse_impl()
649 	td->td_paused = B_FALSE;  in traverse_impl()
650 	td->td_realloc_possible = (txg_start == 0 ? B_FALSE : B_TRUE);  in traverse_impl()
654 		    SPA_FEATURE_HOLE_BIRTH, &td->td_hole_birth_enabled_txg));  in traverse_impl()
656 		td->td_hole_birth_enabled_txg = UINT64_MAX;  in traverse_impl()
665 	SET_BOOKMARK(czb, td->td_objset,  in traverse_impl()
676 		if ((td->td_flags & TRAVERSE_NO_DECRYPT) &&  in traverse_impl()
680 		err = arc_read(NULL, td->td_spa, rootbp, arc_getbuf_func,  in traverse_impl()
688 			if (!(td->td_flags & TRAVERSE_HARD) ||  in traverse_impl()
689 			    !(td->td_flags & TRAVERSE_PRE))  in traverse_impl()
693 			traverse_zil(td, &osp->os_zil_header);  in traverse_impl()
700 	    td, TQ_NOQUEUE) == TASKQID_INVALID)  in traverse_impl()
703 	err = traverse_visitbp(td, NULL, rootbp, czb);  in traverse_impl()
717 	kmem_free(td, sizeof (struct traverse_data));  in traverse_impl()