182 	inject_handler_t *handler;  in zio_handle_panic_injection()  local
186 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_handle_panic_injection()
187 	    handler = list_next(&inject_handlers, handler)) {  in zio_handle_panic_injection()
189 		if (spa != handler->zi_spa)  in zio_handle_panic_injection()
192 		if (handler->zi_record.zi_type == type &&  in zio_handle_panic_injection()
193 		    strcmp(tag, handler->zi_record.zi_func) == 0) {  in zio_handle_panic_injection()
194 			handler->zi_record.zi_match_count++;  in zio_handle_panic_injection()
195 			handler->zi_record.zi_inject_count++;  in zio_handle_panic_injection()
212 	inject_handler_t *handler;  in zio_handle_decrypt_injection()  local
216 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_handle_decrypt_injection()
217 	    handler = list_next(&inject_handlers, handler)) {  in zio_handle_decrypt_injection()
219 		if (spa != handler->zi_spa ||  in zio_handle_decrypt_injection()
220 		    handler->zi_record.zi_cmd != ZINJECT_DECRYPT_FAULT)  in zio_handle_decrypt_injection()
224 		    &handler->zi_record, error)) {  in zio_handle_decrypt_injection()
273 	inject_handler_t *handler;  in zio_handle_fault_injection()  local
295 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_handle_fault_injection()
296 	    handler = list_next(&inject_handlers, handler)) {  in zio_handle_fault_injection()
297 		if (zio->io_spa != handler->zi_spa ||  in zio_handle_fault_injection()
298 		    handler->zi_record.zi_cmd != ZINJECT_DATA_FAULT)  in zio_handle_fault_injection()
304 		    zio_match_dva(zio), &handler->zi_record, error)) {  in zio_handle_fault_injection()
324 	inject_handler_t *handler;  in zio_handle_label_injection()  local
336 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_handle_label_injection()
337 	    handler = list_next(&inject_handlers, handler)) {  in zio_handle_label_injection()
338 		uint64_t start = handler->zi_record.zi_start;  in zio_handle_label_injection()
339 		uint64_t end = handler->zi_record.zi_end;  in zio_handle_label_injection()
341 		if (handler->zi_record.zi_cmd != ZINJECT_LABEL_FAULT)  in zio_handle_label_injection()
353 		if (zio->io_vd->vdev_guid == handler->zi_record.zi_guid &&  in zio_handle_label_injection()
355 			handler->zi_record.zi_match_count++;  in zio_handle_label_injection()
356 			handler->zi_record.zi_inject_count++;  in zio_handle_label_injection()
408 	inject_handler_t *handler;  in zio_handle_device_injection_impl()  local
427 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_handle_device_injection_impl()
428 	    handler = list_next(&inject_handlers, handler)) {  in zio_handle_device_injection_impl()
430 		if (handler->zi_record.zi_cmd != ZINJECT_DEVICE_FAULT)  in zio_handle_device_injection_impl()
433 		if (vd->vdev_guid == handler->zi_record.zi_guid) {  in zio_handle_device_injection_impl()
434 			if (handler->zi_record.zi_failfast &&  in zio_handle_device_injection_impl()
442 			    handler->zi_record.zi_iotype))  in zio_handle_device_injection_impl()
445 			if (handler->zi_record.zi_error == err1 ||  in zio_handle_device_injection_impl()
446 			    handler->zi_record.zi_error == err2) {  in zio_handle_device_injection_impl()
447 				handler->zi_record.zi_match_count++;  in zio_handle_device_injection_impl()
452 				if (!freq_triggered(handler->zi_record.zi_freq))  in zio_handle_device_injection_impl()
455 				handler->zi_record.zi_inject_count++;  in zio_handle_device_injection_impl()
470 				if (!handler->zi_record.zi_failfast &&  in zio_handle_device_injection_impl()
477 				if (handler->zi_record.zi_error == EILSEQ) {  in zio_handle_device_injection_impl()
488 				ret = handler->zi_record.zi_error;  in zio_handle_device_injection_impl()
491 			if (handler->zi_record.zi_error == ENXIO) {  in zio_handle_device_injection_impl()
492 				handler->zi_record.zi_match_count++;  in zio_handle_device_injection_impl()
493 				handler->zi_record.zi_inject_count++;  in zio_handle_device_injection_impl()
524 	inject_handler_t *handler;  in zio_handle_ignored_writes()  local
528 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_handle_ignored_writes()
529 	    handler = list_next(&inject_handlers, handler)) {  in zio_handle_ignored_writes()
532 		if (zio->io_spa != handler->zi_spa ||  in zio_handle_ignored_writes()
533 		    handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)  in zio_handle_ignored_writes()
536 		handler->zi_record.zi_match_count++;  in zio_handle_ignored_writes()
542 		if (handler->zi_record.zi_timer == 0) {  in zio_handle_ignored_writes()
543 			if (handler->zi_record.zi_duration > 0)  in zio_handle_ignored_writes()
544 				handler->zi_record.zi_timer = ddi_get_lbolt64();  in zio_handle_ignored_writes()
546 				handler->zi_record.zi_timer = zio->io_txg;  in zio_handle_ignored_writes()
551 			handler->zi_record.zi_inject_count++;  in zio_handle_ignored_writes()
563 	inject_handler_t *handler;  in spa_handle_ignored_writes()  local
570 	for (handler = list_head(&inject_handlers); handler != NULL;  in spa_handle_ignored_writes()
571 	    handler = list_next(&inject_handlers, handler)) {  in spa_handle_ignored_writes()
573 		if (spa != handler->zi_spa ||  in spa_handle_ignored_writes()
574 		    handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)  in spa_handle_ignored_writes()
577 		handler->zi_record.zi_match_count++;  in spa_handle_ignored_writes()
578 		handler->zi_record.zi_inject_count++;  in spa_handle_ignored_writes()
580 		if (handler->zi_record.zi_duration > 0) {  in spa_handle_ignored_writes()
581 			VERIFY(handler->zi_record.zi_timer == 0 ||  in spa_handle_ignored_writes()
583 			    (int64_t)handler->zi_record.zi_timer +  in spa_handle_ignored_writes()
584 			    handler->zi_record.zi_duration * hz,  in spa_handle_ignored_writes()
588 			VERIFY(handler->zi_record.zi_timer == 0 ||  in spa_handle_ignored_writes()
589 			    handler->zi_record.zi_timer -  in spa_handle_ignored_writes()
590 			    handler->zi_record.zi_duration >=  in spa_handle_ignored_writes()
656 	for (inject_handler_t *handler = list_head(&inject_handlers);  in zio_handle_io_delay()  local
657 	    handler != NULL; handler = list_next(&inject_handlers, handler)) {  in zio_handle_io_delay()
658 		if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)  in zio_handle_io_delay()
661 		if (vd->vdev_guid != handler->zi_record.zi_guid)  in zio_handle_io_delay()
665 		if (!zio_match_iotype(zio, handler->zi_record.zi_iotype))  in zio_handle_io_delay()
672 		ASSERT3P(handler->zi_lanes, !=, NULL);  in zio_handle_io_delay()
678 		ASSERT3U(handler->zi_record.zi_nlanes, !=, 0);  in zio_handle_io_delay()
680 		ASSERT3U(handler->zi_record.zi_nlanes, >,  in zio_handle_io_delay()
681 		    handler->zi_next_lane);  in zio_handle_io_delay()
683 		handler->zi_record.zi_match_count++;  in zio_handle_io_delay()
686 		if (!freq_triggered(handler->zi_record.zi_freq))  in zio_handle_io_delay()
715 		hrtime_t idle = handler->zi_record.zi_timer + gethrtime();  in zio_handle_io_delay()
716 		hrtime_t busy = handler->zi_record.zi_timer +  in zio_handle_io_delay()
717 		    handler->zi_lanes[handler->zi_next_lane];  in zio_handle_io_delay()
721 			min_handler = handler;  in zio_handle_io_delay()
739 			min_handler = handler;  in zio_handle_io_delay()
774 	inject_handler_t *handler;  in zio_handle_pool_delay()  local
780 	for (handler = list_head(&inject_handlers);  in zio_handle_pool_delay()
781 	    handler != NULL && handler->zi_record.zi_cmd == command;  in zio_handle_pool_delay()
782 	    handler = list_next(&inject_handlers, handler)) {  in zio_handle_pool_delay()
783 		ASSERT3P(handler->zi_spa_name, !=, NULL);  in zio_handle_pool_delay()
784 		if (strcmp(spa_name(spa), handler->zi_spa_name) == 0) {  in zio_handle_pool_delay()
785 			handler->zi_record.zi_match_count++;  in zio_handle_pool_delay()
787 			    SEC2NSEC(handler->zi_record.zi_duration);  in zio_handle_pool_delay()
789 				handler->zi_record.zi_inject_count++;  in zio_handle_pool_delay()
792 			id = handler->zi_id;  in zio_handle_pool_delay()
893 	for (inject_handler_t *handler = list_head(&inject_handlers);  in zio_pool_handler_exists()  local
894 	    handler != NULL; handler = list_next(&inject_handlers, handler)) {  in zio_pool_handler_exists()
895 		if (command != handler->zi_record.zi_cmd)  in zio_pool_handler_exists()
898 		const char *pool = (handler->zi_spa_name != NULL) ?  in zio_pool_handler_exists()
899 		    handler->zi_spa_name : spa_name(handler->zi_spa);  in zio_pool_handler_exists()
917 	inject_handler_t *handler;  in zio_inject_fault()  local
993 		handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP);  in zio_inject_fault()
994 		handler->zi_spa = spa;	/* note: can be NULL */  in zio_inject_fault()
995 		handler->zi_record = *record;  in zio_inject_fault()
997 		if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_inject_fault()
998 			handler->zi_lanes = kmem_zalloc(  in zio_inject_fault()
999 			    sizeof (*handler->zi_lanes) *  in zio_inject_fault()
1000 			    handler->zi_record.zi_nlanes, KM_SLEEP);  in zio_inject_fault()
1001 			handler->zi_next_lane = 0;  in zio_inject_fault()
1003 			handler->zi_lanes = NULL;  in zio_inject_fault()
1004 			handler->zi_next_lane = 0;  in zio_inject_fault()
1007 		if (handler->zi_spa == NULL)  in zio_inject_fault()
1008 			handler->zi_spa_name = spa_strdup(name);  in zio_inject_fault()
1010 			handler->zi_spa_name = NULL;  in zio_inject_fault()
1020 		if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_inject_fault()
1026 		*id = handler->zi_id = inject_next_id++;  in zio_inject_fault()
1027 		list_insert_tail(&inject_handlers, handler);  in zio_inject_fault()
1057 	inject_handler_t *handler;  in zio_inject_list_next()  local
1063 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_inject_list_next()
1064 	    handler = list_next(&inject_handlers, handler))  in zio_inject_list_next()
1065 		if (handler->zi_id > *id)  in zio_inject_list_next()
1068 	if (handler) {  in zio_inject_list_next()
1069 		*record = handler->zi_record;  in zio_inject_list_next()
1070 		*id = handler->zi_id;  in zio_inject_list_next()
1071 		ASSERT(handler->zi_spa || handler->zi_spa_name);  in zio_inject_list_next()
1072 		if (handler->zi_spa != NULL)  in zio_inject_list_next()
1073 			(void) strlcpy(name, spa_name(handler->zi_spa), buflen);  in zio_inject_list_next()
1075 			(void) strlcpy(name, handler->zi_spa_name, buflen);  in zio_inject_list_next()
1094 	inject_handler_t *handler;  in zio_clear_fault()  local
1098 	for (handler = list_head(&inject_handlers); handler != NULL;  in zio_clear_fault()
1099 	    handler = list_next(&inject_handlers, handler))  in zio_clear_fault()
1100 		if (handler->zi_id == id)  in zio_clear_fault()
1103 	if (handler == NULL) {  in zio_clear_fault()
1108 	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_clear_fault()
1114 	list_remove(&inject_handlers, handler);  in zio_clear_fault()
1117 	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_clear_fault()
1118 		ASSERT3P(handler->zi_lanes, !=, NULL);  in zio_clear_fault()
1119 		kmem_free(handler->zi_lanes, sizeof (*handler->zi_lanes) *  in zio_clear_fault()
1120 		    handler->zi_record.zi_nlanes);  in zio_clear_fault()
1122 		ASSERT3P(handler->zi_lanes, ==, NULL);  in zio_clear_fault()
1125 	if (handler->zi_spa_name != NULL)  in zio_clear_fault()
1126 		spa_strfree(handler->zi_spa_name);  in zio_clear_fault()
1128 	if (handler->zi_spa != NULL)  in zio_clear_fault()
1129 		spa_inject_delref(handler->zi_spa);  in zio_clear_fault()
1130 	kmem_free(handler, sizeof (inject_handler_t));  in zio_clear_fault()