1 // SPDX-License-Identifier: CDDL-1.0
10  * or https://opensource.org/licenses/CDDL-1.0.
26  * Copyright (c) 2024-2025, Klara, Inc.
139 	if (zb->zb_objset == DMU_META_OBJSET &&  in zio_match_handler()
140 	    record->zi_objset == DMU_META_OBJSET &&  in zio_match_handler()
141 	    record->zi_object == DMU_META_DNODE_OBJECT) {  in zio_match_handler()
142 		if (record->zi_type == DMU_OT_NONE ||  in zio_match_handler()
143 		    type == record->zi_type)  in zio_match_handler()
151 	if (zb->zb_objset == record->zi_objset &&  in zio_match_handler()
152 	    zb->zb_object == record->zi_object &&  in zio_match_handler()
153 	    zb->zb_level == record->zi_level &&  in zio_match_handler()
154 	    zb->zb_blkid >= record->zi_start &&  in zio_match_handler()
155 	    zb->zb_blkid <= record->zi_end &&  in zio_match_handler()
156 	    (record->zi_dvas == 0 ||  in zio_match_handler()
157 	    (dva != ZI_NO_DVA && (record->zi_dvas & (1ULL << dva)))) &&  in zio_match_handler()
158 	    error == record->zi_error) {  in zio_match_handler()
165 		record->zi_match_count++;  in zio_match_handler()
166 		injected = freq_triggered(record->zi_freq);  in zio_match_handler()
170 		record->zi_inject_count++;  in zio_match_handler()
189 		if (spa != handler->zi_spa)  in zio_handle_panic_injection()
192 		if (handler->zi_record.zi_type == type &&  in zio_handle_panic_injection()
193 		    strcmp(tag, handler->zi_record.zi_func) == 0) {  in zio_handle_panic_injection()
194 			handler->zi_record.zi_match_count++;  in zio_handle_panic_injection()
195 			handler->zi_record.zi_inject_count++;  in zio_handle_panic_injection()
219 		if (spa != handler->zi_spa ||  in zio_handle_decrypt_injection()
220 		    handler->zi_record.zi_cmd != ZINJECT_DECRYPT_FAULT)  in zio_handle_decrypt_injection()
224 		    &handler->zi_record, error)) {  in zio_handle_decrypt_injection()
237  * that we end up with ZI_NO_DVA (-1) if we don't find a match.
244 	if (zio->io_bp != NULL && zio->io_vd != NULL &&  in zio_match_dva()
245 	    zio->io_child_type == ZIO_CHILD_VDEV) {  in zio_match_dva()
246 		for (i = BP_GET_NDVAS(zio->io_bp) - 1; i >= 0; i--) {  in zio_match_dva()
247 			dva_t *dva = &zio->io_bp->blk_dva[i];  in zio_match_dva()
249 			vdev_t *vd = vdev_lookup_top(zio->io_spa,  in zio_match_dva()
253 			if (zio->io_vd->vdev_ops->vdev_op_leaf)  in zio_match_dva()
256 			if (zio->io_vd == vd && zio->io_offset == off)  in zio_match_dva()
278 	if (zio->io_logical == NULL)  in zio_handle_fault_injection()
284 	if (zio->io_type != ZIO_TYPE_READ)  in zio_handle_fault_injection()
290 	if (zio->io_priority == ZIO_PRIORITY_REBUILD && error == ECKSUM)  in zio_handle_fault_injection()
297 		if (zio->io_spa != handler->zi_spa ||  in zio_handle_fault_injection()
298 		    handler->zi_record.zi_cmd != ZINJECT_DATA_FAULT)  in zio_handle_fault_injection()
302 		if (zio_match_handler(&zio->io_logical->io_bookmark,  in zio_handle_fault_injection()
303 		    zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE,  in zio_handle_fault_injection()
304 		    zio_match_dva(zio), &handler->zi_record, error)) {  in zio_handle_fault_injection()
325 	vdev_t *vd = zio->io_vd;  in zio_handle_label_injection()
326 	uint64_t offset = zio->io_offset;  in zio_handle_label_injection()
331 	    offset < vd->vdev_psize - VDEV_LABEL_END_SIZE)  in zio_handle_label_injection()
338 		uint64_t start = handler->zi_record.zi_start;  in zio_handle_label_injection()
339 		uint64_t end = handler->zi_record.zi_end;  in zio_handle_label_injection()
341 		if (handler->zi_record.zi_cmd != ZINJECT_LABEL_FAULT)  in zio_handle_label_injection()
349 		label = vdev_label_number(vd->vdev_psize, offset);  in zio_handle_label_injection()
350 		start = vdev_label_offset(vd->vdev_psize, label, start);  in zio_handle_label_injection()
351 		end = vdev_label_offset(vd->vdev_psize, label, end);  in zio_handle_label_injection()
353 		if (zio->io_vd->vdev_guid == handler->zi_record.zi_guid &&  in zio_handle_label_injection()
355 			handler->zi_record.zi_match_count++;  in zio_handle_label_injection()
356 			handler->zi_record.zi_inject_count++;  in zio_handle_label_injection()
372 	ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);  in zio_inject_bitflip_cb()
391 	if (zio->io_flags & ZIO_FLAG_PROBE)  in zio_match_iotype()
396 		return (iotype == zio->io_type);  in zio_match_iotype()
416 	if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH &&  in zio_handle_device_injection_impl()
417 	    !(zio->io_flags & ZIO_FLAG_PROBE)) {  in zio_handle_device_injection_impl()
418 		uint64_t offset = zio->io_offset;  in zio_handle_device_injection_impl()
421 		    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE)  in zio_handle_device_injection_impl()
430 		if (handler->zi_record.zi_cmd != ZINJECT_DEVICE_FAULT)  in zio_handle_device_injection_impl()
433 		if (vd->vdev_guid == handler->zi_record.zi_guid) {  in zio_handle_device_injection_impl()
434 			if (handler->zi_record.zi_failfast &&  in zio_handle_device_injection_impl()
435 			    (zio == NULL || (zio->io_flags &  in zio_handle_device_injection_impl()
442 			    handler->zi_record.zi_iotype))  in zio_handle_device_injection_impl()
445 			if (handler->zi_record.zi_error == err1 ||  in zio_handle_device_injection_impl()
446 			    handler->zi_record.zi_error == err2) {  in zio_handle_device_injection_impl()
447 				handler->zi_record.zi_match_count++;  in zio_handle_device_injection_impl()
452 				if (!freq_triggered(handler->zi_record.zi_freq))  in zio_handle_device_injection_impl()
455 				handler->zi_record.zi_inject_count++;  in zio_handle_device_injection_impl()
462 					vd->vdev_stat.vs_aux =  in zio_handle_device_injection_impl()
470 				if (!handler->zi_record.zi_failfast &&  in zio_handle_device_injection_impl()
472 					zio->io_flags |= ZIO_FLAG_IO_RETRY;  in zio_handle_device_injection_impl()
477 				if (handler->zi_record.zi_error == EILSEQ) {  in zio_handle_device_injection_impl()
482 					(void) abd_iterate_func(zio->io_abd, 0,  in zio_handle_device_injection_impl()
483 					    zio->io_size, zio_inject_bitflip_cb,  in zio_handle_device_injection_impl()
488 				ret = handler->zi_record.zi_error;  in zio_handle_device_injection_impl()
491 			if (handler->zi_record.zi_error == ENXIO) {  in zio_handle_device_injection_impl()
492 				handler->zi_record.zi_match_count++;  in zio_handle_device_injection_impl()
493 				handler->zi_record.zi_inject_count++;  in zio_handle_device_injection_impl()
532 		if (zio->io_spa != handler->zi_spa ||  in zio_handle_ignored_writes()
533 		    handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)  in zio_handle_ignored_writes()
536 		handler->zi_record.zi_match_count++;  in zio_handle_ignored_writes()
542 		if (handler->zi_record.zi_timer == 0) {  in zio_handle_ignored_writes()
543 			if (handler->zi_record.zi_duration > 0)  in zio_handle_ignored_writes()
544 				handler->zi_record.zi_timer = ddi_get_lbolt64();  in zio_handle_ignored_writes()
546 				handler->zi_record.zi_timer = zio->io_txg;  in zio_handle_ignored_writes()
551 			handler->zi_record.zi_inject_count++;  in zio_handle_ignored_writes()
552 			zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;  in zio_handle_ignored_writes()
573 		if (spa != handler->zi_spa ||  in spa_handle_ignored_writes()
574 		    handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)  in spa_handle_ignored_writes()
577 		handler->zi_record.zi_match_count++;  in spa_handle_ignored_writes()
578 		handler->zi_record.zi_inject_count++;  in spa_handle_ignored_writes()
580 		if (handler->zi_record.zi_duration > 0) {  in spa_handle_ignored_writes()
581 			VERIFY(handler->zi_record.zi_timer == 0 ||  in spa_handle_ignored_writes()
583 			    (int64_t)handler->zi_record.zi_timer +  in spa_handle_ignored_writes()
584 			    handler->zi_record.zi_duration * hz,  in spa_handle_ignored_writes()
588 			VERIFY(handler->zi_record.zi_timer == 0 ||  in spa_handle_ignored_writes()
589 			    handler->zi_record.zi_timer -  in spa_handle_ignored_writes()
590 			    handler->zi_record.zi_duration >=  in spa_handle_ignored_writes()
601 	vdev_t *vd = zio->io_vd;  in zio_handle_io_delay()
630 	 * it. Each lane is able to handle requests independently of one  in zio_handle_io_delay()
633 	 * a single lane with a 10ms latency, it will delay requests  in zio_handle_io_delay()
635 	 * if more than one request is attempted per each 10ms interval,  in zio_handle_io_delay()
641 	 * threads being assigned to the same lane of a given inject  in zio_handle_io_delay()
647 	 *	2. update that minimum handler's lane array  in zio_handle_io_delay()
650 	 * lane in step (1), and then conflict with each other in step  in zio_handle_io_delay()
651 	 * (2). This could allow a single lane handler to process  in zio_handle_io_delay()
658 		if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)  in zio_handle_io_delay()
661 		if (vd->vdev_guid != handler->zi_record.zi_guid)  in zio_handle_io_delay()
664 		/* also match on I/O type (e.g., -T read) */  in zio_handle_io_delay()
665 		if (!zio_match_iotype(zio, handler->zi_record.zi_iotype))  in zio_handle_io_delay()
672 		ASSERT3P(handler->zi_lanes, !=, NULL);  in zio_handle_io_delay()
678 		ASSERT3U(handler->zi_record.zi_nlanes, !=, 0);  in zio_handle_io_delay()
680 		ASSERT3U(handler->zi_record.zi_nlanes, >,  in zio_handle_io_delay()
681 		    handler->zi_next_lane);  in zio_handle_io_delay()
683 		handler->zi_record.zi_match_count++;  in zio_handle_io_delay()
686 		if (!freq_triggered(handler->zi_record.zi_freq))  in zio_handle_io_delay()
690 		 * We want to issue this IO to the lane that will become  in zio_handle_io_delay()
694 		 * lanes. We then use this lane to submit the request.  in zio_handle_io_delay()
697 		 * delay, we can just use the "next" lane for that  in zio_handle_io_delay()
698 		 * handler; as it will always be the lane with the  in zio_handle_io_delay()
700 		 * lane that will become idle the soonest). This saves a  in zio_handle_io_delay()
705 		 * lane is idle, we want to "submit" the request now so  in zio_handle_io_delay()
709 		 * If the lane is busy, we want this request to complete  in zio_handle_io_delay()
710 		 * zi_timer milliseconds after the lane becomes idle.  in zio_handle_io_delay()
712 		 * each lane will become idle, we use that value to  in zio_handle_io_delay()
715 		hrtime_t idle = handler->zi_record.zi_timer + gethrtime();  in zio_handle_io_delay()
716 		hrtime_t busy = handler->zi_record.zi_timer +  in zio_handle_io_delay()
717 		    handler->zi_lanes[handler->zi_next_lane];  in zio_handle_io_delay()
730 		 * We don't yet increment the "next lane" variable since  in zio_handle_io_delay()
731 		 * we still might find a lower value lane in another  in zio_handle_io_delay()
734 		 * the lane and increment the handler's "next lane"  in zio_handle_io_delay()
747 	 * the lane that will become idle the soonest.  in zio_handle_io_delay()
751 		min_handler->zi_lanes[min_handler->zi_next_lane] = min_target;  in zio_handle_io_delay()
755 		 * loop back and start using the first lane again;  in zio_handle_io_delay()
756 		 * otherwise, just increment the lane index.  in zio_handle_io_delay()
758 		min_handler->zi_next_lane = (min_handler->zi_next_lane + 1) %  in zio_handle_io_delay()
759 		    min_handler->zi_record.zi_nlanes;  in zio_handle_io_delay()
761 		min_handler->zi_record.zi_inject_count++;  in zio_handle_io_delay()
781 	    handler != NULL && handler->zi_record.zi_cmd == command;  in zio_handle_pool_delay()
783 		ASSERT3P(handler->zi_spa_name, !=, NULL);  in zio_handle_pool_delay()
784 		if (strcmp(spa_name(spa), handler->zi_spa_name) == 0) {  in zio_handle_pool_delay()
785 			handler->zi_record.zi_match_count++;  in zio_handle_pool_delay()
787 			    SEC2NSEC(handler->zi_record.zi_duration);  in zio_handle_pool_delay()
789 				handler->zi_record.zi_inject_count++;  in zio_handle_pool_delay()
790 				delay = pause - elapsed;  in zio_handle_pool_delay()
792 			id = handler->zi_id;  in zio_handle_pool_delay()
807 		/* all done with this one-shot handler */  in zio_handle_pool_delay()
846 	error = dsl_dataset_hold_obj(dp, record->zi_objset, FTAG, &ds);  in zio_calculate_range()
856 	error = dnode_hold(os, record->zi_object, FTAG, &dn);  in zio_calculate_range()
863 	if (record->zi_start != 0 || record->zi_end != -1ULL) {  in zio_calculate_range()
864 		record->zi_start >>= dn->dn_datablkshift;  in zio_calculate_range()
865 		record->zi_end >>= dn->dn_datablkshift;  in zio_calculate_range()
867 	if (record->zi_level > 0) {  in zio_calculate_range()
868 		if (record->zi_level >= dn->dn_nlevels) {  in zio_calculate_range()
873 		if (record->zi_start != 0 || record->zi_end != 0) {  in zio_calculate_range()
874 			int shift = dn->dn_indblkshift - SPA_BLKPTRSHIFT;  in zio_calculate_range()
876 			for (int level = record->zi_level; level > 0; level--) {  in zio_calculate_range()
877 				record->zi_start >>= shift;  in zio_calculate_range()
878 				record->zi_end >>= shift;  in zio_calculate_range()
895 		if (command != handler->zi_record.zi_cmd)  in zio_pool_handler_exists()
898 		const char *pool = (handler->zi_spa_name != NULL) ?  in zio_pool_handler_exists()
899 		    handler->zi_spa_name : spa_name(handler->zi_spa);  in zio_pool_handler_exists()
922 	 * If this is pool-wide metadata, make sure we unload the corresponding  in zio_inject_fault()
930 	if (record->zi_cmd == ZINJECT_DELAY_IO) {  in zio_inject_fault()
935 		if (record->zi_timer == 0 || record->zi_nlanes == 0)  in zio_inject_fault()
944 		if (record->zi_nlanes >= UINT16_MAX)  in zio_inject_fault()
949 	 * If the supplied range was in bytes -- calculate the actual blkid  in zio_inject_fault()
963 		if (record->zi_cmd == ZINJECT_DELAY_IMPORT ||  in zio_inject_fault()
964 		    record->zi_cmd == ZINJECT_DELAY_EXPORT) {  in zio_inject_fault()
965 			if (record->zi_duration <= 0)  in zio_inject_fault()
968 			 * Only one import | export delay handler per pool.  in zio_inject_fault()
970 			if (zio_pool_handler_exists(name, record->zi_cmd))  in zio_inject_fault()
977 			if (record->zi_cmd == ZINJECT_DELAY_IMPORT && has_spa)  in zio_inject_fault()
979 			if (record->zi_cmd == ZINJECT_DELAY_EXPORT && !has_spa)  in zio_inject_fault()
994 		handler->zi_spa = spa;	/* note: can be NULL */  in zio_inject_fault()
995 		handler->zi_record = *record;  in zio_inject_fault()
997 		if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_inject_fault()
998 			handler->zi_lanes = kmem_zalloc(  in zio_inject_fault()
999 			    sizeof (*handler->zi_lanes) *  in zio_inject_fault()
1000 			    handler->zi_record.zi_nlanes, KM_SLEEP);  in zio_inject_fault()
1001 			handler->zi_next_lane = 0;  in zio_inject_fault()
1003 			handler->zi_lanes = NULL;  in zio_inject_fault()
1004 			handler->zi_next_lane = 0;  in zio_inject_fault()
1007 		if (handler->zi_spa == NULL)  in zio_inject_fault()
1008 			handler->zi_spa_name = spa_strdup(name);  in zio_inject_fault()
1010 			handler->zi_spa_name = NULL;  in zio_inject_fault()
1020 		if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_inject_fault()
1026 		*id = handler->zi_id = inject_next_id++;  in zio_inject_fault()
1065 		if (handler->zi_id > *id)  in zio_inject_list_next()
1069 		*record = handler->zi_record;  in zio_inject_list_next()
1070 		*id = handler->zi_id;  in zio_inject_list_next()
1071 		ASSERT(handler->zi_spa || handler->zi_spa_name);  in zio_inject_list_next()
1072 		if (handler->zi_spa != NULL)  in zio_inject_list_next()
1073 			(void) strlcpy(name, spa_name(handler->zi_spa), buflen);  in zio_inject_list_next()
1075 			(void) strlcpy(name, handler->zi_spa_name, buflen);  in zio_inject_list_next()
1100 		if (handler->zi_id == id)  in zio_clear_fault()
1108 	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_clear_fault()
1110 		inject_delay_count--;  in zio_clear_fault()
1117 	if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {  in zio_clear_fault()
1118 		ASSERT3P(handler->zi_lanes, !=, NULL);  in zio_clear_fault()
1119 		kmem_free(handler->zi_lanes, sizeof (*handler->zi_lanes) *  in zio_clear_fault()
1120 		    handler->zi_record.zi_nlanes);  in zio_clear_fault()
1122 		ASSERT3P(handler->zi_lanes, ==, NULL);  in zio_clear_fault()
1125 	if (handler->zi_spa_name != NULL)  in zio_clear_fault()
1126 		spa_strfree(handler->zi_spa_name);  in zio_clear_fault()
1128 	if (handler->zi_spa != NULL)  in zio_clear_fault()
1129 		spa_inject_delref(handler->zi_spa);  in zio_clear_fault()