36 static void pqisrc_increment_io_counters(pqisrc_softstate_t *softs, rcb_t *rcb);
37 static int fill_lba_for_scsi_rw(pqisrc_softstate_t *softs, uint8_t *cdb, aio_req_locator_t *l);
88 pqisrc_num_elem_needed(pqisrc_softstate_t *softs, uint32_t SG_Count,  in pqisrc_num_elem_needed()  argument
107 	if(SG_Count > softs->max_sg_per_spanning_cmd ||  in pqisrc_num_elem_needed()
177 pqisrc_show_raid_req(pqisrc_softstate_t *softs, pqisrc_raid_req_t *raid_req)
204 pqisrc_build_raid_io(pqisrc_softstate_t *softs, rcb_t *rcb,  in pqisrc_build_raid_io()  argument
211 	raid_req->response_queue_id = OS_GET_IO_RESP_QID(softs, rcb);  in pqisrc_build_raid_io()
276 	pqisrc_show_raid_req(softs, raid_req);  in pqisrc_build_raid_io()
291 pqisrc_show_aio_req(pqisrc_softstate_t *softs, pqi_aio_req_t *aio_req)
336 pqisrc_build_aio_common(pqisrc_softstate_t *softs, pqi_aio_req_t *aio_req,  in pqisrc_build_aio_common()  argument
342 	aio_req->response_queue_id = OS_GET_IO_RESP_QID(softs, rcb);  in pqisrc_build_aio_common()
407 pqisrc_show_aio_common(pqisrc_softstate_t *softs, rcb_t *rcb,  in pqisrc_show_aio_common()  argument
432 	pqisrc_show_cdb(softs, "AIOC", rcb, aio_req->cdb);  in pqisrc_show_aio_common()
450 pqisrc_build_aio_R1_write(pqisrc_softstate_t *softs,  in pqisrc_build_aio_R1_write()  argument
466 	aio_req->response_queue_id = OS_GET_IO_RESP_QID(softs, rcb);  in pqisrc_build_aio_R1_write()
525 pqisrc_show_aio_R1_write(pqisrc_softstate_t *softs, rcb_t *rcb,  in pqisrc_show_aio_R1_write()  argument
552 	pqisrc_show_cdb(softs, "AIOR1W", rcb, aio_req->cdb);  in pqisrc_show_aio_R1_write()
572 pqisrc_build_aio_R5or6_write(pqisrc_softstate_t *softs,  in pqisrc_build_aio_R5or6_write()  argument
602 	aio_req->response_queue_id = OS_GET_IO_RESP_QID(softs, rcb);  in pqisrc_build_aio_R5or6_write()
677 pqisrc_show_aio_R5or6_write(pqisrc_softstate_t *softs, rcb_t *rcb,  in pqisrc_show_aio_R5or6_write()  argument
706 	pqisrc_show_cdb(softs, "AIOR56W", rcb, aio_req->cdb);  in pqisrc_show_aio_R5or6_write()
750 pqisrc_show_aio_io(pqisrc_softstate_t *softs, rcb_t *rcb,  in pqisrc_show_aio_io()  argument
759 		pqisrc_show_aio_common(softs, rcb, aio_req);  in pqisrc_show_aio_io()
765 		pqisrc_show_aio_common(softs, rcb, aio_req);  in pqisrc_show_aio_io()
769 		pqisrc_show_aio_R1_write(softs, rcb,  in pqisrc_show_aio_io()
774 		pqisrc_show_aio_R5or6_write(softs, rcb,  in pqisrc_show_aio_io()
786 pqisrc_build_aio_io(pqisrc_softstate_t *softs, rcb_t *rcb,  in pqisrc_build_aio_io()  argument
797 			pqisrc_build_aio_common(softs, aio_req,  in pqisrc_build_aio_io()
802 			pqisrc_build_aio_R1_write(softs,  in pqisrc_build_aio_io()
809 			pqisrc_build_aio_R5or6_write(softs,  in pqisrc_build_aio_io()
815 		pqisrc_build_aio_common(softs, aio_req, rcb, num_elem_alloted);  in pqisrc_build_aio_io()
818 	pqisrc_show_aio_io(softs, rcb, aio_req, num_elem_alloted);  in pqisrc_build_aio_io()
829 pqisrc_is_parity_write_stream(pqisrc_softstate_t *softs, rcb_t *rcb)  in pqisrc_is_parity_write_stream()  argument
841 	rc = fill_lba_for_scsi_rw(softs, rcb->cdbp , &loc);  in pqisrc_is_parity_write_stream()
851 	if (!softs->enable_stream_detection) {  in pqisrc_is_parity_write_stream()
871 	if ((device->raid_level == SA_RAID_5 && !softs->aio_raid5_write_bypass) ||  in pqisrc_is_parity_write_stream()
872 		(device->raid_level == SA_RAID_6 && !softs->aio_raid6_write_bypass)) {  in pqisrc_is_parity_write_stream()
926 determine_io_path_build_bypass(pqisrc_softstate_t *softs,rcb_t *rcb)  in determine_io_path_build_bypass()  argument
945 		if (devp->offload_enabled && !pqisrc_is_parity_write_stream(softs, rcb)) {  in determine_io_path_build_bypass()
946 			ret = pqisrc_build_scsi_cmd_raidbypass(softs, devp, rcb);  in determine_io_path_build_bypass()
986 pqisrc_build_send_io(pqisrc_softstate_t *softs,rcb_t *rcb)  in pqisrc_build_send_io()  argument
988 	ib_queue_t *ib_q_array = softs->op_aio_ib_q;  in pqisrc_build_send_io()
993 	int first_qindex = OS_GET_IO_REQ_QINDEX(softs, rcb);  in pqisrc_build_send_io()
995 	uint32_t num_op_ib_q = softs->num_op_aio_ibq;  in pqisrc_build_send_io()
1004 	io_path = determine_io_path_build_bypass(softs, rcb);  in pqisrc_build_send_io()
1010 		num_op_ib_q = softs->num_op_raid_ibq;  in pqisrc_build_send_io()
1011 		ib_q_array = softs->op_raid_ib_q;  in pqisrc_build_send_io()
1022 	num_elem_needed = pqisrc_num_elem_needed(softs,  in pqisrc_build_send_io()
1056 	pqisrc_increment_device_active_io(softs,devp);  in pqisrc_build_send_io()
1063 		pqisrc_build_aio_io(softs, rcb, (pqi_aio_req_t *)ib_iu,  in pqisrc_build_send_io()
1067 		pqisrc_build_raid_io(softs, rcb, (pqisrc_raid_req_t *)ib_iu,  in pqisrc_build_send_io()
1075 	pqisrc_increment_io_counters(softs, rcb);  in pqisrc_build_send_io()
1084 	PCI_MEM_PUT32(softs, ib_q->pi_register_abs, ib_q->pi_register_offset, ib_q->pi_local);  in pqisrc_build_send_io()
1130 fill_lba_for_scsi_rw(pqisrc_softstate_t *softs, uint8_t *cdb, aio_req_locator_t *l)  in fill_lba_for_scsi_rw()  argument
1184 pqisrc_is_supported_write(pqisrc_softstate_t *softs,  in pqisrc_is_supported_write()  argument
1198 	if (!softs->adv_aio_capable)  in pqisrc_is_supported_write()
1207 		if (softs->aio_raid1_write_bypass)  in pqisrc_is_supported_write()
1211 		if (softs->aio_raid5_write_bypass)  in pqisrc_is_supported_write()
1215 		if (softs->aio_raid6_write_bypass)  in pqisrc_is_supported_write()
1228 pqisrc_is_invalid_block(pqisrc_softstate_t *softs, aio_req_locator_t *l)  in pqisrc_is_invalid_block()  argument
1252 pqisrc_calc_disk_params(pqisrc_softstate_t *softs, aio_req_locator_t *l,  rcb_t *rcb)  in pqisrc_calc_disk_params()  argument
1289 pqisrc_is_single_row_column(pqisrc_softstate_t *softs, aio_req_locator_t *l)  in pqisrc_is_single_row_column()  argument
1304 pqisrc_set_map_row_and_idx(pqisrc_softstate_t *softs, aio_req_locator_t *l, rcb_t *rcb)  in pqisrc_set_map_row_and_idx()  argument
1327 pqisrc_set_read_mirror(pqisrc_softstate_t *softs,  in pqisrc_set_read_mirror()  argument
1348 	pqisrc_softstate_t *softs,  in pqisrc_set_write_mirrors()  argument
1371 pqisrc_is_r5or6_single_group(pqisrc_softstate_t *softs, aio_req_locator_t *l)  in pqisrc_is_r5or6_single_group()  argument
1394 pqisrc_is_r5or6_single_row(pqisrc_softstate_t *softs, aio_req_locator_t *l)  in pqisrc_is_r5or6_single_row()  argument
1417 pqisrc_is_r5or6_single_column(pqisrc_softstate_t *softs, aio_req_locator_t *l)  in pqisrc_is_r5or6_single_column()  argument
1496 pqisrc_aio_req_too_big(pqisrc_softstate_t *softs,  in pqisrc_aio_req_too_big()  argument
1507 		if (softs->max_aio_rw_xfer_crypto_nvme != 0) {  in pqisrc_aio_req_too_big()
1508 			dev_max = MIN(dev_max,softs->max_aio_rw_xfer_crypto_nvme);  in pqisrc_aio_req_too_big()
1517 		if (softs->max_aio_write_raid5_6 != 0) {  in pqisrc_aio_req_too_big()
1518 			dev_max = MIN(dev_max,softs->max_aio_write_raid5_6);  in pqisrc_aio_req_too_big()
1525 		(softs->max_aio_write_raid1_10_3drv != 0)) {  in pqisrc_aio_req_too_big()
1527 				softs->max_aio_write_raid1_10_3drv);  in pqisrc_aio_req_too_big()
1533 		(softs->max_aio_write_raid1_10_2drv != 0)) {  in pqisrc_aio_req_too_big()
1535 				softs->max_aio_write_raid1_10_2drv);  in pqisrc_aio_req_too_big()
1550 pqisrc_aio_show_raid_map(pqisrc_softstate_t *softs, struct raid_map *m)  in pqisrc_aio_show_raid_map()  argument
1585 pqisrc_aio_show_locator_info(pqisrc_softstate_t *softs,  in pqisrc_aio_show_locator_info()  argument
1589 	pqisrc_aio_show_raid_map(softs, l->raid_map);  in pqisrc_aio_show_locator_info()
1668 pqisrc_print_buffer(pqisrc_softstate_t *softs, char *msg, void *user_buf,  in pqisrc_print_buffer()  argument
1716 pqisrc_show_cdb(pqisrc_softstate_t *softs, char *msg, rcb_t *rcb, uint8_t *cdb)  in pqisrc_show_cdb()  argument
1719 	pqisrc_print_buffer(softs, msg, cdb, rcb->cmdlen, PRINT_FLAG_HDR_COLUMN);  in pqisrc_show_cdb()
1723 pqisrc_show_rcb_details(pqisrc_softstate_t *softs, rcb_t *rcb, char *msg, void *err_info)  in pqisrc_show_rcb_details()  argument
1738 	pqisrc_print_buffer(softs, "host cdb", OS_GET_CDBP(rcb), rcb->cmdlen, PRINT_FLAG_HDR_COLUMN);  in pqisrc_show_rcb_details()
1740 		pqisrc_print_buffer(softs, "aio mapped cdb", rcb->cdbp, rcb->cmdlen, 0);  in pqisrc_show_rcb_details()
1755 			pqisrc_show_aio_error_info(softs, rcb, err_info);  in pqisrc_show_rcb_details()
1761 			pqisrc_show_raid_error_info(softs, rcb, err_info);  in pqisrc_show_rcb_details()
1773 pqisrc_build_scsi_cmd_raidbypass(pqisrc_softstate_t *softs,  in pqisrc_build_scsi_cmd_raidbypass()  argument
1795 	rc =  fill_lba_for_scsi_rw(softs, OS_GET_CDBP(rcb), l);  in pqisrc_build_scsi_cmd_raidbypass()
1799 	if (l->is_write && !pqisrc_is_supported_write(softs, device))  in pqisrc_build_scsi_cmd_raidbypass()
1806 	if (pqisrc_is_invalid_block(softs, l))  in pqisrc_build_scsi_cmd_raidbypass()
1809 	if (!pqisrc_calc_disk_params(softs, l, rcb))  in pqisrc_build_scsi_cmd_raidbypass()
1812 	if (!pqisrc_is_single_row_column(softs, l))  in pqisrc_build_scsi_cmd_raidbypass()
1815 	if (!pqisrc_set_map_row_and_idx(softs, l, rcb))  in pqisrc_build_scsi_cmd_raidbypass()
1825 			if (!pqisrc_set_write_mirrors(softs, device, l, rcb))  in pqisrc_build_scsi_cmd_raidbypass()
1828 			pqisrc_set_read_mirror(softs, device, l);  in pqisrc_build_scsi_cmd_raidbypass()
1834 			if (!pqisrc_is_r5or6_single_group(softs, l))  in pqisrc_build_scsi_cmd_raidbypass()
1837 			if (!pqisrc_is_r5or6_single_row(softs, l))  in pqisrc_build_scsi_cmd_raidbypass()
1840 			if (!pqisrc_is_r5or6_single_column(softs, l))  in pqisrc_build_scsi_cmd_raidbypass()
1871 	if (pqisrc_aio_req_too_big(softs, device, rcb, l, disk_blk_cnt))  in pqisrc_build_scsi_cmd_raidbypass()
1880 	pqisrc_aio_show_locator_info(softs, l, disk_blk_cnt, rcb);  in pqisrc_build_scsi_cmd_raidbypass()
1892 pqisrc_send_aio_tmf(pqisrc_softstate_t *softs, pqi_scsi_dev_t *devp,  in pqisrc_send_aio_tmf()  argument
1911 	tmf_req.resp_qid = OS_GET_TMF_RESP_QID(softs, rcb);  in pqisrc_send_aio_tmf()
1912 	op_ib_q = &softs->op_aio_ib_q[0];  in pqisrc_send_aio_tmf()
1945 	rval = pqisrc_submit_cmnd(softs, op_ib_q, &tmf_req);  in pqisrc_send_aio_tmf()
1951 	rval = pqisrc_wait_on_condition(softs, rcb, PQISRC_TMF_TIMEOUT);  in pqisrc_send_aio_tmf()
1969 pqisrc_send_raid_tmf(pqisrc_softstate_t *softs, pqi_scsi_dev_t *devp,  in pqisrc_send_raid_tmf()  argument
1988 	tmf_req.resp_qid = OS_GET_TMF_RESP_QID(softs, rcb);  in pqisrc_send_raid_tmf()
1996 	if (softs->timeout_in_tmf &&  in pqisrc_send_raid_tmf()
2005 	op_ib_q = &softs->op_raid_ib_q[0];  in pqisrc_send_raid_tmf()
2015 	rval = pqisrc_submit_cmnd(softs, op_ib_q, &tmf_req);  in pqisrc_send_raid_tmf()
2021 	rval = pqisrc_wait_on_condition(softs, rcb, PQISRC_TMF_TIMEOUT);  in pqisrc_send_raid_tmf()
2038 dump_tmf_details(pqisrc_softstate_t *softs, rcb_t *rcb, char *msg)  in dump_tmf_details()  argument
2048 pqisrc_send_tmf(pqisrc_softstate_t *softs, pqi_scsi_dev_t *devp,  in pqisrc_send_tmf()  argument
2056 		softs->max_outstanding_io - softs->taglist.num_elem);  in pqisrc_send_tmf()
2059 	rcb->softs = softs;  in pqisrc_send_tmf()
2065 		dump_tmf_details(softs, rcb_to_manage, "rcb_to_manage");  in pqisrc_send_tmf()
2069 	dump_tmf_details(softs, rcb, "rcb");  in pqisrc_send_tmf()
2075 					ret = pqisrc_send_aio_tmf(softs, devp, rcb, rcb_to_manage, tmf_type);  in pqisrc_send_tmf()
2083 			ret = pqisrc_send_raid_tmf(softs, devp, rcb, rcb_to_manage, tmf_type);  in pqisrc_send_tmf()
2087 			ret = pqisrc_send_aio_tmf(softs, devp, rcb, rcb_to_manage, tmf_type);  in pqisrc_send_tmf()
2089 			ret = pqisrc_send_raid_tmf(softs, devp, rcb, rcb_to_manage, tmf_type);  in pqisrc_send_tmf()
2214 pqisrc_increment_io_counters(pqisrc_softstate_t *softs, rcb_t *rcb)  in pqisrc_increment_io_counters()  argument
2218 	io_counters_t *pcounter = &softs->counters[type_index];  in pqisrc_increment_io_counters()
2241 print_this_counter(pqisrc_softstate_t *softs, io_counters_t *pcounter, char *msg)  in print_this_counter()  argument
2247 	if (!softs->log_io_counters)  in print_this_counter()
2299 print_all_counters(pqisrc_softstate_t *softs, uint32_t flags)  in print_all_counters()  argument
2307 		pcounter = &softs->counters[ii];  in print_all_counters()
2316 		print_this_counter(softs, pcounter, raid_type);  in print_all_counters()
2322 		memset(softs->counters, 0, sizeof(softs->counters));  in print_all_counters()