106 #define	TT_HASH(icc, tt)	(G_PPOD_TAG(tt) & (icc)->cmp_hash_mask)  argument
332 finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp)  in finalize_pdu()  argument
344 	ulp_submode = icc->ulp_submode;  in finalize_pdu()
373 	if (data_len > icc->ic.ic_max_send_data_segment_length) {  in finalize_pdu()
389 		data_len = icc->ic.ic_max_send_data_segment_length;  in finalize_pdu()
432 	struct icl_cxgbei_conn *icc = arg;  in icl_cxgbei_tx_main()  local
433 	struct icl_conn *ic = &icc->ic;  in icl_cxgbei_tx_main()
434 	struct toepcb *toep = icc->toep;  in icl_cxgbei_tx_main()
446 		while (STAILQ_EMPTY(&icc->sent_pdus)) {  in icl_cxgbei_tx_main()
447 			icc->tx_active = false;  in icl_cxgbei_tx_main()
448 			mtx_sleep(&icc->tx_active, ic->ic_lock, 0, "-", 0);  in icl_cxgbei_tx_main()
451 			MPASS(icc->tx_active);  in icl_cxgbei_tx_main()
454 		STAILQ_SWAP(&icc->sent_pdus, &tx_pdus, icl_pdu);  in icl_cxgbei_tx_main()
460 			m = finalize_pdu(icc, ip_to_icp(ip));  in icl_cxgbei_tx_main()
484 			t4_push_pdus(icc->sc, toep, 0);  in icl_cxgbei_tx_main()
501 	struct icl_cxgbei_conn *icc = arg;  in icl_cxgbei_rx_main()  local
502 	struct icl_conn *ic = &icc->ic;  in icl_cxgbei_rx_main()
511 		while (STAILQ_EMPTY(&icc->rcvd_pdus)) {  in icl_cxgbei_rx_main()
512 			icc->rx_active = false;  in icl_cxgbei_rx_main()
513 			mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0);  in icl_cxgbei_rx_main()
516 			MPASS(icc->rx_active);  in icl_cxgbei_rx_main()
526 			parse_pdus(icc, sb);  in icl_cxgbei_rx_main()
530 		STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu);  in icl_cxgbei_rx_main()
551 	while (!icc->rx_exiting)  in icl_cxgbei_rx_main()
552 		mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0);  in icl_cxgbei_rx_main()
870 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);  in icl_cxgbei_conn_pdu_queue_cb()  local
889 	STAILQ_INSERT_TAIL(&icc->sent_pdus, ip, ip_next);  in icl_cxgbei_conn_pdu_queue_cb()
890 	if (!icc->tx_active) {  in icl_cxgbei_conn_pdu_queue_cb()
891 		icc->tx_active = true;  in icl_cxgbei_conn_pdu_queue_cb()
892 		wakeup(&icc->tx_active);  in icl_cxgbei_conn_pdu_queue_cb()
899 	struct icl_cxgbei_conn *icc;  in icl_cxgbei_new_conn()  local
904 	icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE,  in icl_cxgbei_new_conn()
906 	icc->icc_signature = CXGBEI_CONN_SIGNATURE;  in icl_cxgbei_new_conn()
907 	STAILQ_INIT(&icc->rcvd_pdus);  in icl_cxgbei_new_conn()
908 	STAILQ_INIT(&icc->sent_pdus);  in icl_cxgbei_new_conn()
910 	icc->cmp_table = hashinit(64, M_CXGBEI, &icc->cmp_hash_mask);  in icl_cxgbei_new_conn()
911 	mtx_init(&icc->cmp_lock, "cxgbei_cmp", NULL, MTX_DEF);  in icl_cxgbei_new_conn()
913 	ic = &icc->ic;  in icl_cxgbei_new_conn()
923 	CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);  in icl_cxgbei_new_conn()
931 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);  in icl_cxgbei_conn_free()  local
933 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);  in icl_cxgbei_conn_free()
935 	CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);  in icl_cxgbei_conn_free()
937 	mtx_destroy(&icc->cmp_lock);  in icl_cxgbei_conn_free()
938 	hashdestroy(icc->cmp_table, M_CXGBEI, icc->cmp_hash_mask);  in icl_cxgbei_conn_free()
939 	kobj_delete((struct kobj *)icc, M_CXGBE);  in icl_cxgbei_conn_free()
1095 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);  in icl_cxgbei_conn_handoff()  local
1106 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);  in icl_cxgbei_conn_handoff()
1148 	icc->sc = fa.sc;  in icl_cxgbei_conn_handoff()
1177 	MPASS(toep->vi->adapter == icc->sc);  in icl_cxgbei_conn_handoff()
1185 	icc->toep = toep;  in icl_cxgbei_conn_handoff()
1187 	icc->ulp_submode = 0;  in icl_cxgbei_conn_handoff()
1189 		icc->ulp_submode |= ULP_CRC_HEADER;  in icl_cxgbei_conn_handoff()
1191 		icc->ulp_submode |= ULP_CRC_DATA;  in icl_cxgbei_conn_handoff()
1193 	if (icc->sc->tt.iso && chip_id(icc->sc) >= CHELSIO_T5 &&  in icl_cxgbei_conn_handoff()
1194 	    !is_memfree(icc->sc)) {  in icl_cxgbei_conn_handoff()
1204 	toep->ulpcb = icc;  in icl_cxgbei_conn_handoff()
1206 	send_iscsi_flowc_wr(icc->sc, toep,  in icl_cxgbei_conn_handoff()
1208 	set_ulp_mode_iscsi(icc->sc, toep, icc->ulp_submode);  in icl_cxgbei_conn_handoff()
1211 	error = kthread_add(icl_cxgbei_tx_main, icc, NULL, &icc->tx_thread, 0,  in icl_cxgbei_conn_handoff()
1216 	error = kthread_add(icl_cxgbei_rx_main, icc, NULL, &icc->rx_thread, 0,  in icl_cxgbei_conn_handoff()
1231 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);  in icl_cxgbei_conn_close()  local
1236 	struct toepcb *toep = icc->toep;  in icl_cxgbei_conn_close()
1238 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);  in icl_cxgbei_conn_close()
1244 		CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p",  in icl_cxgbei_conn_close()
1245 		    __func__, icc, ic->ic_disconnecting, so);  in icl_cxgbei_conn_close()
1257 	CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1,  in icl_cxgbei_conn_close()
1258 	    icc);  in icl_cxgbei_conn_close()
1264 	if (icc->tx_thread != NULL) {  in icl_cxgbei_conn_close()
1265 		wakeup(&icc->tx_active);  in icl_cxgbei_conn_close()
1266 		mtx_sleep(icc->tx_thread, ic->ic_lock, 0, "conclo", 0);  in icl_cxgbei_conn_close()
1270 	while (!STAILQ_EMPTY(&icc->sent_pdus)) {  in icl_cxgbei_conn_close()
1271 		ip = STAILQ_FIRST(&icc->sent_pdus);  in icl_cxgbei_conn_close()
1272 		STAILQ_REMOVE_HEAD(&icc->sent_pdus, ip_next);  in icl_cxgbei_conn_close()
1285 	if (icc->rx_thread != NULL) {  in icl_cxgbei_conn_close()
1286 		icc->rx_exiting = true;  in icl_cxgbei_conn_close()
1287 		wakeup(&icc->rx_active);  in icl_cxgbei_conn_close()
1288 		mtx_sleep(icc->rx_thread, SOCKBUF_MTX(sb), 0, "conclo", 0);  in icl_cxgbei_conn_close()
1294 	while (!STAILQ_EMPTY(&icc->rcvd_pdus)) {  in icl_cxgbei_conn_close()
1295 		ip = STAILQ_FIRST(&icc->rcvd_pdus);  in icl_cxgbei_conn_close()
1296 		STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next);  in icl_cxgbei_conn_close()
1352 cxgbei_insert_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp,  in cxgbei_insert_cmp()  argument
1361 	mtx_lock(&icc->cmp_lock);  in cxgbei_insert_cmp()
1363 	LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, tt)], link) {  in cxgbei_insert_cmp()
1367 	LIST_INSERT_HEAD(&icc->cmp_table[TT_HASH(icc, tt)], cmp, link);  in cxgbei_insert_cmp()
1368 	mtx_unlock(&icc->cmp_lock);  in cxgbei_insert_cmp()
1372 cxgbei_find_cmp(struct icl_cxgbei_conn *icc, uint32_t tt)  in cxgbei_find_cmp()  argument
1376 	mtx_lock(&icc->cmp_lock);  in cxgbei_find_cmp()
1377 	LIST_FOREACH(cmp, &icc->cmp_table[TT_HASH(icc, tt)], link) {  in cxgbei_find_cmp()
1381 	mtx_unlock(&icc->cmp_lock);  in cxgbei_find_cmp()
1386 cxgbei_rm_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp)  in cxgbei_rm_cmp()  argument
1392 	mtx_lock(&icc->cmp_lock);  in cxgbei_rm_cmp()
1395 	LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, cmp->tt)], link) {  in cxgbei_rm_cmp()
1403 	mtx_unlock(&icc->cmp_lock);  in cxgbei_rm_cmp()
1410 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);  in icl_cxgbei_conn_task_setup()  local
1411 	struct toepcb *toep = icc->toep;  in icl_cxgbei_conn_task_setup()
1412 	struct adapter *sc = icc->sc;  in icl_cxgbei_conn_task_setup()
1517 	cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);  in icl_cxgbei_conn_task_setup()
1571 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);  in icl_cxgbei_conn_transfer_setup()  local
1572 	struct toepcb *toep = icc->toep;  in icl_cxgbei_conn_transfer_setup()
1574 	struct adapter *sc = icc->sc;  in icl_cxgbei_conn_transfer_setup()
1696 		cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);  in icl_cxgbei_conn_transfer_setup()
1719 	cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);  in icl_cxgbei_conn_transfer_setup()