384 static struct rdma_done_list *rdma_done_add(rib_qp_t *qp, uint32_t xid);
385 static void rdma_done_rm(rib_qp_t *qp, struct rdma_done_list *rd);
387 static void rdma_done_notify(rib_qp_t *qp, uint32_t xid);
1010 	rib_qp_t	*qp;  in rib_clnt_scq_handler()  local
1031 			qp = wd->qp;  in rib_clnt_scq_handler()
1032 			conn = qptoc(qp);  in rib_clnt_scq_handler()
1080 					rib_rbuf_free(qptoc(wd->qp),  in rib_clnt_scq_handler()
1086 				rib_send_rele(qp);  in rib_clnt_scq_handler()
1102 	rib_qp_t	*qp;  in rib_svc_scq_handler()  local
1124 			qp = wd->qp;  in rib_svc_scq_handler()
1125 			conn = qptoc(qp);  in rib_svc_scq_handler()
1158 					rib_rbuf_free(qptoc(wd->qp),  in rib_svc_scq_handler()
1164 				rib_send_rele(qp);  in rib_svc_scq_handler()
1180 	rib_qp_t	*qp;  in rib_clnt_rcq_handler()  local
1199 		qp = rwid->qp;  in rib_clnt_rcq_handler()
1205 			CONN *conn = qptoc(qp);  in rib_clnt_rcq_handler()
1237 				rib_recv_rele(qp);  in rib_clnt_rcq_handler()
1241 			mutex_enter(&qp->replylist_lock);  in rib_clnt_rcq_handler()
1242 			for (r = qp->replylist; r != NULL; r = r->next) {  in rib_clnt_rcq_handler()
1256 						rib_rbuf_free(qptoc(qp),  in rib_clnt_rcq_handler()
1265 			mutex_exit(&qp->replylist_lock);  in rib_clnt_rcq_handler()
1272 				rib_rbuf_free(qptoc(qp), RECV_BUFFER,  in rib_clnt_rcq_handler()
1276 			CONN *conn = qptoc(qp);  in rib_clnt_rcq_handler()
1285 			CONN *conn = qptoc(qp);  in rib_clnt_rcq_handler()
1308 		rib_recv_rele(qp);  in rib_clnt_rcq_handler()
1318 	rib_qp_t	*qp;  in rib_svc_rcq_handler()  local
1339 		qp = s_recvp->qp;  in rib_svc_rcq_handler()
1340 		conn = qptoc(qp);  in rib_svc_rcq_handler()
1365 				rib_recv_rele(qp);  in rib_svc_rcq_handler()
1378 				rib_recv_rele(qp);  in rib_svc_rcq_handler()
1392 				mutex_enter(&qp->rdlist_lock);  in rib_svc_rcq_handler()
1393 				rdma_done_notify(qp, xid);  in rib_svc_rcq_handler()
1394 				mutex_exit(&qp->rdlist_lock);  in rib_svc_rcq_handler()
1395 				rib_recv_rele(qp);  in rib_svc_rcq_handler()
1443 		rib_recv_rele(qp);  in rib_svc_rcq_handler()
1565 rib_clnt_create_chan(rib_hca_t *hca, struct netbuf *raddr, rib_qp_t **qp)  in rib_clnt_create_chan()  argument
1571 	ASSERT(qp != NULL);  in rib_clnt_create_chan()
1572 	*qp = NULL;  in rib_clnt_create_chan()
1608 	*qp = kqp;  in rib_clnt_create_chan()
1614 rib_svc_create_chan(rib_hca_t *hca, caddr_t q, uint8_t port, rib_qp_t **qp)  in rib_svc_create_chan()  argument
1622 	*qp = NULL;  in rib_svc_create_chan()
1693 	*qp = kqp;  in rib_svc_create_chan()
1719 		rib_qp_t *qp;  in rib_clnt_cm_handler()  local
1742 			qp = ibt_get_chan_private(event->cm_channel);  in rib_clnt_cm_handler()
1743 			conn = qptoc(qp);  in rib_clnt_cm_handler()
1794 rib_conn_to_srv(rib_hca_t *hca, rib_qp_t *qp, rpcib_ping_t *rptp)  in rib_conn_to_srv()  argument
1870 		    &qp_attr, &qp->qp_hdl,  in rib_conn_to_srv()
1886 	mutex_enter(&qp->cb_lock);  in rib_conn_to_srv()
1887 	ibt_status = ibt_open_rc_channel(qp->qp_hdl, IBT_OCHAN_NO_FLAGS,  in rib_conn_to_srv()
1893 		(void) ibt_free_channel(qp->qp_hdl);  in rib_conn_to_srv()
1894 		qp->qp_hdl = NULL;  in rib_conn_to_srv()
1895 		mutex_exit(&qp->cb_lock);  in rib_conn_to_srv()
1909 	mutex_exit(&qp->cb_lock);  in rib_conn_to_srv()
1913 	ibt_set_chan_private(qp->qp_hdl, (void *)qp);  in rib_conn_to_srv()
2023 	rib_qp_t	*qp = ctoqp(conn);  in rib_disconnect_channel()  local
2041 	hca = qp->hca;  in rib_disconnect_channel()
2051 	if (qp->qp_hdl != NULL) {  in rib_disconnect_channel()
2052 		mutex_enter(&qp->posted_rbufs_lock);  in rib_disconnect_channel()
2053 		while (qp->n_posted_rbufs)  in rib_disconnect_channel()
2054 			cv_wait(&qp->posted_rbufs_cv, &qp->posted_rbufs_lock);  in rib_disconnect_channel()
2055 		mutex_exit(&qp->posted_rbufs_lock);  in rib_disconnect_channel()
2057 		mutex_enter(&qp->send_rbufs_lock);  in rib_disconnect_channel()
2058 		while (qp->n_send_rbufs)  in rib_disconnect_channel()
2059 			cv_wait(&qp->send_rbufs_cv, &qp->send_rbufs_lock);  in rib_disconnect_channel()
2060 			mutex_exit(&qp->send_rbufs_lock);  in rib_disconnect_channel()
2062 		(void) ibt_free_channel(qp->qp_hdl);  in rib_disconnect_channel()
2063 			qp->qp_hdl = NULL;  in rib_disconnect_channel()
2066 	ASSERT(qp->rdlist == NULL);  in rib_disconnect_channel()
2068 	if (qp->replylist != NULL) {  in rib_disconnect_channel()
2069 		(void) rib_rem_replylist(qp);  in rib_disconnect_channel()
2072 	cv_destroy(&qp->cb_conn_cv);  in rib_disconnect_channel()
2073 	cv_destroy(&qp->posted_rbufs_cv);  in rib_disconnect_channel()
2074 	cv_destroy(&qp->send_rbufs_cv);  in rib_disconnect_channel()
2075 	mutex_destroy(&qp->cb_lock);  in rib_disconnect_channel()
2076 	mutex_destroy(&qp->replylist_lock);  in rib_disconnect_channel()
2077 	mutex_destroy(&qp->posted_rbufs_lock);  in rib_disconnect_channel()
2078 	mutex_destroy(&qp->send_rbufs_lock);  in rib_disconnect_channel()
2079 	mutex_destroy(&qp->rdlist_lock);  in rib_disconnect_channel()
2100 	if (qp->rdmaconn.c_cc_type == RDMA_CC_CLNT) {  in rib_disconnect_channel()
2102 		cc_info = &qp->rdmaconn.rdma_conn_cred_ctrl_u.c_clnt_cc;  in rib_disconnect_channel()
2106 	kmem_free(qp, sizeof (rib_qp_t));  in rib_disconnect_channel()
2145 rib_send_hold(rib_qp_t *qp)  in rib_send_hold()  argument
2147 	mutex_enter(&qp->send_rbufs_lock);  in rib_send_hold()
2148 	qp->n_send_rbufs++;  in rib_send_hold()
2149 	mutex_exit(&qp->send_rbufs_lock);  in rib_send_hold()
2153 rib_send_rele(rib_qp_t *qp)  in rib_send_rele()  argument
2155 	mutex_enter(&qp->send_rbufs_lock);  in rib_send_rele()
2156 	qp->n_send_rbufs--;  in rib_send_rele()
2157 	if (qp->n_send_rbufs == 0)  in rib_send_rele()
2158 		cv_signal(&qp->send_rbufs_cv);  in rib_send_rele()
2159 	mutex_exit(&qp->send_rbufs_lock);  in rib_send_rele()
2163 rib_recv_rele(rib_qp_t *qp)  in rib_recv_rele()  argument
2165 	mutex_enter(&qp->posted_rbufs_lock);  in rib_recv_rele()
2166 	qp->n_posted_rbufs--;  in rib_recv_rele()
2167 	if (qp->n_posted_rbufs == 0)  in rib_recv_rele()
2168 		cv_signal(&qp->posted_rbufs_cv);  in rib_recv_rele()
2169 	mutex_exit(&qp->posted_rbufs_lock);  in rib_recv_rele()
2178 rib_sendwait(rib_qp_t *qp, struct send_wid *wd)  in rib_sendwait()  argument
2193 		if (qp->mode == RIB_SERVER) {  in rib_sendwait()
2245 			rib_rbuf_free(qptoc(qp), SEND_BUFFER,  in rib_sendwait()
2249 		rib_send_rele(qp);  in rib_sendwait()
2261 rib_init_sendwait(uint32_t xid, int cv_sig, rib_qp_t *qp)  in rib_init_sendwait()  argument
2268 	wd->qp = qp;  in rib_init_sendwait()
2287 rib_rem_rep(rib_qp_t *qp, struct reply *rep)  in rib_rem_rep()  argument
2289 	mutex_enter(&qp->replylist_lock);  in rib_rem_rep()
2291 		(void) rib_remreply(qp, rep);  in rib_rem_rep()
2292 		mutex_exit(&qp->replylist_lock);  in rib_rem_rep()
2295 	mutex_exit(&qp->replylist_lock);  in rib_rem_rep()
2316 	rib_qp_t	*qp;  in rib_send_and_wait()  local
2318 	qp = ctoqp(conn);  in rib_send_and_wait()
2343 		wdesc = rib_init_sendwait(msgid, cv_sig, qp);  in rib_send_and_wait()
2364 		ibt_status = ibt_post_send(qp->qp_hdl, &tx_wr, 1, NULL);  in rib_send_and_wait()
2385 		rib_send_hold(qp);  in rib_send_and_wait()
2393 			ret = rib_sendwait(qp, wdesc);  in rib_send_and_wait()
2426 	rib_qp_t *qp = ctoqp(conn);  in rib_send_resp()  local
2428 	mutex_enter(&qp->rdlist_lock);  in rib_send_resp()
2429 	rd = rdma_done_add(qp, msgid);  in rib_send_resp()
2435 		rdma_done_rm(qp, rd);  in rib_send_resp()
2441 		    &qp->rdlist_lock, drv_usectohz(REPLY_WAIT_TIME * 1000000),  in rib_send_resp()
2444 		rdma_done_rm(qp, rd);  in rib_send_resp()
2451 	mutex_exit(&qp->rdlist_lock);  in rib_send_resp()
2456 rib_create_wid(rib_qp_t *qp, ibt_wr_ds_t *sgl, uint32_t msgid)  in rib_create_wid()  argument
2463 	rwid->qp = qp;  in rib_create_wid()
2477 	rib_qp_t	*qp = ctoqp(conn);  in rib_clnt_post()  local
2513 	rwid = rib_create_wid(qp, &sgl[0], msgid);  in rib_clnt_post()
2520 	rep = rib_addreplylist(qp, msgid);  in rib_clnt_post()
2530 		ibt_status = ibt_post_recv(qp->qp_hdl, &recv_wr, 1, NULL);  in rib_clnt_post()
2539 		(void) rib_rem_rep(qp, rep);  in rib_clnt_post()
2544 	mutex_enter(&qp->posted_rbufs_lock);  in rib_clnt_post()
2545 	qp->n_posted_rbufs++;  in rib_clnt_post()
2546 	mutex_exit(&qp->posted_rbufs_lock);  in rib_clnt_post()
2563 	rib_qp_t	*qp = ctoqp(conn);  in rib_svc_post()  local
2593 	s_recvp = rib_init_svc_recv(qp, &sgl[0]);  in rib_svc_post()
2598 		ibt_status = ibt_post_recv(qp->qp_hdl, &recv_wr, 1, NULL);  in rib_svc_post()
2627 	rib_qp_t	*qp = ctoqp(conn);  in rib_post_resp_remove()  local
2630 	mutex_enter(&qp->replylist_lock);  in rib_post_resp_remove()
2631 	for (rep = qp->replylist; rep != NULL; rep = rep->next) {  in rib_post_resp_remove()
2637 			(void) rib_remreply(qp, rep);  in rib_post_resp_remove()
2641 	mutex_exit(&qp->replylist_lock);  in rib_post_resp_remove()
2650 	rib_qp_t	*qp = ctoqp(conn);  in rib_post_recv()  local
2653 		mutex_enter(&qp->posted_rbufs_lock);  in rib_post_recv()
2654 		qp->n_posted_rbufs++;  in rib_post_recv()
2655 		mutex_exit(&qp->posted_rbufs_lock);  in rib_post_recv()
2671 	rib_qp_t *qp = ctoqp(conn);  in rib_recv()  local
2676 	mutex_enter(&qp->replylist_lock);  in rib_recv()
2678 	for (rep = qp->replylist; rep != NULL; rep = rep->next) {  in rib_recv()
2692 			    &qp->replylist_lock, timout)) > 0 &&  in rib_recv()
2728 		(void) rib_remreply(qp, rep);  in rib_recv()
2741 	mutex_exit(&qp->replylist_lock);  in rib_recv()
2757 	rib_qp_t	*qp = ctoqp(conn);  in rib_write()  local
2787 				wdesc = rib_init_sendwait(0, cv_sig, qp);  in rib_write()
2802 				    ibt_post_send(qp->qp_hdl, &tx_wr, 1, NULL);  in rib_write()
2823 				rib_send_hold(qp);  in rib_write()
2826 				ret = rib_sendwait(qp, wdesc);  in rib_write()
2849 	rib_qp_t	*qp = ctoqp(conn);  in rib_read()  local
2876 			wdesc = rib_init_sendwait(0, cv_sig, qp);  in rib_read()
2891 			ibt_status = ibt_post_send(qp->qp_hdl, &rx_wr, 1, NULL);  in rib_read()
2912 			rib_send_hold(qp);  in rib_read()
2915 			ret = rib_sendwait(qp, wdesc);  in rib_read()
2936 	rib_qp_t	*qp;  in rib_srv_cm_handler()  local
2981 		    event->cm_event.req.req_prim_hca_port, &qp);  in rib_srv_cm_handler()
2987 		ret_args->cm_ret.rep.cm_channel = qp->qp_hdl;  in rib_srv_cm_handler()
2995 		conn = qptoc(qp);  in rib_srv_cm_handler()
3007 				(void) ibt_flush_channel(qp->qp_hdl);  in rib_srv_cm_handler()
3026 				(void) ibt_flush_channel(qp->qp_hdl);  in rib_srv_cm_handler()
3136 		rib_qp_t	*qp;  in rib_srv_cm_handler()  local
3158 			qp = ibt_get_chan_private(event->cm_channel);  in rib_srv_cm_handler()
3159 			conn = qptoc(qp);  in rib_srv_cm_handler()
3564 rib_init_svc_recv(rib_qp_t *qp, ibt_wr_ds_t *sgl)  in rib_init_svc_recv()  argument
3570 	recvp->qp = qp;  in rib_init_svc_recv()
3584 rib_addreplylist(rib_qp_t *qp, uint32_t msgid)  in rib_addreplylist()  argument
3601 	mutex_enter(&qp->replylist_lock);  in rib_addreplylist()
3602 	if (qp->replylist) {  in rib_addreplylist()
3603 		rep->next = qp->replylist;  in rib_addreplylist()
3604 		qp->replylist->prev = rep;  in rib_addreplylist()
3606 	qp->rep_list_size++;  in rib_addreplylist()
3609 	    int, qp->rep_list_size);  in rib_addreplylist()
3611 	qp->replylist = rep;  in rib_addreplylist()
3612 	mutex_exit(&qp->replylist_lock);  in rib_addreplylist()
3618 rib_rem_replylist(rib_qp_t *qp)  in rib_rem_replylist()  argument
3622 	mutex_enter(&qp->replylist_lock);  in rib_rem_replylist()
3623 	for (r = qp->replylist; r != NULL; r = n) {  in rib_rem_replylist()
3625 		(void) rib_remreply(qp, r);  in rib_rem_replylist()
3627 	mutex_exit(&qp->replylist_lock);  in rib_rem_replylist()
3633 rib_remreply(rib_qp_t *qp, struct reply *rep)  in rib_remreply()  argument
3636 	ASSERT(MUTEX_HELD(&qp->replylist_lock));  in rib_remreply()
3643 	if (qp->replylist == rep)  in rib_remreply()
3644 		qp->replylist = rep->next;  in rib_remreply()
3647 	qp->rep_list_size--;  in rib_remreply()
3650 	    int, qp->rep_list_size);  in rib_remreply()
4061 	rib_qp_t	*qp = ctoqp(conn);  in rib_rbuf_alloc()  local
4062 	rib_hca_t	*hca = qp->hca;  in rib_rbuf_alloc()
4133 	rib_qp_t *qp = ctoqp(conn);  in rib_rbuf_free()  local
4134 	rib_hca_t *hca = qp->hca;  in rib_rbuf_free()
4340 	rib_qp_t *qp;  in rib_connect()  local
4384 	if (rib_clnt_create_chan(hca, d_svcaddr, &qp) != RDMA_SUCCESS) {  in rib_connect()
4387 	cn = qptoc(qp);  in rib_connect()
4434 	status = rib_conn_to_srv(hca, qp, rpt);  in rib_connect()
4475 	rib_qp_t *qp = ctoqp(conn);  in rib_conn_close()  local
4498 	if (qp->mode == RIB_SERVER)  in rib_conn_close()
4500 		    &qp->hca->srv_conn_list);  in rib_conn_close()
4503 		    &qp->hca->cl_conn_list);  in rib_conn_close()
4605 rdma_done_add(rib_qp_t *qp, uint32_t xid)  in rdma_done_add()  argument
4609 	ASSERT(MUTEX_HELD(&qp->rdlist_lock));  in rdma_done_add()
4616 	rd->next = qp->rdlist;  in rdma_done_add()
4617 	if (qp->rdlist != NULL)  in rdma_done_add()
4618 		qp->rdlist->prev = rd;  in rdma_done_add()
4619 	qp->rdlist = rd;  in rdma_done_add()
4625 rdma_done_rm(rib_qp_t *qp, struct rdma_done_list *rd)  in rdma_done_rm()  argument
4629 	ASSERT(MUTEX_HELD(&qp->rdlist_lock));  in rdma_done_rm()
4640 		qp->rdlist = rd->next;  in rdma_done_rm()
4648 rdma_done_rem_list(rib_qp_t *qp)  in rdma_done_rem_list()  argument
4652 	mutex_enter(&qp->rdlist_lock);  in rdma_done_rem_list()
4653 	for (r = qp->rdlist; r != NULL; r = n) {  in rdma_done_rem_list()
4655 		rdma_done_rm(qp, r);  in rdma_done_rem_list()
4657 	mutex_exit(&qp->rdlist_lock);  in rdma_done_rem_list()
4661 rdma_done_notify(rib_qp_t *qp, uint32_t xid)  in rdma_done_notify()  argument
4663 	struct rdma_done_list *r = qp->rdlist;  in rdma_done_notify()
4665 	ASSERT(MUTEX_HELD(&qp->rdlist_lock));  in rdma_done_notify()
4686 	rib_qp_t	*qp;  in rib_close_a_channel()  local
4687 	qp = ctoqp(conn);  in rib_close_a_channel()
4689 	if (qp->qp_hdl == NULL) {  in rib_close_a_channel()
4698 	(void) ibt_close_rc_channel(qp->qp_hdl, IBT_NOCALLBACKS,  in rib_close_a_channel()