229 static void irdma_alloc_push_page(struct irdma_qp *iwqp)  in irdma_alloc_push_page()  argument
233 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_alloc_push_page()
234 	struct irdma_sc_qp *qp = &iwqp->sc_qp;  in irdma_alloc_push_page()
457 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq)  in irdma_clean_cqes()  argument
463 	irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq);  in irdma_clean_cqes()
467 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp)  in irdma_remove_push_mmap_entries()  argument
469 	if (iwqp->push_db_mmap_entry) {  in irdma_remove_push_mmap_entries()
470 		rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry);  in irdma_remove_push_mmap_entries()
471 		iwqp->push_db_mmap_entry = NULL;  in irdma_remove_push_mmap_entries()
473 	if (iwqp->push_wqe_mmap_entry) {  in irdma_remove_push_mmap_entries()
474 		rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);  in irdma_remove_push_mmap_entries()
475 		iwqp->push_wqe_mmap_entry = NULL;  in irdma_remove_push_mmap_entries()
480 					 struct irdma_qp *iwqp,  in irdma_setup_push_mmap_entries()  argument
492 	bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE;  in irdma_setup_push_mmap_entries()
493 	iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext,  in irdma_setup_push_mmap_entries()
496 	if (!iwqp->push_wqe_mmap_entry)  in irdma_setup_push_mmap_entries()
501 	iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext,  in irdma_setup_push_mmap_entries()
504 	if (!iwqp->push_db_mmap_entry) {  in irdma_setup_push_mmap_entries()
505 		rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);  in irdma_setup_push_mmap_entries()
519 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_destroy_qp()  local
520 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_destroy_qp()
522 	iwqp->sc_qp.qp_uk.destroy_pending = true;  in irdma_destroy_qp()
524 	if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS)  in irdma_destroy_qp()
525 		irdma_modify_qp_to_err(&iwqp->sc_qp);  in irdma_destroy_qp()
527 	if (!iwqp->user_mode)  in irdma_destroy_qp()
528 		cancel_delayed_work_sync(&iwqp->dwork_flush);  in irdma_destroy_qp()
530 	if (!iwqp->user_mode) {  in irdma_destroy_qp()
531 		if (iwqp->iwscq) {  in irdma_destroy_qp()
532 			irdma_clean_cqes(iwqp, iwqp->iwscq);  in irdma_destroy_qp()
533 			if (iwqp->iwrcq != iwqp->iwscq)  in irdma_destroy_qp()
534 				irdma_clean_cqes(iwqp, iwqp->iwrcq);  in irdma_destroy_qp()
538 	irdma_qp_rem_ref(&iwqp->ibqp);  in irdma_destroy_qp()
539 	wait_for_completion(&iwqp->free_qp);  in irdma_destroy_qp()
540 	irdma_free_lsmm_rsrc(iwqp);  in irdma_destroy_qp()
541 	irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp);  in irdma_destroy_qp()
543 	irdma_remove_push_mmap_entries(iwqp);  in irdma_destroy_qp()
544 	irdma_free_qp_rsrc(iwqp);  in irdma_destroy_qp()
556 			       struct irdma_qp *iwqp,  in irdma_setup_virt_qp()  argument
559 	struct irdma_pbl *iwpbl = iwqp->iwpbl;  in irdma_setup_virt_qp()
562 	iwqp->page = qpmr->sq_page;  in irdma_setup_virt_qp()
584 				struct irdma_qp *iwqp,  in irdma_setup_umode_qp()  argument
602 	iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;  in irdma_setup_umode_qp()
603 	iwqp->user_mode = 1;  in irdma_setup_umode_qp()
607 		iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,  in irdma_setup_umode_qp()
611 		if (!iwqp->iwpbl) {  in irdma_setup_umode_qp()
625 		iwqp->max_send_wr = init_attr->cap.max_send_wr;  in irdma_setup_umode_qp()
626 		iwqp->max_recv_wr = init_attr->cap.max_recv_wr;  in irdma_setup_umode_qp()
642 		iwqp->max_send_wr =  in irdma_setup_umode_qp()
644 		iwqp->max_recv_wr =  in irdma_setup_umode_qp()
650 	irdma_setup_virt_qp(iwdev, iwqp, info);  in irdma_setup_umode_qp()
663 				struct irdma_qp *iwqp,  in irdma_setup_kmode_qp()  argument
667 	struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem;  in irdma_setup_kmode_qp()
682 	iwqp->kqp.sq_wrid_mem =  in irdma_setup_kmode_qp()
683 		kcalloc(ukinfo->sq_depth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL);  in irdma_setup_kmode_qp()
684 	if (!iwqp->kqp.sq_wrid_mem)  in irdma_setup_kmode_qp()
687 	iwqp->kqp.rq_wrid_mem =  in irdma_setup_kmode_qp()
688 		kcalloc(ukinfo->rq_depth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL);  in irdma_setup_kmode_qp()
690 	if (!iwqp->kqp.rq_wrid_mem) {  in irdma_setup_kmode_qp()
691 		kfree(iwqp->kqp.sq_wrid_mem);  in irdma_setup_kmode_qp()
692 		iwqp->kqp.sq_wrid_mem = NULL;  in irdma_setup_kmode_qp()
696 	ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem;  in irdma_setup_kmode_qp()
697 	ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem;  in irdma_setup_kmode_qp()
706 		kfree(iwqp->kqp.sq_wrid_mem);  in irdma_setup_kmode_qp()
707 		iwqp->kqp.sq_wrid_mem = NULL;  in irdma_setup_kmode_qp()
708 		kfree(iwqp->kqp.rq_wrid_mem);  in irdma_setup_kmode_qp()
709 		iwqp->kqp.rq_wrid_mem = NULL;  in irdma_setup_kmode_qp()
723 	iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift;  in irdma_setup_kmode_qp()
724 	iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift;  in irdma_setup_kmode_qp()
725 	init_attr->cap.max_send_wr = iwqp->max_send_wr;  in irdma_setup_kmode_qp()
726 	init_attr->cap.max_recv_wr = iwqp->max_recv_wr;  in irdma_setup_kmode_qp()
731 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp)  in irdma_cqp_create_qp_cmd()  argument
733 	struct irdma_pci_f *rf = iwqp->iwdev->rf;  in irdma_cqp_create_qp_cmd()
752 	cqp_info->in.u.qp_create.qp = &iwqp->sc_qp;  in irdma_cqp_create_qp_cmd()
760 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,  in irdma_roce_fill_and_set_qpctx_info()  argument
763 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_roce_fill_and_set_qpctx_info()
768 	udp_info = &iwqp->udp_info;  in irdma_roce_fill_and_set_qpctx_info()
775 	roce_info = &iwqp->roce_info;  in irdma_roce_fill_and_set_qpctx_info()
788 	if (!iwqp->user_mode) {  in irdma_roce_fill_and_set_qpctx_info()
795 	ctx_info->roce_info = &iwqp->roce_info;  in irdma_roce_fill_and_set_qpctx_info()
796 	ctx_info->udp_info = &iwqp->udp_info;  in irdma_roce_fill_and_set_qpctx_info()
797 	irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_roce_fill_and_set_qpctx_info()
800 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,  in irdma_iw_fill_and_set_qpctx_info()  argument
803 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_iw_fill_and_set_qpctx_info()
807 	iwarp_info = &iwqp->iwarp_info;  in irdma_iw_fill_and_set_qpctx_info()
817 	if (!iwqp->user_mode) {  in irdma_iw_fill_and_set_qpctx_info()
824 	ctx_info->iwarp_info = &iwqp->iwarp_info;  in irdma_iw_fill_and_set_qpctx_info()
826 	irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_iw_fill_and_set_qpctx_info()
862 	struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush);  in irdma_flush_worker()  local
864 	irdma_generate_flush_completions(iwqp);  in irdma_flush_worker()
883 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_create_qp()  local
909 	qp = &iwqp->sc_qp;  in irdma_create_qp()
910 	qp->qp_uk.back_qp = iwqp;  in irdma_create_qp()
913 	iwqp->iwdev = iwdev;  in irdma_create_qp()
914 	iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE,  in irdma_create_qp()
916 	iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device,  in irdma_create_qp()
917 						 iwqp->q2_ctx_mem.size,  in irdma_create_qp()
918 						 &iwqp->q2_ctx_mem.pa,  in irdma_create_qp()
920 	if (!iwqp->q2_ctx_mem.va)  in irdma_create_qp()
923 	init_info.q2 = iwqp->q2_ctx_mem.va;  in irdma_create_qp()
924 	init_info.q2_pa = iwqp->q2_ctx_mem.pa;  in irdma_create_qp()
936 	iwqp->iwpd = iwpd;  in irdma_create_qp()
937 	iwqp->ibqp.qp_num = qp_num;  in irdma_create_qp()
938 	qp = &iwqp->sc_qp;  in irdma_create_qp()
939 	iwqp->iwscq = to_iwcq(init_attr->send_cq);  in irdma_create_qp()
940 	iwqp->iwrcq = to_iwcq(init_attr->recv_cq);  in irdma_create_qp()
941 	iwqp->host_ctx.va = init_info.host_ctx;  in irdma_create_qp()
942 	iwqp->host_ctx.pa = init_info.host_ctx_pa;  in irdma_create_qp()
943 	iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE;  in irdma_create_qp()
949 	iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;  in irdma_create_qp()
950 	init_waitqueue_head(&iwqp->waitq);  in irdma_create_qp()
951 	init_waitqueue_head(&iwqp->mod_qp_waitq);  in irdma_create_qp()
955 		err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info,  in irdma_create_qp()
958 		INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker);  in irdma_create_qp()
960 		err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr);  in irdma_create_qp()
993 	ctx_info = &iwqp->ctx_info;  in irdma_create_qp()
994 	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;  in irdma_create_qp()
995 	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;  in irdma_create_qp()
998 		irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info);  in irdma_create_qp()
1000 		irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info);  in irdma_create_qp()
1002 	err_code = irdma_cqp_create_qp_cmd(iwqp);  in irdma_create_qp()
1006 	refcount_set(&iwqp->refcnt, 1);  in irdma_create_qp()
1007 	spin_lock_init(&iwqp->lock);  in irdma_create_qp()
1008 	spin_lock_init(&iwqp->sc_qp.pfpdu.lock);  in irdma_create_qp()
1009 	iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;  in irdma_create_qp()
1010 	rf->qp_table[qp_num] = iwqp;  in irdma_create_qp()
1014 			irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp);  in irdma_create_qp()
1019 		irdma_qp_add_qos(&iwqp->sc_qp);  in irdma_create_qp()
1040 			irdma_destroy_qp(&iwqp->ibqp, udata);  in irdma_create_qp()
1045 	init_completion(&iwqp->free_qp);  in irdma_create_qp()
1049 	irdma_free_qp_rsrc(iwqp);  in irdma_create_qp()
1053 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp)  in irdma_get_ib_acc_flags()  argument
1057 	if (rdma_protocol_roce(iwqp->ibqp.device, 1)) {  in irdma_get_ib_acc_flags()
1058 		if (iwqp->roce_info.wr_rdresp_en) {  in irdma_get_ib_acc_flags()
1062 		if (iwqp->roce_info.rd_en)  in irdma_get_ib_acc_flags()
1064 		if (iwqp->roce_info.bind_en)  in irdma_get_ib_acc_flags()
1067 		if (iwqp->iwarp_info.wr_rdresp_en) {  in irdma_get_ib_acc_flags()
1071 		if (iwqp->iwarp_info.rd_en)  in irdma_get_ib_acc_flags()
1073 		if (iwqp->iwarp_info.bind_en)  in irdma_get_ib_acc_flags()
1089 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_query_qp()  local
1090 	struct irdma_sc_qp *qp = &iwqp->sc_qp;  in irdma_query_qp()
1095 	attr->qp_state = iwqp->ibqp_state;  in irdma_query_qp()
1096 	attr->cur_qp_state = iwqp->ibqp_state;  in irdma_query_qp()
1097 	attr->cap.max_send_wr = iwqp->max_send_wr;  in irdma_query_qp()
1098 	attr->cap.max_recv_wr = iwqp->max_recv_wr;  in irdma_query_qp()
1102 	attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp);  in irdma_query_qp()
1105 		attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss);  in irdma_query_qp()
1106 		attr->qkey = iwqp->roce_info.qkey;  in irdma_query_qp()
1107 		attr->rq_psn = iwqp->udp_info.epsn;  in irdma_query_qp()
1108 		attr->sq_psn = iwqp->udp_info.psn_nxt;  in irdma_query_qp()
1109 		attr->dest_qp_num = iwqp->roce_info.dest_qp;  in irdma_query_qp()
1110 		attr->pkey_index = iwqp->roce_info.p_key;  in irdma_query_qp()
1111 		attr->retry_cnt = iwqp->udp_info.rexmit_thresh;  in irdma_query_qp()
1112 		attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh;  in irdma_query_qp()
1113 		attr->max_rd_atomic = iwqp->roce_info.ord_size;  in irdma_query_qp()
1114 		attr->max_dest_rd_atomic = iwqp->roce_info.ird_size;  in irdma_query_qp()
1117 	init_attr->event_handler = iwqp->ibqp.event_handler;  in irdma_query_qp()
1118 	init_attr->qp_context = iwqp->ibqp.qp_context;  in irdma_query_qp()
1119 	init_attr->send_cq = iwqp->ibqp.send_cq;  in irdma_query_qp()
1120 	init_attr->recv_cq = iwqp->ibqp.recv_cq;  in irdma_query_qp()
1161 static int irdma_wait_for_suspend(struct irdma_qp *iwqp)  in irdma_wait_for_suspend()  argument
1163 	if (!wait_event_timeout(iwqp->iwdev->suspend_wq,  in irdma_wait_for_suspend()
1164 				!iwqp->suspend_pending,  in irdma_wait_for_suspend()
1166 		iwqp->suspend_pending = false;  in irdma_wait_for_suspend()
1167 		ibdev_warn(&iwqp->iwdev->ibdev,  in irdma_wait_for_suspend()
1169 			   iwqp->ibqp.qp_num, iwqp->last_aeq);  in irdma_wait_for_suspend()
1189 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_modify_qp_roce()  local
1190 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_modify_qp_roce()
1202 	ctx_info = &iwqp->ctx_info;  in irdma_modify_qp_roce()
1203 	roce_info = &iwqp->roce_info;  in irdma_modify_qp_roce()
1204 	udp_info = &iwqp->udp_info;  in irdma_modify_qp_roce()
1251 		struct irdma_av *av = &iwqp->roce_ah.av;  in irdma_modify_qp_roce()
1257 		memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah));  in irdma_modify_qp_roce()
1266 			irdma_qp_rem_qos(&iwqp->sc_qp);  in irdma_modify_qp_roce()
1267 			dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri);  in irdma_modify_qp_roce()
1268 			if (iwqp->sc_qp.vsi->dscp_mode)  in irdma_modify_qp_roce()
1270 				iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(udp_info->tos)];  in irdma_modify_qp_roce()
1280 		if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri))  in irdma_modify_qp_roce()
1282 		iwqp->sc_qp.user_pri = ctx_info->user_pri;  in irdma_modify_qp_roce()
1283 		irdma_qp_add_qos(&iwqp->sc_qp);  in irdma_modify_qp_roce()
1367 	wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend));  in irdma_modify_qp_roce()
1372 		  iwqp->ibqp_state, iwqp->iwarp_state, attr_mask);  in irdma_modify_qp_roce()
1374 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1376 		if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state,  in irdma_modify_qp_roce()
1377 					iwqp->ibqp.qp_type, attr_mask)) {  in irdma_modify_qp_roce()
1379 				   iwqp->ibqp.qp_num, iwqp->ibqp_state,  in irdma_modify_qp_roce()
1384 		info.curr_iwarp_state = iwqp->iwarp_state;  in irdma_modify_qp_roce()
1388 			if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {  in irdma_modify_qp_roce()
1393 			if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {  in irdma_modify_qp_roce()
1399 			if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {  in irdma_modify_qp_roce()
1409 			if (iwqp->ibqp_state < IB_QPS_RTR ||  in irdma_modify_qp_roce()
1410 			    iwqp->ibqp_state == IB_QPS_ERR) {  in irdma_modify_qp_roce()
1421 			    iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp_roce()
1423 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1424 				irdma_alloc_push_page(iwqp);  in irdma_modify_qp_roce()
1425 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1429 			if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD)  in irdma_modify_qp_roce()
1432 			if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) {  in irdma_modify_qp_roce()
1439 			iwqp->suspend_pending = true;  in irdma_modify_qp_roce()
1444 			if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {  in irdma_modify_qp_roce()
1445 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1451 					irdma_flush_wqes(iwqp,  in irdma_modify_qp_roce()
1467 		iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp_roce()
1470 	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp_roce()
1471 	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp_roce()
1472 	irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_modify_qp_roce()
1473 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1478 			if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))  in irdma_modify_qp_roce()
1481 				ret = irdma_wait_for_suspend(iwqp);  in irdma_modify_qp_roce()
1485 			spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1486 			if (iwqp->iwarp_state == info.curr_iwarp_state) {  in irdma_modify_qp_roce()
1487 				iwqp->iwarp_state = info.next_iwarp_state;  in irdma_modify_qp_roce()
1488 				iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp_roce()
1490 			if (iwqp->ibqp_state > IB_QPS_RTS &&  in irdma_modify_qp_roce()
1491 			    !iwqp->flush_issued) {  in irdma_modify_qp_roce()
1492 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1493 				irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ |  in irdma_modify_qp_roce()
1496 				iwqp->flush_issued = 1;  in irdma_modify_qp_roce()
1498 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1501 			iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp_roce()
1508 			if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp_roce()
1509 			    !iwqp->push_wqe_mmap_entry &&  in irdma_modify_qp_roce()
1510 			    !irdma_setup_push_mmap_entries(ucontext, iwqp,  in irdma_modify_qp_roce()
1513 				uresp.push_offset = iwqp->sc_qp.push_offset;  in irdma_modify_qp_roce()
1518 				irdma_remove_push_mmap_entries(iwqp);  in irdma_modify_qp_roce()
1528 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1545 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_modify_qp()  local
1546 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_modify_qp()
1569 	ctx_info = &iwqp->ctx_info;  in irdma_modify_qp()
1570 	offload_info = &iwqp->iwarp_info;  in irdma_modify_qp()
1571 	tcp_info = &iwqp->tcp_info;  in irdma_modify_qp()
1572 	wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend));  in irdma_modify_qp()
1576 		  iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq,  in irdma_modify_qp()
1577 		  iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask);  in irdma_modify_qp()
1579 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1581 		info.curr_iwarp_state = iwqp->iwarp_state;  in irdma_modify_qp()
1585 			if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {  in irdma_modify_qp()
1590 			if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {  in irdma_modify_qp()
1595 			    iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp()
1597 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1598 				irdma_alloc_push_page(iwqp);  in irdma_modify_qp()
1599 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1603 			if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS ||  in irdma_modify_qp()
1604 			    !iwqp->cm_id) {  in irdma_modify_qp()
1610 			iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED;  in irdma_modify_qp()
1611 			iwqp->hte_added = 1;  in irdma_modify_qp()
1619 			if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) {  in irdma_modify_qp()
1624 			if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING ||  in irdma_modify_qp()
1625 			    iwqp->iwarp_state < IRDMA_QP_STATE_RTS) {  in irdma_modify_qp()
1630 			if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) {  in irdma_modify_qp()
1639 			if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) {  in irdma_modify_qp()
1649 			if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {  in irdma_modify_qp()
1650 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1656 					irdma_flush_wqes(iwqp,  in irdma_modify_qp()
1664 			if (iwqp->sc_qp.term_flags) {  in irdma_modify_qp()
1665 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1666 				irdma_terminate_del_timer(&iwqp->sc_qp);  in irdma_modify_qp()
1667 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1670 			if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED &&  in irdma_modify_qp()
1672 			    iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT)  in irdma_modify_qp()
1685 		iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp()
1698 		ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp()
1699 		ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp()
1700 		irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_modify_qp()
1702 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1707 			if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))  in irdma_modify_qp()
1711 		spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1712 		if (iwqp->iwarp_state == info.curr_iwarp_state) {  in irdma_modify_qp()
1713 			iwqp->iwarp_state = info.next_iwarp_state;  in irdma_modify_qp()
1714 			iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp()
1716 		spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1719 	if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) {  in irdma_modify_qp()
1721 			if (iwqp->hw_tcp_state) {  in irdma_modify_qp()
1722 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1723 				iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;  in irdma_modify_qp()
1724 				iwqp->last_aeq = IRDMA_AE_RESET_SENT;  in irdma_modify_qp()
1725 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1727 			irdma_cm_disconn(iwqp);  in irdma_modify_qp()
1733 			if (iwqp->cm_node) {  in irdma_modify_qp()
1734 				refcount_inc(&iwqp->cm_node->refcnt);  in irdma_modify_qp()
1736 				close_timer_started = atomic_inc_return(&iwqp->close_timer_started);  in irdma_modify_qp()
1737 				if (iwqp->cm_id && close_timer_started == 1)  in irdma_modify_qp()
1738 					irdma_schedule_cm_timer(iwqp->cm_node,  in irdma_modify_qp()
1739 						(struct irdma_puda_buf *)iwqp,  in irdma_modify_qp()
1742 				irdma_rem_ref_cm_node(iwqp->cm_node);  in irdma_modify_qp()
1754 		if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp()
1755 		    !iwqp->push_wqe_mmap_entry &&  in irdma_modify_qp()
1756 		    !irdma_setup_push_mmap_entries(ucontext, iwqp,  in irdma_modify_qp()
1759 			uresp.push_offset = iwqp->sc_qp.push_offset;  in irdma_modify_qp()
1765 			irdma_remove_push_mmap_entries(iwqp);  in irdma_modify_qp()
1774 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
3432 	struct irdma_qp *iwqp;  in irdma_post_send()  local
3441 	iwqp = to_iwqp(ibqp);  in irdma_post_send()
3442 	ukqp = &iwqp->sc_qp.qp_uk;  in irdma_post_send()
3443 	dev = &iwqp->iwdev->rf->sc_dev;  in irdma_post_send()
3445 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_post_send()
3450 		if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all)  in irdma_post_send()
3482 			if (iwqp->ibqp.qp_type == IB_QPT_UD ||  in irdma_post_send()
3483 			    iwqp->ibqp.qp_type == IB_QPT_GSI) {  in irdma_post_send()
3562 			err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,  in irdma_post_send()
3568 			ibdev_dbg(&iwqp->iwdev->ibdev,  in irdma_post_send()
3579 	if (!iwqp->flush_issued) {  in irdma_post_send()
3580 		if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS)  in irdma_post_send()
3582 		spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_post_send()
3584 		spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_post_send()
3585 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_post_send()
3604 	struct irdma_qp *iwqp;  in irdma_post_recv()  local
3610 	iwqp = to_iwqp(ibqp);  in irdma_post_recv()
3611 	ukqp = &iwqp->sc_qp.qp_uk;  in irdma_post_recv()
3613 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_post_recv()
3620 			ibdev_dbg(&iwqp->iwdev->ibdev,  in irdma_post_recv()
3629 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_post_recv()
3630 	if (iwqp->flush_issued)  in irdma_post_recv()
3631 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_post_recv()
4188 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_attach_mcast()  local
4189 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_attach_mcast()
4267 			iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle;  in irdma_attach_mcast()
4280 	mcg_info.qp_id = iwqp->ibqp.qp_num;  in irdma_attach_mcast()
4328 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_detach_mcast()  local
4329 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_detach_mcast()
4354 	mcg_info.qp_id = iwqp->ibqp.qp_num;  in irdma_detach_mcast()