14 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
17 static inline void retry_first_write_send(struct rxe_qp *qp,  in retry_first_write_send()  argument
23 		int to_send = (wqe->dma.resid > qp->mtu) ?  in retry_first_write_send()
24 				qp->mtu : wqe->dma.resid;  in retry_first_write_send()
26 		qp->req.opcode = next_opcode(qp, wqe,  in retry_first_write_send()
38 static void req_retry(struct rxe_qp *qp)  in req_retry()  argument
45 	struct rxe_queue *q = qp->sq.queue;  in req_retry()
52 	qp->req.wqe_index	= cons;  in req_retry()
53 	qp->req.psn		= qp->comp.psn;  in req_retry()
54 	qp->req.opcode		= -1;  in req_retry()
58 		wqe = queue_addr_from_index(qp->sq.queue, wqe_index);  in req_retry()
59 		mask = wr_opcode_mask(wqe->wr.opcode, qp);  in req_retry()
83 				npsn = (qp->comp.psn - wqe->first_psn) &  in req_retry()
85 				retry_first_write_send(qp, wqe, npsn);  in req_retry()
90 					qp->mtu;  in req_retry()
91 				wqe->iova += npsn * qp->mtu;  in req_retry()
101 	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);  in rnr_nak_timer()  local
104 	rxe_dbg_qp(qp, "nak timer fired\n");  in rnr_nak_timer()
106 	spin_lock_irqsave(&qp->state_lock, flags);  in rnr_nak_timer()
107 	if (qp->valid) {  in rnr_nak_timer()
109 		qp->req.need_retry = 1;  in rnr_nak_timer()
110 		qp->req.wait_for_rnr_timer = 0;  in rnr_nak_timer()
111 		rxe_sched_task(&qp->send_task);  in rnr_nak_timer()
113 	spin_unlock_irqrestore(&qp->state_lock, flags);  in rnr_nak_timer()
116 static void req_check_sq_drain_done(struct rxe_qp *qp)  in req_check_sq_drain_done()  argument
124 	spin_lock_irqsave(&qp->state_lock, flags);  in req_check_sq_drain_done()
125 	if (qp_state(qp) == IB_QPS_SQD) {  in req_check_sq_drain_done()
126 		q = qp->sq.queue;  in req_check_sq_drain_done()
127 		index = qp->req.wqe_index;  in req_check_sq_drain_done()
135 			if (!qp->attr.sq_draining)  in req_check_sq_drain_done()
144 			qp->attr.sq_draining = 0;  in req_check_sq_drain_done()
145 			spin_unlock_irqrestore(&qp->state_lock, flags);  in req_check_sq_drain_done()
147 			if (qp->ibqp.event_handler) {  in req_check_sq_drain_done()
150 				ev.device = qp->ibqp.device;  in req_check_sq_drain_done()
151 				ev.element.qp = &qp->ibqp;  in req_check_sq_drain_done()
153 				qp->ibqp.event_handler(&ev,  in req_check_sq_drain_done()
154 					qp->ibqp.qp_context);  in req_check_sq_drain_done()
159 	spin_unlock_irqrestore(&qp->state_lock, flags);  in req_check_sq_drain_done()
162 static struct rxe_send_wqe *__req_next_wqe(struct rxe_qp *qp)  in __req_next_wqe()  argument
164 	struct rxe_queue *q = qp->sq.queue;  in __req_next_wqe()
165 	unsigned int index = qp->req.wqe_index;  in __req_next_wqe()
175 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)  in req_next_wqe()  argument
180 	req_check_sq_drain_done(qp);  in req_next_wqe()
182 	wqe = __req_next_wqe(qp);  in req_next_wqe()
186 	spin_lock_irqsave(&qp->state_lock, flags);  in req_next_wqe()
187 	if (unlikely((qp_state(qp) == IB_QPS_SQD) &&  in req_next_wqe()
189 		spin_unlock_irqrestore(&qp->state_lock, flags);  in req_next_wqe()
192 	spin_unlock_irqrestore(&qp->state_lock, flags);  in req_next_wqe()
194 	wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);  in req_next_wqe()
200  * @qp: the queue pair
206 static int rxe_wqe_is_fenced(struct rxe_qp *qp, struct rxe_send_wqe *wqe)  in rxe_wqe_is_fenced()  argument
213 		return qp->req.wqe_index != queue_get_consumer(qp->sq.queue,  in rxe_wqe_is_fenced()
221 		atomic_read(&qp->req.rd_atomic) != qp->attr.max_rd_atomic;  in rxe_wqe_is_fenced()
224 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits)  in next_opcode_rc()  argument
228 		if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||  in next_opcode_rc()
229 		    qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)  in next_opcode_rc()
239 		if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||  in next_opcode_rc()
240 		    qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)  in next_opcode_rc()
250 		if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||  in next_opcode_rc()
251 		    qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)  in next_opcode_rc()
261 		if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||  in next_opcode_rc()
262 		    qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)  in next_opcode_rc()
284 		if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||  in next_opcode_rc()
285 		    qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)  in next_opcode_rc()
303 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits)  in next_opcode_uc()  argument
307 		if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||  in next_opcode_uc()
308 		    qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)  in next_opcode_uc()
318 		if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||  in next_opcode_uc()
319 		    qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)  in next_opcode_uc()
329 		if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||  in next_opcode_uc()
330 		    qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)  in next_opcode_uc()
340 		if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||  in next_opcode_uc()
341 		    qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)  in next_opcode_uc()
354 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,  in next_opcode()  argument
357 	int fits = (wqe->dma.resid <= qp->mtu);  in next_opcode()
359 	switch (qp_type(qp)) {  in next_opcode()
361 		return next_opcode_rc(qp, opcode, fits);  in next_opcode()
364 		return next_opcode_uc(qp, opcode, fits);  in next_opcode()
384 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe)  in check_init_depth()  argument
391 	qp->req.need_rd_atomic = 1;  in check_init_depth()
392 	depth = atomic_dec_return(&qp->req.rd_atomic);  in check_init_depth()
395 		qp->req.need_rd_atomic = 0;  in check_init_depth()
400 	atomic_inc(&qp->req.rd_atomic);  in check_init_depth()
404 static inline int get_mtu(struct rxe_qp *qp)  in get_mtu()  argument
406 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);  in get_mtu()
408 	if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC))  in get_mtu()
409 		return qp->mtu;  in get_mtu()
414 static struct sk_buff *init_req_packet(struct rxe_qp *qp,  in init_req_packet()  argument
420 	struct rxe_dev		*rxe = to_rdev(qp->ibqp.device);  in init_req_packet()
446 					 qp->attr.dest_qp_num;  in init_req_packet()
448 	if (qp_type(qp) != IB_QPT_UD && qp_type(qp) != IB_QPT_UC)  in init_req_packet()
450 			   (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK));  in init_req_packet()
452 		qp->req.noack_pkts = 0;  in init_req_packet()
489 		if (qp->ibqp.qp_num == 1)  in init_req_packet()
493 		deth_set_sqp(pkt, qp->ibqp.qp_num);  in init_req_packet()
499 static int finish_packet(struct rxe_qp *qp, struct rxe_av *av,  in finish_packet()  argument
518 			err = copy_data(qp->pd, 0, &wqe->dma,  in finish_packet()
542 static void update_wqe_state(struct rxe_qp *qp,  in update_wqe_state()  argument
547 		if (qp_type(qp) == IB_QPT_RC)  in update_wqe_state()
556 static void update_wqe_psn(struct rxe_qp *qp,  in update_wqe_psn()  argument
562 	int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu;  in update_wqe_psn()
569 		wqe->first_psn = qp->req.psn;  in update_wqe_psn()
570 		wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK;  in update_wqe_psn()
574 		qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK;  in update_wqe_psn()
576 		qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;  in update_wqe_psn()
579 static void update_state(struct rxe_qp *qp, struct rxe_pkt_info *pkt)  in update_state()  argument
581 	qp->req.opcode = pkt->opcode;  in update_state()
584 		qp->req.wqe_index = queue_next_index(qp->sq.queue,  in update_state()
585 						     qp->req.wqe_index);  in update_state()
587 	qp->need_req_skb = 0;  in update_state()
589 	if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer))  in update_state()
590 		mod_timer(&qp->retrans_timer,  in update_state()
591 			  jiffies + qp->qp_timeout_jiffies);  in update_state()
594 static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)  in rxe_do_local_ops()  argument
604 			ret = rxe_invalidate_mw(qp, rkey);  in rxe_do_local_ops()
606 			ret = rxe_invalidate_mr(qp, rkey);  in rxe_do_local_ops()
614 		ret = rxe_reg_fast_mr(qp, wqe);  in rxe_do_local_ops()
621 		ret = rxe_bind_mw(qp, wqe);  in rxe_do_local_ops()
628 		rxe_dbg_qp(qp, "Unexpected send wqe opcode %d\n", opcode);  in rxe_do_local_ops()
635 	qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index);  in rxe_do_local_ops()
640 int rxe_requester(struct rxe_qp *qp)  in rxe_requester()  argument
642 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);  in rxe_requester()
652 	struct rxe_queue *q = qp->sq.queue;  in rxe_requester()
657 	spin_lock_irqsave(&qp->state_lock, flags);  in rxe_requester()
658 	if (unlikely(!qp->valid)) {  in rxe_requester()
659 		spin_unlock_irqrestore(&qp->state_lock, flags);  in rxe_requester()
663 	if (unlikely(qp_state(qp) == IB_QPS_ERR)) {  in rxe_requester()
664 		wqe = __req_next_wqe(qp);  in rxe_requester()
665 		spin_unlock_irqrestore(&qp->state_lock, flags);  in rxe_requester()
672 	if (unlikely(qp_state(qp) == IB_QPS_RESET)) {  in rxe_requester()
673 		qp->req.wqe_index = queue_get_consumer(q,  in rxe_requester()
675 		qp->req.opcode = -1;  in rxe_requester()
676 		qp->req.need_rd_atomic = 0;  in rxe_requester()
677 		qp->req.wait_psn = 0;  in rxe_requester()
678 		qp->req.need_retry = 0;  in rxe_requester()
679 		qp->req.wait_for_rnr_timer = 0;  in rxe_requester()
680 		spin_unlock_irqrestore(&qp->state_lock, flags);  in rxe_requester()
683 	spin_unlock_irqrestore(&qp->state_lock, flags);  in rxe_requester()
691 	if (unlikely(qp->req.need_retry && !qp->req.wait_for_rnr_timer)) {  in rxe_requester()
692 		req_retry(qp);  in rxe_requester()
693 		qp->req.need_retry = 0;  in rxe_requester()
696 	wqe = req_next_wqe(qp);  in rxe_requester()
700 	if (rxe_wqe_is_fenced(qp, wqe)) {  in rxe_requester()
701 		qp->req.wait_fence = 1;  in rxe_requester()
706 		err = rxe_do_local_ops(qp, wqe);  in rxe_requester()
713 	if (unlikely(qp_type(qp) == IB_QPT_RC &&  in rxe_requester()
714 		psn_compare(qp->req.psn, (qp->comp.psn +  in rxe_requester()
716 		qp->req.wait_psn = 1;  in rxe_requester()
720 	/* Limit the number of inflight SKBs per QP */  in rxe_requester()
721 	if (unlikely(atomic_read(&qp->skb_out) >  in rxe_requester()
723 		qp->need_req_skb = 1;  in rxe_requester()
727 	opcode = next_opcode(qp, wqe, wqe->wr.opcode);  in rxe_requester()
736 		if (check_init_depth(qp, wqe))  in rxe_requester()
740 	mtu = get_mtu(qp);  in rxe_requester()
744 		if (qp_type(qp) == IB_QPT_UD) {  in rxe_requester()
752 			wqe->first_psn = qp->req.psn;  in rxe_requester()
753 			wqe->last_psn = qp->req.psn;  in rxe_requester()
754 			qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;  in rxe_requester()
755 			qp->req.opcode = IB_OPCODE_UD_SEND_ONLY;  in rxe_requester()
756 			qp->req.wqe_index = queue_next_index(qp->sq.queue,  in rxe_requester()
757 						       qp->req.wqe_index);  in rxe_requester()
767 	pkt.qp = qp;  in rxe_requester()
768 	pkt.psn = qp->req.psn;  in rxe_requester()
774 		rxe_dbg_qp(qp, "Failed no address vector\n");  in rxe_requester()
779 	skb = init_req_packet(qp, av, wqe, opcode, payload, &pkt);  in rxe_requester()
781 		rxe_dbg_qp(qp, "Failed allocating skb\n");  in rxe_requester()
788 	err = finish_packet(qp, av, wqe, &pkt, skb, payload);  in rxe_requester()
790 		rxe_dbg_qp(qp, "Error during finish packet\n");  in rxe_requester()
804 	err = rxe_xmit_packet(qp, &pkt, skb);  in rxe_requester()
810 	update_wqe_state(qp, wqe, &pkt);  in rxe_requester()
811 	update_wqe_psn(qp, wqe, &pkt, payload);  in rxe_requester()
812 	update_state(qp, &pkt);  in rxe_requester()
823 	qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index);  in rxe_requester()
825 	rxe_qp_error(qp);  in rxe_requester()
832 int rxe_sender(struct rxe_qp *qp)  in rxe_sender()  argument
838 	req_ret = rxe_requester(qp);  in rxe_sender()
841 	comp_ret = rxe_completer(qp);  in rxe_sender()