1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #include <linux/skbuff.h>
8
9 #include "rxe.h"
10 #include "rxe_loc.h"
11 #include "rxe_queue.h"
12
13 static char *resp_state_name[] = {
14 [RESPST_NONE] = "NONE",
15 [RESPST_GET_REQ] = "GET_REQ",
16 [RESPST_CHK_PSN] = "CHK_PSN",
17 [RESPST_CHK_OP_SEQ] = "CHK_OP_SEQ",
18 [RESPST_CHK_OP_VALID] = "CHK_OP_VALID",
19 [RESPST_CHK_RESOURCE] = "CHK_RESOURCE",
20 [RESPST_CHK_LENGTH] = "CHK_LENGTH",
21 [RESPST_CHK_RKEY] = "CHK_RKEY",
22 [RESPST_EXECUTE] = "EXECUTE",
23 [RESPST_READ_REPLY] = "READ_REPLY",
24 [RESPST_ATOMIC_REPLY] = "ATOMIC_REPLY",
25 [RESPST_ATOMIC_WRITE_REPLY] = "ATOMIC_WRITE_REPLY",
26 [RESPST_PROCESS_FLUSH] = "PROCESS_FLUSH",
27 [RESPST_COMPLETE] = "COMPLETE",
28 [RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE",
29 [RESPST_CLEANUP] = "CLEANUP",
30 [RESPST_DUPLICATE_REQUEST] = "DUPLICATE_REQUEST",
31 [RESPST_ERR_MALFORMED_WQE] = "ERR_MALFORMED_WQE",
32 [RESPST_ERR_UNSUPPORTED_OPCODE] = "ERR_UNSUPPORTED_OPCODE",
33 [RESPST_ERR_MISALIGNED_ATOMIC] = "ERR_MISALIGNED_ATOMIC",
34 [RESPST_ERR_PSN_OUT_OF_SEQ] = "ERR_PSN_OUT_OF_SEQ",
35 [RESPST_ERR_MISSING_OPCODE_FIRST] = "ERR_MISSING_OPCODE_FIRST",
36 [RESPST_ERR_MISSING_OPCODE_LAST_C] = "ERR_MISSING_OPCODE_LAST_C",
37 [RESPST_ERR_MISSING_OPCODE_LAST_D1E] = "ERR_MISSING_OPCODE_LAST_D1E",
38 [RESPST_ERR_TOO_MANY_RDMA_ATM_REQ] = "ERR_TOO_MANY_RDMA_ATM_REQ",
39 [RESPST_ERR_RNR] = "ERR_RNR",
40 [RESPST_ERR_RKEY_VIOLATION] = "ERR_RKEY_VIOLATION",
41 [RESPST_ERR_INVALIDATE_RKEY] = "ERR_INVALIDATE_RKEY_VIOLATION",
42 [RESPST_ERR_LENGTH] = "ERR_LENGTH",
43 [RESPST_ERR_CQ_OVERFLOW] = "ERR_CQ_OVERFLOW",
44 [RESPST_ERROR] = "ERROR",
45 [RESPST_DONE] = "DONE",
46 [RESPST_EXIT] = "EXIT",
47 };
48
49 /* rxe_recv calls here to add a request packet to the input queue */
rxe_resp_queue_pkt(struct rxe_qp * qp,struct sk_buff * skb)50 void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
51 {
52 skb_queue_tail(&qp->req_pkts, skb);
53 rxe_sched_task(&qp->recv_task);
54 }
55
get_req(struct rxe_qp * qp,struct rxe_pkt_info ** pkt_p)56 static inline enum resp_states get_req(struct rxe_qp *qp,
57 struct rxe_pkt_info **pkt_p)
58 {
59 struct sk_buff *skb;
60
61 skb = skb_peek(&qp->req_pkts);
62 if (!skb)
63 return RESPST_EXIT;
64
65 *pkt_p = SKB_TO_PKT(skb);
66
67 return (qp->resp.res) ? RESPST_READ_REPLY : RESPST_CHK_PSN;
68 }
69
check_psn(struct rxe_qp * qp,struct rxe_pkt_info * pkt)70 static enum resp_states check_psn(struct rxe_qp *qp,
71 struct rxe_pkt_info *pkt)
72 {
73 int diff = psn_compare(pkt->psn, qp->resp.psn);
74 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
75
76 switch (qp_type(qp)) {
77 case IB_QPT_RC:
78 if (diff > 0) {
79 if (qp->resp.sent_psn_nak)
80 return RESPST_CLEANUP;
81
82 qp->resp.sent_psn_nak = 1;
83 rxe_counter_inc(rxe, RXE_CNT_OUT_OF_SEQ_REQ);
84 return RESPST_ERR_PSN_OUT_OF_SEQ;
85
86 } else if (diff < 0) {
87 rxe_counter_inc(rxe, RXE_CNT_DUP_REQ);
88 return RESPST_DUPLICATE_REQUEST;
89 }
90
91 if (qp->resp.sent_psn_nak)
92 qp->resp.sent_psn_nak = 0;
93
94 break;
95
96 case IB_QPT_UC:
97 if (qp->resp.drop_msg || diff != 0) {
98 if (pkt->mask & RXE_START_MASK) {
99 qp->resp.drop_msg = 0;
100 return RESPST_CHK_OP_SEQ;
101 }
102
103 qp->resp.drop_msg = 1;
104 return RESPST_CLEANUP;
105 }
106 break;
107 default:
108 break;
109 }
110
111 return RESPST_CHK_OP_SEQ;
112 }
113
check_op_seq(struct rxe_qp * qp,struct rxe_pkt_info * pkt)114 static enum resp_states check_op_seq(struct rxe_qp *qp,
115 struct rxe_pkt_info *pkt)
116 {
117 switch (qp_type(qp)) {
118 case IB_QPT_RC:
119 switch (qp->resp.opcode) {
120 case IB_OPCODE_RC_SEND_FIRST:
121 case IB_OPCODE_RC_SEND_MIDDLE:
122 switch (pkt->opcode) {
123 case IB_OPCODE_RC_SEND_MIDDLE:
124 case IB_OPCODE_RC_SEND_LAST:
125 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:
126 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:
127 return RESPST_CHK_OP_VALID;
128 default:
129 return RESPST_ERR_MISSING_OPCODE_LAST_C;
130 }
131
132 case IB_OPCODE_RC_RDMA_WRITE_FIRST:
133 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
134 switch (pkt->opcode) {
135 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
136 case IB_OPCODE_RC_RDMA_WRITE_LAST:
137 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
138 return RESPST_CHK_OP_VALID;
139 default:
140 return RESPST_ERR_MISSING_OPCODE_LAST_C;
141 }
142
143 default:
144 switch (pkt->opcode) {
145 case IB_OPCODE_RC_SEND_MIDDLE:
146 case IB_OPCODE_RC_SEND_LAST:
147 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:
148 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:
149 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
150 case IB_OPCODE_RC_RDMA_WRITE_LAST:
151 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
152 return RESPST_ERR_MISSING_OPCODE_FIRST;
153 default:
154 return RESPST_CHK_OP_VALID;
155 }
156 }
157 break;
158
159 case IB_QPT_UC:
160 switch (qp->resp.opcode) {
161 case IB_OPCODE_UC_SEND_FIRST:
162 case IB_OPCODE_UC_SEND_MIDDLE:
163 switch (pkt->opcode) {
164 case IB_OPCODE_UC_SEND_MIDDLE:
165 case IB_OPCODE_UC_SEND_LAST:
166 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:
167 return RESPST_CHK_OP_VALID;
168 default:
169 return RESPST_ERR_MISSING_OPCODE_LAST_D1E;
170 }
171
172 case IB_OPCODE_UC_RDMA_WRITE_FIRST:
173 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
174 switch (pkt->opcode) {
175 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
176 case IB_OPCODE_UC_RDMA_WRITE_LAST:
177 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
178 return RESPST_CHK_OP_VALID;
179 default:
180 return RESPST_ERR_MISSING_OPCODE_LAST_D1E;
181 }
182
183 default:
184 switch (pkt->opcode) {
185 case IB_OPCODE_UC_SEND_MIDDLE:
186 case IB_OPCODE_UC_SEND_LAST:
187 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:
188 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
189 case IB_OPCODE_UC_RDMA_WRITE_LAST:
190 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
191 qp->resp.drop_msg = 1;
192 return RESPST_CLEANUP;
193 default:
194 return RESPST_CHK_OP_VALID;
195 }
196 }
197 break;
198
199 default:
200 return RESPST_CHK_OP_VALID;
201 }
202 }
203
check_qp_attr_access(struct rxe_qp * qp,struct rxe_pkt_info * pkt)204 static bool check_qp_attr_access(struct rxe_qp *qp,
205 struct rxe_pkt_info *pkt)
206 {
207 if (((pkt->mask & RXE_READ_MASK) &&
208 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) ||
209 ((pkt->mask & (RXE_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) &&
210 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) ||
211 ((pkt->mask & RXE_ATOMIC_MASK) &&
212 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
213 return false;
214
215 if (pkt->mask & RXE_FLUSH_MASK) {
216 u32 flush_type = feth_plt(pkt);
217
218 if ((flush_type & IB_FLUSH_GLOBAL &&
219 !(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_GLOBAL)) ||
220 (flush_type & IB_FLUSH_PERSISTENT &&
221 !(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_PERSISTENT)))
222 return false;
223 }
224
225 return true;
226 }
227
check_op_valid(struct rxe_qp * qp,struct rxe_pkt_info * pkt)228 static enum resp_states check_op_valid(struct rxe_qp *qp,
229 struct rxe_pkt_info *pkt)
230 {
231 switch (qp_type(qp)) {
232 case IB_QPT_RC:
233 if (!check_qp_attr_access(qp, pkt))
234 return RESPST_ERR_UNSUPPORTED_OPCODE;
235
236 break;
237
238 case IB_QPT_UC:
239 if ((pkt->mask & RXE_WRITE_MASK) &&
240 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) {
241 qp->resp.drop_msg = 1;
242 return RESPST_CLEANUP;
243 }
244
245 break;
246
247 case IB_QPT_UD:
248 case IB_QPT_GSI:
249 break;
250
251 default:
252 WARN_ON_ONCE(1);
253 break;
254 }
255
256 return RESPST_CHK_RESOURCE;
257 }
258
get_srq_wqe(struct rxe_qp * qp)259 static enum resp_states get_srq_wqe(struct rxe_qp *qp)
260 {
261 struct rxe_srq *srq = qp->srq;
262 struct rxe_queue *q = srq->rq.queue;
263 struct rxe_recv_wqe *wqe;
264 struct ib_event ev;
265 unsigned int count;
266 size_t size;
267 unsigned long flags;
268
269 if (srq->error)
270 return RESPST_ERR_RNR;
271
272 spin_lock_irqsave(&srq->rq.consumer_lock, flags);
273
274 wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT);
275 if (!wqe) {
276 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
277 return RESPST_ERR_RNR;
278 }
279
280 /* don't trust user space data */
281 if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) {
282 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
283 rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n");
284 return RESPST_ERR_MALFORMED_WQE;
285 }
286 size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);
287 memcpy(&qp->resp.srq_wqe, wqe, size);
288
289 qp->resp.wqe = &qp->resp.srq_wqe.wqe;
290 queue_advance_consumer(q, QUEUE_TYPE_FROM_CLIENT);
291 count = queue_count(q, QUEUE_TYPE_FROM_CLIENT);
292
293 if (srq->limit && srq->ibsrq.event_handler && (count < srq->limit)) {
294 srq->limit = 0;
295 goto event;
296 }
297
298 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
299 return RESPST_CHK_LENGTH;
300
301 event:
302 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags);
303 ev.device = qp->ibqp.device;
304 ev.element.srq = qp->ibqp.srq;
305 ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
306 srq->ibsrq.event_handler(&ev, srq->ibsrq.srq_context);
307 return RESPST_CHK_LENGTH;
308 }
309
check_resource(struct rxe_qp * qp,struct rxe_pkt_info * pkt)310 static enum resp_states check_resource(struct rxe_qp *qp,
311 struct rxe_pkt_info *pkt)
312 {
313 struct rxe_srq *srq = qp->srq;
314
315 if (pkt->mask & (RXE_READ_OR_ATOMIC_MASK | RXE_ATOMIC_WRITE_MASK)) {
316 /* it is the requesters job to not send
317 * too many read/atomic ops, we just
318 * recycle the responder resource queue
319 */
320 if (likely(qp->attr.max_dest_rd_atomic > 0))
321 return RESPST_CHK_LENGTH;
322 else
323 return RESPST_ERR_TOO_MANY_RDMA_ATM_REQ;
324 }
325
326 if (pkt->mask & RXE_RWR_MASK) {
327 if (srq)
328 return get_srq_wqe(qp);
329
330 qp->resp.wqe = queue_head(qp->rq.queue,
331 QUEUE_TYPE_FROM_CLIENT);
332 return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR;
333 }
334
335 return RESPST_CHK_LENGTH;
336 }
337
rxe_resp_check_length(struct rxe_qp * qp,struct rxe_pkt_info * pkt)338 static enum resp_states rxe_resp_check_length(struct rxe_qp *qp,
339 struct rxe_pkt_info *pkt)
340 {
341 /*
342 * See IBA C9-92
343 * For UD QPs we only check if the packet will fit in the
344 * receive buffer later. For RDMA operations additional
345 * length checks are performed in check_rkey.
346 */
347 if ((qp_type(qp) == IB_QPT_GSI) || (qp_type(qp) == IB_QPT_UD)) {
348 unsigned int payload = payload_size(pkt);
349 unsigned int recv_buffer_len = 0;
350 int i;
351
352 for (i = 0; i < qp->resp.wqe->dma.num_sge; i++)
353 recv_buffer_len += qp->resp.wqe->dma.sge[i].length;
354 if (payload + sizeof(union rdma_network_hdr) > recv_buffer_len) {
355 rxe_dbg_qp(qp, "The receive buffer is too small for this UD packet.\n");
356 return RESPST_ERR_LENGTH;
357 }
358 }
359
360 if (pkt->mask & RXE_PAYLOAD_MASK && ((qp_type(qp) == IB_QPT_RC) ||
361 (qp_type(qp) == IB_QPT_UC))) {
362 unsigned int mtu = qp->mtu;
363 unsigned int payload = payload_size(pkt);
364
365 if ((pkt->mask & RXE_START_MASK) &&
366 (pkt->mask & RXE_END_MASK)) {
367 if (unlikely(payload > mtu)) {
368 rxe_dbg_qp(qp, "only packet too long\n");
369 return RESPST_ERR_LENGTH;
370 }
371 } else if ((pkt->mask & RXE_START_MASK) ||
372 (pkt->mask & RXE_MIDDLE_MASK)) {
373 if (unlikely(payload != mtu)) {
374 rxe_dbg_qp(qp, "first or middle packet not mtu\n");
375 return RESPST_ERR_LENGTH;
376 }
377 } else if (pkt->mask & RXE_END_MASK) {
378 if (unlikely((payload == 0) || (payload > mtu))) {
379 rxe_dbg_qp(qp, "last packet zero or too long\n");
380 return RESPST_ERR_LENGTH;
381 }
382 }
383 }
384
385 /* See IBA C9-94 */
386 if (pkt->mask & RXE_RETH_MASK) {
387 if (reth_len(pkt) > (1U << 31)) {
388 rxe_dbg_qp(qp, "dma length too long\n");
389 return RESPST_ERR_LENGTH;
390 }
391 }
392
393 if (pkt->mask & RXE_RDMA_OP_MASK)
394 return RESPST_CHK_RKEY;
395 else
396 return RESPST_EXECUTE;
397 }
398
399 /* if the reth length field is zero we can assume nothing
400 * about the rkey value and should not validate or use it.
401 * Instead set qp->resp.rkey to 0 which is an invalid rkey
402 * value since the minimum index part is 1.
403 */
qp_resp_from_reth(struct rxe_qp * qp,struct rxe_pkt_info * pkt)404 static void qp_resp_from_reth(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
405 {
406 unsigned int length = reth_len(pkt);
407
408 qp->resp.va = reth_va(pkt);
409 qp->resp.offset = 0;
410 qp->resp.resid = length;
411 qp->resp.length = length;
412 if (pkt->mask & RXE_READ_OR_WRITE_MASK && length == 0)
413 qp->resp.rkey = 0;
414 else
415 qp->resp.rkey = reth_rkey(pkt);
416 }
417
qp_resp_from_atmeth(struct rxe_qp * qp,struct rxe_pkt_info * pkt)418 static void qp_resp_from_atmeth(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
419 {
420 qp->resp.va = atmeth_va(pkt);
421 qp->resp.offset = 0;
422 qp->resp.rkey = atmeth_rkey(pkt);
423 qp->resp.resid = sizeof(u64);
424 }
425
426 /* resolve the packet rkey to qp->resp.mr or set qp->resp.mr to NULL
427 * if an invalid rkey is received or the rdma length is zero. For middle
428 * or last packets use the stored value of mr.
429 */
check_rkey(struct rxe_qp * qp,struct rxe_pkt_info * pkt)430 static enum resp_states check_rkey(struct rxe_qp *qp,
431 struct rxe_pkt_info *pkt)
432 {
433 struct rxe_mr *mr = NULL;
434 struct rxe_mw *mw = NULL;
435 u64 va;
436 u32 rkey;
437 u32 resid;
438 u32 pktlen;
439 int mtu = qp->mtu;
440 enum resp_states state;
441 int access = 0;
442
443 /* parse RETH or ATMETH header for first/only packets
444 * for va, length, rkey, etc. or use current value for
445 * middle/last packets.
446 */
447 if (pkt->mask & (RXE_READ_OR_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) {
448 if (pkt->mask & RXE_RETH_MASK)
449 qp_resp_from_reth(qp, pkt);
450
451 access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ
452 : IB_ACCESS_REMOTE_WRITE;
453 } else if (pkt->mask & RXE_FLUSH_MASK) {
454 u32 flush_type = feth_plt(pkt);
455
456 if (pkt->mask & RXE_RETH_MASK)
457 qp_resp_from_reth(qp, pkt);
458
459 if (flush_type & IB_FLUSH_GLOBAL)
460 access |= IB_ACCESS_FLUSH_GLOBAL;
461 if (flush_type & IB_FLUSH_PERSISTENT)
462 access |= IB_ACCESS_FLUSH_PERSISTENT;
463 } else if (pkt->mask & RXE_ATOMIC_MASK) {
464 qp_resp_from_atmeth(qp, pkt);
465 access = IB_ACCESS_REMOTE_ATOMIC;
466 } else {
467 /* shouldn't happen */
468 WARN_ON(1);
469 }
470
471 /* A zero-byte read or write op is not required to
472 * set an addr or rkey. See C9-88
473 */
474 if ((pkt->mask & RXE_READ_OR_WRITE_MASK) &&
475 (pkt->mask & RXE_RETH_MASK) && reth_len(pkt) == 0) {
476 qp->resp.mr = NULL;
477 return RESPST_EXECUTE;
478 }
479
480 va = qp->resp.va;
481 rkey = qp->resp.rkey;
482 resid = qp->resp.resid;
483 pktlen = payload_size(pkt);
484
485 if (rkey_is_mw(rkey)) {
486 mw = rxe_lookup_mw(qp, access, rkey);
487 if (!mw) {
488 rxe_dbg_qp(qp, "no MW matches rkey %#x\n", rkey);
489 state = RESPST_ERR_RKEY_VIOLATION;
490 goto err;
491 }
492
493 mr = mw->mr;
494 if (!mr) {
495 rxe_dbg_qp(qp, "MW doesn't have an MR\n");
496 state = RESPST_ERR_RKEY_VIOLATION;
497 goto err;
498 }
499
500 if (mw->access & IB_ZERO_BASED)
501 qp->resp.offset = mw->addr;
502
503 rxe_get(mr);
504 rxe_put(mw);
505 mw = NULL;
506 } else {
507 mr = lookup_mr(qp->pd, access, rkey, RXE_LOOKUP_REMOTE);
508 if (!mr) {
509 rxe_dbg_qp(qp, "no MR matches rkey %#x\n", rkey);
510 state = RESPST_ERR_RKEY_VIOLATION;
511 goto err;
512 }
513 }
514
515 if (pkt->mask & RXE_FLUSH_MASK) {
516 /* FLUSH MR may not set va or resid
517 * no need to check range since we will flush whole mr
518 */
519 if (feth_sel(pkt) == IB_FLUSH_MR)
520 goto skip_check_range;
521 }
522
523 if (mr_check_range(mr, va + qp->resp.offset, resid)) {
524 state = RESPST_ERR_RKEY_VIOLATION;
525 goto err;
526 }
527
528 skip_check_range:
529 if (pkt->mask & (RXE_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) {
530 if (resid > mtu) {
531 if (pktlen != mtu || bth_pad(pkt)) {
532 state = RESPST_ERR_LENGTH;
533 goto err;
534 }
535 } else {
536 if (pktlen != resid) {
537 state = RESPST_ERR_LENGTH;
538 goto err;
539 }
540 if ((bth_pad(pkt) != (0x3 & (-resid)))) {
541 /* This case may not be exactly that
542 * but nothing else fits.
543 */
544 state = RESPST_ERR_LENGTH;
545 goto err;
546 }
547 }
548 }
549
550 WARN_ON_ONCE(qp->resp.mr);
551
552 qp->resp.mr = mr;
553 return RESPST_EXECUTE;
554
555 err:
556 qp->resp.mr = NULL;
557 if (mr)
558 rxe_put(mr);
559 if (mw)
560 rxe_put(mw);
561
562 return state;
563 }
564
send_data_in(struct rxe_qp * qp,void * data_addr,int data_len)565 static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,
566 int data_len)
567 {
568 int err;
569
570 err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
571 data_addr, data_len, RXE_TO_MR_OBJ);
572 if (unlikely(err))
573 return (err == -ENOSPC) ? RESPST_ERR_LENGTH
574 : RESPST_ERR_MALFORMED_WQE;
575
576 return RESPST_NONE;
577 }
578
write_data_in(struct rxe_qp * qp,struct rxe_pkt_info * pkt)579 static enum resp_states write_data_in(struct rxe_qp *qp,
580 struct rxe_pkt_info *pkt)
581 {
582 enum resp_states rc = RESPST_NONE;
583 int err;
584 int data_len = payload_size(pkt);
585
586 err = rxe_mr_copy(qp->resp.mr, qp->resp.va + qp->resp.offset,
587 payload_addr(pkt), data_len, RXE_TO_MR_OBJ);
588 if (err) {
589 rc = RESPST_ERR_RKEY_VIOLATION;
590 goto out;
591 }
592
593 qp->resp.va += data_len;
594 qp->resp.resid -= data_len;
595
596 out:
597 return rc;
598 }
599
rxe_prepare_res(struct rxe_qp * qp,struct rxe_pkt_info * pkt,int type)600 static struct resp_res *rxe_prepare_res(struct rxe_qp *qp,
601 struct rxe_pkt_info *pkt,
602 int type)
603 {
604 struct resp_res *res;
605 u32 pkts;
606
607 res = &qp->resp.resources[qp->resp.res_head];
608 rxe_advance_resp_resource(qp);
609 free_rd_atomic_resource(res);
610
611 res->type = type;
612 res->replay = 0;
613
614 switch (type) {
615 case RXE_READ_MASK:
616 res->read.va = qp->resp.va + qp->resp.offset;
617 res->read.va_org = qp->resp.va + qp->resp.offset;
618 res->read.resid = qp->resp.resid;
619 res->read.length = qp->resp.resid;
620 res->read.rkey = qp->resp.rkey;
621
622 pkts = max_t(u32, (reth_len(pkt) + qp->mtu - 1)/qp->mtu, 1);
623 res->first_psn = pkt->psn;
624 res->cur_psn = pkt->psn;
625 res->last_psn = (pkt->psn + pkts - 1) & BTH_PSN_MASK;
626
627 res->state = rdatm_res_state_new;
628 break;
629 case RXE_ATOMIC_MASK:
630 case RXE_ATOMIC_WRITE_MASK:
631 res->first_psn = pkt->psn;
632 res->last_psn = pkt->psn;
633 res->cur_psn = pkt->psn;
634 break;
635 case RXE_FLUSH_MASK:
636 res->flush.va = qp->resp.va + qp->resp.offset;
637 res->flush.length = qp->resp.length;
638 res->flush.type = feth_plt(pkt);
639 res->flush.level = feth_sel(pkt);
640 }
641
642 return res;
643 }
644
process_flush(struct rxe_qp * qp,struct rxe_pkt_info * pkt)645 static enum resp_states process_flush(struct rxe_qp *qp,
646 struct rxe_pkt_info *pkt)
647 {
648 u64 length, start;
649 struct rxe_mr *mr = qp->resp.mr;
650 struct resp_res *res = qp->resp.res;
651
652 /* oA19-14, oA19-15 */
653 if (res && res->replay)
654 return RESPST_ACKNOWLEDGE;
655 else if (!res) {
656 res = rxe_prepare_res(qp, pkt, RXE_FLUSH_MASK);
657 qp->resp.res = res;
658 }
659
660 if (res->flush.level == IB_FLUSH_RANGE) {
661 start = res->flush.va;
662 length = res->flush.length;
663 } else { /* level == IB_FLUSH_MR */
664 start = mr->ibmr.iova;
665 length = mr->ibmr.length;
666 }
667
668 if (res->flush.type & IB_FLUSH_PERSISTENT) {
669 if (rxe_flush_pmem_iova(mr, start, length))
670 return RESPST_ERR_RKEY_VIOLATION;
671 /* Make data persistent. */
672 wmb();
673 } else if (res->flush.type & IB_FLUSH_GLOBAL) {
674 /* Make data global visibility. */
675 wmb();
676 }
677
678 qp->resp.msn++;
679
680 /* next expected psn, read handles this separately */
681 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
682 qp->resp.ack_psn = qp->resp.psn;
683
684 qp->resp.opcode = pkt->opcode;
685 qp->resp.status = IB_WC_SUCCESS;
686
687 return RESPST_ACKNOWLEDGE;
688 }
689
atomic_reply(struct rxe_qp * qp,struct rxe_pkt_info * pkt)690 static enum resp_states atomic_reply(struct rxe_qp *qp,
691 struct rxe_pkt_info *pkt)
692 {
693 struct rxe_mr *mr = qp->resp.mr;
694 struct resp_res *res = qp->resp.res;
695 int err;
696
697 if (!res) {
698 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_MASK);
699 qp->resp.res = res;
700 }
701
702 if (!res->replay) {
703 u64 iova = qp->resp.va + qp->resp.offset;
704
705 err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode,
706 atmeth_comp(pkt),
707 atmeth_swap_add(pkt),
708 &res->atomic.orig_val);
709 if (err)
710 return err;
711
712 qp->resp.msn++;
713
714 /* next expected psn, read handles this separately */
715 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
716 qp->resp.ack_psn = qp->resp.psn;
717
718 qp->resp.opcode = pkt->opcode;
719 qp->resp.status = IB_WC_SUCCESS;
720 }
721
722 return RESPST_ACKNOWLEDGE;
723 }
724
atomic_write_reply(struct rxe_qp * qp,struct rxe_pkt_info * pkt)725 static enum resp_states atomic_write_reply(struct rxe_qp *qp,
726 struct rxe_pkt_info *pkt)
727 {
728 struct resp_res *res = qp->resp.res;
729 struct rxe_mr *mr;
730 u64 value;
731 u64 iova;
732 int err;
733
734 if (!res) {
735 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK);
736 qp->resp.res = res;
737 }
738
739 if (res->replay)
740 return RESPST_ACKNOWLEDGE;
741
742 mr = qp->resp.mr;
743 value = *(u64 *)payload_addr(pkt);
744 iova = qp->resp.va + qp->resp.offset;
745
746 err = rxe_mr_do_atomic_write(mr, iova, value);
747 if (err)
748 return err;
749
750 qp->resp.resid = 0;
751 qp->resp.msn++;
752
753 /* next expected psn, read handles this separately */
754 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
755 qp->resp.ack_psn = qp->resp.psn;
756
757 qp->resp.opcode = pkt->opcode;
758 qp->resp.status = IB_WC_SUCCESS;
759
760 return RESPST_ACKNOWLEDGE;
761 }
762
prepare_ack_packet(struct rxe_qp * qp,struct rxe_pkt_info * ack,int opcode,int payload,u32 psn,u8 syndrome)763 static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
764 struct rxe_pkt_info *ack,
765 int opcode,
766 int payload,
767 u32 psn,
768 u8 syndrome)
769 {
770 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
771 struct sk_buff *skb;
772 int paylen;
773 int pad;
774 int err;
775
776 /*
777 * allocate packet
778 */
779 pad = (-payload) & 0x3;
780 paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
781
782 skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack);
783 if (!skb)
784 return NULL;
785
786 ack->qp = qp;
787 ack->opcode = opcode;
788 ack->mask = rxe_opcode[opcode].mask;
789 ack->paylen = paylen;
790 ack->psn = psn;
791
792 bth_init(ack, opcode, 0, 0, pad, IB_DEFAULT_PKEY_FULL,
793 qp->attr.dest_qp_num, 0, psn);
794
795 if (ack->mask & RXE_AETH_MASK) {
796 aeth_set_syn(ack, syndrome);
797 aeth_set_msn(ack, qp->resp.msn);
798 }
799
800 if (ack->mask & RXE_ATMACK_MASK)
801 atmack_set_orig(ack, qp->resp.res->atomic.orig_val);
802
803 err = rxe_prepare(&qp->pri_av, ack, skb);
804 if (err) {
805 kfree_skb(skb);
806 return NULL;
807 }
808
809 return skb;
810 }
811
812 /**
813 * rxe_recheck_mr - revalidate MR from rkey and get a reference
814 * @qp: the qp
815 * @rkey: the rkey
816 *
817 * This code allows the MR to be invalidated or deregistered or
818 * the MW if one was used to be invalidated or deallocated.
819 * It is assumed that the access permissions if originally good
820 * are OK and the mappings to be unchanged.
821 *
822 * TODO: If someone reregisters an MR to change its size or
823 * access permissions during the processing of an RDMA read
824 * we should kill the responder resource and complete the
825 * operation with an error.
826 *
827 * Return: mr on success else NULL
828 */
rxe_recheck_mr(struct rxe_qp * qp,u32 rkey)829 static struct rxe_mr *rxe_recheck_mr(struct rxe_qp *qp, u32 rkey)
830 {
831 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
832 struct rxe_mr *mr;
833 struct rxe_mw *mw;
834
835 if (rkey_is_mw(rkey)) {
836 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8);
837 if (!mw)
838 return NULL;
839
840 mr = mw->mr;
841 if (mw->rkey != rkey || mw->state != RXE_MW_STATE_VALID ||
842 !mr || mr->state != RXE_MR_STATE_VALID) {
843 rxe_put(mw);
844 return NULL;
845 }
846
847 rxe_get(mr);
848 rxe_put(mw);
849
850 return mr;
851 }
852
853 mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
854 if (!mr)
855 return NULL;
856
857 if (mr->rkey != rkey || mr->state != RXE_MR_STATE_VALID) {
858 rxe_put(mr);
859 return NULL;
860 }
861
862 return mr;
863 }
864
865 /* RDMA read response. If res is not NULL, then we have a current RDMA request
866 * being processed or replayed.
867 */
read_reply(struct rxe_qp * qp,struct rxe_pkt_info * req_pkt)868 static enum resp_states read_reply(struct rxe_qp *qp,
869 struct rxe_pkt_info *req_pkt)
870 {
871 struct rxe_pkt_info ack_pkt;
872 struct sk_buff *skb;
873 int mtu = qp->mtu;
874 enum resp_states state;
875 int payload;
876 int opcode;
877 int err;
878 struct resp_res *res = qp->resp.res;
879 struct rxe_mr *mr;
880
881 if (!res) {
882 res = rxe_prepare_res(qp, req_pkt, RXE_READ_MASK);
883 qp->resp.res = res;
884 }
885
886 if (res->state == rdatm_res_state_new) {
887 if (!res->replay || qp->resp.length == 0) {
888 /* if length == 0 mr will be NULL (is ok)
889 * otherwise qp->resp.mr holds a ref on mr
890 * which we transfer to mr and drop below.
891 */
892 mr = qp->resp.mr;
893 qp->resp.mr = NULL;
894 } else {
895 mr = rxe_recheck_mr(qp, res->read.rkey);
896 if (!mr)
897 return RESPST_ERR_RKEY_VIOLATION;
898 }
899
900 if (res->read.resid <= mtu)
901 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY;
902 else
903 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
904 } else {
905 /* re-lookup mr from rkey on all later packets.
906 * length will be non-zero. This can fail if someone
907 * modifies or destroys the mr since the first packet.
908 */
909 mr = rxe_recheck_mr(qp, res->read.rkey);
910 if (!mr)
911 return RESPST_ERR_RKEY_VIOLATION;
912
913 if (res->read.resid > mtu)
914 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
915 else
916 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
917 }
918
919 res->state = rdatm_res_state_next;
920
921 payload = min_t(int, res->read.resid, mtu);
922
923 skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload,
924 res->cur_psn, AETH_ACK_UNLIMITED);
925 if (!skb) {
926 state = RESPST_ERR_RNR;
927 goto err_out;
928 }
929
930 err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
931 payload, RXE_FROM_MR_OBJ);
932 if (err) {
933 kfree_skb(skb);
934 state = RESPST_ERR_RKEY_VIOLATION;
935 goto err_out;
936 }
937
938 if (bth_pad(&ack_pkt)) {
939 u8 *pad = payload_addr(&ack_pkt) + payload;
940
941 memset(pad, 0, bth_pad(&ack_pkt));
942 }
943
944 /* rxe_xmit_packet always consumes the skb */
945 err = rxe_xmit_packet(qp, &ack_pkt, skb);
946 if (err) {
947 state = RESPST_ERR_RNR;
948 goto err_out;
949 }
950
951 res->read.va += payload;
952 res->read.resid -= payload;
953 res->cur_psn = (res->cur_psn + 1) & BTH_PSN_MASK;
954
955 if (res->read.resid > 0) {
956 state = RESPST_DONE;
957 } else {
958 qp->resp.res = NULL;
959 if (!res->replay)
960 qp->resp.opcode = -1;
961 if (psn_compare(res->cur_psn, qp->resp.psn) >= 0)
962 qp->resp.psn = res->cur_psn;
963 state = RESPST_CLEANUP;
964 }
965
966 err_out:
967 if (mr)
968 rxe_put(mr);
969 return state;
970 }
971
invalidate_rkey(struct rxe_qp * qp,u32 rkey)972 static int invalidate_rkey(struct rxe_qp *qp, u32 rkey)
973 {
974 if (rkey_is_mw(rkey))
975 return rxe_invalidate_mw(qp, rkey);
976 else
977 return rxe_invalidate_mr(qp, rkey);
978 }
979
980 /* Executes a new request. A retried request never reach that function (send
981 * and writes are discarded, and reads and atomics are retried elsewhere.
982 */
execute(struct rxe_qp * qp,struct rxe_pkt_info * pkt)983 static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
984 {
985 enum resp_states err;
986 struct sk_buff *skb = PKT_TO_SKB(pkt);
987 union rdma_network_hdr hdr;
988
989 if (pkt->mask & RXE_SEND_MASK) {
990 if (qp_type(qp) == IB_QPT_UD ||
991 qp_type(qp) == IB_QPT_GSI) {
992 if (skb->protocol == htons(ETH_P_IP)) {
993 memset(&hdr.reserved, 0,
994 sizeof(hdr.reserved));
995 memcpy(&hdr.roce4grh, ip_hdr(skb),
996 sizeof(hdr.roce4grh));
997 err = send_data_in(qp, &hdr, sizeof(hdr));
998 } else {
999 err = send_data_in(qp, ipv6_hdr(skb),
1000 sizeof(hdr));
1001 }
1002 if (err)
1003 return err;
1004 }
1005 err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));
1006 if (err)
1007 return err;
1008 } else if (pkt->mask & RXE_WRITE_MASK) {
1009 err = write_data_in(qp, pkt);
1010 if (err)
1011 return err;
1012 } else if (pkt->mask & RXE_READ_MASK) {
1013 /* For RDMA Read we can increment the msn now. See C9-148. */
1014 qp->resp.msn++;
1015 return RESPST_READ_REPLY;
1016 } else if (pkt->mask & RXE_ATOMIC_MASK) {
1017 return RESPST_ATOMIC_REPLY;
1018 } else if (pkt->mask & RXE_ATOMIC_WRITE_MASK) {
1019 return RESPST_ATOMIC_WRITE_REPLY;
1020 } else if (pkt->mask & RXE_FLUSH_MASK) {
1021 return RESPST_PROCESS_FLUSH;
1022 } else {
1023 /* Unreachable */
1024 WARN_ON_ONCE(1);
1025 }
1026
1027 if (pkt->mask & RXE_IETH_MASK) {
1028 u32 rkey = ieth_rkey(pkt);
1029
1030 err = invalidate_rkey(qp, rkey);
1031 if (err)
1032 return RESPST_ERR_INVALIDATE_RKEY;
1033 }
1034
1035 if (pkt->mask & RXE_END_MASK)
1036 /* We successfully processed this new request. */
1037 qp->resp.msn++;
1038
1039 /* next expected psn, read handles this separately */
1040 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
1041 qp->resp.ack_psn = qp->resp.psn;
1042
1043 qp->resp.opcode = pkt->opcode;
1044 qp->resp.status = IB_WC_SUCCESS;
1045
1046 if (pkt->mask & RXE_COMP_MASK)
1047 return RESPST_COMPLETE;
1048 else if (qp_type(qp) == IB_QPT_RC)
1049 return RESPST_ACKNOWLEDGE;
1050 else
1051 return RESPST_CLEANUP;
1052 }
1053
do_complete(struct rxe_qp * qp,struct rxe_pkt_info * pkt)1054 static enum resp_states do_complete(struct rxe_qp *qp,
1055 struct rxe_pkt_info *pkt)
1056 {
1057 struct rxe_cqe cqe;
1058 struct ib_wc *wc = &cqe.ibwc;
1059 struct ib_uverbs_wc *uwc = &cqe.uibwc;
1060 struct rxe_recv_wqe *wqe = qp->resp.wqe;
1061 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
1062 unsigned long flags;
1063
1064 if (!wqe)
1065 goto finish;
1066
1067 memset(&cqe, 0, sizeof(cqe));
1068
1069 if (qp->rcq->is_user) {
1070 uwc->status = qp->resp.status;
1071 uwc->qp_num = qp->ibqp.qp_num;
1072 uwc->wr_id = wqe->wr_id;
1073 } else {
1074 wc->status = qp->resp.status;
1075 wc->qp = &qp->ibqp;
1076 wc->wr_id = wqe->wr_id;
1077 }
1078
1079 if (wc->status == IB_WC_SUCCESS) {
1080 rxe_counter_inc(rxe, RXE_CNT_RDMA_RECV);
1081 wc->opcode = (pkt->mask & RXE_IMMDT_MASK &&
1082 pkt->mask & RXE_WRITE_MASK) ?
1083 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
1084 wc->byte_len = (pkt->mask & RXE_IMMDT_MASK &&
1085 pkt->mask & RXE_WRITE_MASK) ?
1086 qp->resp.length : wqe->dma.length - wqe->dma.resid;
1087
1088 /* fields after byte_len are different between kernel and user
1089 * space
1090 */
1091 if (qp->rcq->is_user) {
1092 uwc->wc_flags = IB_WC_GRH;
1093
1094 if (pkt->mask & RXE_IMMDT_MASK) {
1095 uwc->wc_flags |= IB_WC_WITH_IMM;
1096 uwc->ex.imm_data = immdt_imm(pkt);
1097 }
1098
1099 if (pkt->mask & RXE_IETH_MASK) {
1100 uwc->wc_flags |= IB_WC_WITH_INVALIDATE;
1101 uwc->ex.invalidate_rkey = ieth_rkey(pkt);
1102 }
1103
1104 if (pkt->mask & RXE_DETH_MASK)
1105 uwc->src_qp = deth_sqp(pkt);
1106
1107 uwc->port_num = qp->attr.port_num;
1108 } else {
1109 struct sk_buff *skb = PKT_TO_SKB(pkt);
1110
1111 wc->wc_flags = IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE;
1112 if (skb->protocol == htons(ETH_P_IP))
1113 wc->network_hdr_type = RDMA_NETWORK_IPV4;
1114 else
1115 wc->network_hdr_type = RDMA_NETWORK_IPV6;
1116
1117 if (is_vlan_dev(skb->dev)) {
1118 wc->wc_flags |= IB_WC_WITH_VLAN;
1119 wc->vlan_id = vlan_dev_vlan_id(skb->dev);
1120 }
1121
1122 if (pkt->mask & RXE_IMMDT_MASK) {
1123 wc->wc_flags |= IB_WC_WITH_IMM;
1124 wc->ex.imm_data = immdt_imm(pkt);
1125 }
1126
1127 if (pkt->mask & RXE_IETH_MASK) {
1128 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
1129 wc->ex.invalidate_rkey = ieth_rkey(pkt);
1130 }
1131
1132 if (pkt->mask & RXE_DETH_MASK)
1133 wc->src_qp = deth_sqp(pkt);
1134
1135 wc->port_num = qp->attr.port_num;
1136 }
1137 } else {
1138 if (wc->status != IB_WC_WR_FLUSH_ERR)
1139 rxe_err_qp(qp, "non-flush error status = %d\n",
1140 wc->status);
1141 }
1142
1143 /* have copy for srq and reference for !srq */
1144 if (!qp->srq)
1145 queue_advance_consumer(qp->rq.queue, QUEUE_TYPE_FROM_CLIENT);
1146
1147 qp->resp.wqe = NULL;
1148
1149 if (rxe_cq_post(qp->rcq, &cqe, pkt ? bth_se(pkt) : 1))
1150 return RESPST_ERR_CQ_OVERFLOW;
1151
1152 finish:
1153 spin_lock_irqsave(&qp->state_lock, flags);
1154 if (unlikely(qp_state(qp) == IB_QPS_ERR)) {
1155 spin_unlock_irqrestore(&qp->state_lock, flags);
1156 return RESPST_CHK_RESOURCE;
1157 }
1158 spin_unlock_irqrestore(&qp->state_lock, flags);
1159
1160 if (unlikely(!pkt))
1161 return RESPST_DONE;
1162 if (qp_type(qp) == IB_QPT_RC)
1163 return RESPST_ACKNOWLEDGE;
1164 else
1165 return RESPST_CLEANUP;
1166 }
1167
1168
send_common_ack(struct rxe_qp * qp,u8 syndrome,u32 psn,int opcode,const char * msg)1169 static int send_common_ack(struct rxe_qp *qp, u8 syndrome, u32 psn,
1170 int opcode, const char *msg)
1171 {
1172 int err;
1173 struct rxe_pkt_info ack_pkt;
1174 struct sk_buff *skb;
1175
1176 skb = prepare_ack_packet(qp, &ack_pkt, opcode, 0, psn, syndrome);
1177 if (!skb)
1178 return -ENOMEM;
1179
1180 err = rxe_xmit_packet(qp, &ack_pkt, skb);
1181 if (err)
1182 rxe_dbg_qp(qp, "Failed sending %s\n", msg);
1183
1184 return err;
1185 }
1186
send_ack(struct rxe_qp * qp,u8 syndrome,u32 psn)1187 static int send_ack(struct rxe_qp *qp, u8 syndrome, u32 psn)
1188 {
1189 return send_common_ack(qp, syndrome, psn,
1190 IB_OPCODE_RC_ACKNOWLEDGE, "ACK");
1191 }
1192
send_atomic_ack(struct rxe_qp * qp,u8 syndrome,u32 psn)1193 static int send_atomic_ack(struct rxe_qp *qp, u8 syndrome, u32 psn)
1194 {
1195 int ret = send_common_ack(qp, syndrome, psn,
1196 IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE, "ATOMIC ACK");
1197
1198 /* have to clear this since it is used to trigger
1199 * long read replies
1200 */
1201 qp->resp.res = NULL;
1202 return ret;
1203 }
1204
send_read_response_ack(struct rxe_qp * qp,u8 syndrome,u32 psn)1205 static int send_read_response_ack(struct rxe_qp *qp, u8 syndrome, u32 psn)
1206 {
1207 int ret = send_common_ack(qp, syndrome, psn,
1208 IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY,
1209 "RDMA READ response of length zero ACK");
1210
1211 /* have to clear this since it is used to trigger
1212 * long read replies
1213 */
1214 qp->resp.res = NULL;
1215 return ret;
1216 }
1217
acknowledge(struct rxe_qp * qp,struct rxe_pkt_info * pkt)1218 static enum resp_states acknowledge(struct rxe_qp *qp,
1219 struct rxe_pkt_info *pkt)
1220 {
1221 if (qp_type(qp) != IB_QPT_RC)
1222 return RESPST_CLEANUP;
1223
1224 if (qp->resp.aeth_syndrome != AETH_ACK_UNLIMITED)
1225 send_ack(qp, qp->resp.aeth_syndrome, pkt->psn);
1226 else if (pkt->mask & RXE_ATOMIC_MASK)
1227 send_atomic_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1228 else if (pkt->mask & (RXE_FLUSH_MASK | RXE_ATOMIC_WRITE_MASK))
1229 send_read_response_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1230 else if (bth_ack(pkt))
1231 send_ack(qp, AETH_ACK_UNLIMITED, pkt->psn);
1232
1233 return RESPST_CLEANUP;
1234 }
1235
cleanup(struct rxe_qp * qp,struct rxe_pkt_info * pkt)1236 static enum resp_states cleanup(struct rxe_qp *qp,
1237 struct rxe_pkt_info *pkt)
1238 {
1239 struct sk_buff *skb;
1240
1241 if (pkt) {
1242 skb = skb_dequeue(&qp->req_pkts);
1243 rxe_put(qp);
1244 kfree_skb(skb);
1245 ib_device_put(qp->ibqp.device);
1246 }
1247
1248 if (qp->resp.mr) {
1249 rxe_put(qp->resp.mr);
1250 qp->resp.mr = NULL;
1251 }
1252
1253 return RESPST_DONE;
1254 }
1255
find_resource(struct rxe_qp * qp,u32 psn)1256 static struct resp_res *find_resource(struct rxe_qp *qp, u32 psn)
1257 {
1258 int i;
1259
1260 for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
1261 struct resp_res *res = &qp->resp.resources[i];
1262
1263 if (res->type == 0)
1264 continue;
1265
1266 if (psn_compare(psn, res->first_psn) >= 0 &&
1267 psn_compare(psn, res->last_psn) <= 0) {
1268 return res;
1269 }
1270 }
1271
1272 return NULL;
1273 }
1274
duplicate_request(struct rxe_qp * qp,struct rxe_pkt_info * pkt)1275 static enum resp_states duplicate_request(struct rxe_qp *qp,
1276 struct rxe_pkt_info *pkt)
1277 {
1278 enum resp_states rc;
1279 u32 prev_psn = (qp->resp.ack_psn - 1) & BTH_PSN_MASK;
1280
1281 if (pkt->mask & RXE_SEND_MASK ||
1282 pkt->mask & RXE_WRITE_MASK) {
1283 /* SEND. Ack again and cleanup. C9-105. */
1284 send_ack(qp, AETH_ACK_UNLIMITED, prev_psn);
1285 return RESPST_CLEANUP;
1286 } else if (pkt->mask & RXE_FLUSH_MASK) {
1287 struct resp_res *res;
1288
1289 /* Find the operation in our list of responder resources. */
1290 res = find_resource(qp, pkt->psn);
1291 if (res) {
1292 res->replay = 1;
1293 res->cur_psn = pkt->psn;
1294 qp->resp.res = res;
1295 rc = RESPST_PROCESS_FLUSH;
1296 goto out;
1297 }
1298
1299 /* Resource not found. Class D error. Drop the request. */
1300 rc = RESPST_CLEANUP;
1301 goto out;
1302 } else if (pkt->mask & RXE_READ_MASK) {
1303 struct resp_res *res;
1304
1305 res = find_resource(qp, pkt->psn);
1306 if (!res) {
1307 /* Resource not found. Class D error. Drop the
1308 * request.
1309 */
1310 rc = RESPST_CLEANUP;
1311 goto out;
1312 } else {
1313 /* Ensure this new request is the same as the previous
1314 * one or a subset of it.
1315 */
1316 u64 iova = reth_va(pkt);
1317 u32 resid = reth_len(pkt);
1318
1319 if (iova < res->read.va_org ||
1320 resid > res->read.length ||
1321 (iova + resid) > (res->read.va_org +
1322 res->read.length)) {
1323 rc = RESPST_CLEANUP;
1324 goto out;
1325 }
1326
1327 if (reth_rkey(pkt) != res->read.rkey) {
1328 rc = RESPST_CLEANUP;
1329 goto out;
1330 }
1331
1332 res->cur_psn = pkt->psn;
1333 res->state = (pkt->psn == res->first_psn) ?
1334 rdatm_res_state_new :
1335 rdatm_res_state_replay;
1336 res->replay = 1;
1337
1338 /* Reset the resource, except length. */
1339 res->read.va_org = iova;
1340 res->read.va = iova;
1341 res->read.resid = resid;
1342
1343 /* Replay the RDMA read reply. */
1344 qp->resp.res = res;
1345 rc = RESPST_READ_REPLY;
1346 goto out;
1347 }
1348 } else {
1349 struct resp_res *res;
1350
1351 /* Find the operation in our list of responder resources. */
1352 res = find_resource(qp, pkt->psn);
1353 if (res) {
1354 res->replay = 1;
1355 res->cur_psn = pkt->psn;
1356 qp->resp.res = res;
1357 rc = pkt->mask & RXE_ATOMIC_MASK ?
1358 RESPST_ATOMIC_REPLY :
1359 RESPST_ATOMIC_WRITE_REPLY;
1360 goto out;
1361 }
1362
1363 /* Resource not found. Class D error. Drop the request. */
1364 rc = RESPST_CLEANUP;
1365 goto out;
1366 }
1367 out:
1368 return rc;
1369 }
1370
1371 /* Process a class A or C. Both are treated the same in this implementation. */
do_class_ac_error(struct rxe_qp * qp,u8 syndrome,enum ib_wc_status status)1372 static void do_class_ac_error(struct rxe_qp *qp, u8 syndrome,
1373 enum ib_wc_status status)
1374 {
1375 qp->resp.aeth_syndrome = syndrome;
1376 qp->resp.status = status;
1377
1378 /* indicate that we should go through the ERROR state */
1379 qp->resp.goto_error = 1;
1380 }
1381
do_class_d1e_error(struct rxe_qp * qp)1382 static enum resp_states do_class_d1e_error(struct rxe_qp *qp)
1383 {
1384 /* UC */
1385 if (qp->srq) {
1386 /* Class E */
1387 qp->resp.drop_msg = 1;
1388 if (qp->resp.wqe) {
1389 qp->resp.status = IB_WC_REM_INV_REQ_ERR;
1390 return RESPST_COMPLETE;
1391 } else {
1392 return RESPST_CLEANUP;
1393 }
1394 } else {
1395 /* Class D1. This packet may be the start of a
1396 * new message and could be valid. The previous
1397 * message is invalid and ignored. reset the
1398 * recv wr to its original state
1399 */
1400 if (qp->resp.wqe) {
1401 qp->resp.wqe->dma.resid = qp->resp.wqe->dma.length;
1402 qp->resp.wqe->dma.cur_sge = 0;
1403 qp->resp.wqe->dma.sge_offset = 0;
1404 qp->resp.opcode = -1;
1405 }
1406
1407 if (qp->resp.mr) {
1408 rxe_put(qp->resp.mr);
1409 qp->resp.mr = NULL;
1410 }
1411
1412 return RESPST_CLEANUP;
1413 }
1414 }
1415
1416 /* drain incoming request packet queue */
drain_req_pkts(struct rxe_qp * qp)1417 static void drain_req_pkts(struct rxe_qp *qp)
1418 {
1419 struct sk_buff *skb;
1420
1421 while ((skb = skb_dequeue(&qp->req_pkts))) {
1422 rxe_put(qp);
1423 kfree_skb(skb);
1424 ib_device_put(qp->ibqp.device);
1425 }
1426 }
1427
1428 /* complete receive wqe with flush error */
flush_recv_wqe(struct rxe_qp * qp,struct rxe_recv_wqe * wqe)1429 static int flush_recv_wqe(struct rxe_qp *qp, struct rxe_recv_wqe *wqe)
1430 {
1431 struct rxe_cqe cqe = {};
1432 struct ib_wc *wc = &cqe.ibwc;
1433 struct ib_uverbs_wc *uwc = &cqe.uibwc;
1434 int err;
1435
1436 if (qp->rcq->is_user) {
1437 uwc->wr_id = wqe->wr_id;
1438 uwc->status = IB_WC_WR_FLUSH_ERR;
1439 uwc->qp_num = qp_num(qp);
1440 } else {
1441 wc->wr_id = wqe->wr_id;
1442 wc->status = IB_WC_WR_FLUSH_ERR;
1443 wc->qp = &qp->ibqp;
1444 }
1445
1446 err = rxe_cq_post(qp->rcq, &cqe, 0);
1447 if (err)
1448 rxe_dbg_cq(qp->rcq, "post cq failed err = %d\n", err);
1449
1450 return err;
1451 }
1452
1453 /* drain and optionally complete the recive queue
1454 * if unable to complete a wqe stop completing and
1455 * just flush the remaining wqes
1456 */
flush_recv_queue(struct rxe_qp * qp,bool notify)1457 static void flush_recv_queue(struct rxe_qp *qp, bool notify)
1458 {
1459 struct rxe_queue *q = qp->rq.queue;
1460 struct rxe_recv_wqe *wqe;
1461 int err;
1462
1463 if (qp->srq) {
1464 if (notify && qp->ibqp.event_handler) {
1465 struct ib_event ev;
1466
1467 ev.device = qp->ibqp.device;
1468 ev.element.qp = &qp->ibqp;
1469 ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
1470 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
1471 }
1472 return;
1473 }
1474
1475 /* recv queue not created. nothing to do. */
1476 if (!qp->rq.queue)
1477 return;
1478
1479 while ((wqe = queue_head(q, q->type))) {
1480 if (notify) {
1481 err = flush_recv_wqe(qp, wqe);
1482 if (err)
1483 notify = 0;
1484 }
1485 queue_advance_consumer(q, q->type);
1486 }
1487
1488 qp->resp.wqe = NULL;
1489 }
1490
rxe_receiver(struct rxe_qp * qp)1491 int rxe_receiver(struct rxe_qp *qp)
1492 {
1493 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
1494 enum resp_states state;
1495 struct rxe_pkt_info *pkt = NULL;
1496 int ret;
1497 unsigned long flags;
1498
1499 spin_lock_irqsave(&qp->state_lock, flags);
1500 if (!qp->valid || qp_state(qp) == IB_QPS_ERR ||
1501 qp_state(qp) == IB_QPS_RESET) {
1502 bool notify = qp->valid && (qp_state(qp) == IB_QPS_ERR);
1503
1504 drain_req_pkts(qp);
1505 flush_recv_queue(qp, notify);
1506 spin_unlock_irqrestore(&qp->state_lock, flags);
1507 goto exit;
1508 }
1509 spin_unlock_irqrestore(&qp->state_lock, flags);
1510
1511 qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;
1512
1513 state = RESPST_GET_REQ;
1514
1515 while (1) {
1516 rxe_dbg_qp(qp, "state = %s\n", resp_state_name[state]);
1517 switch (state) {
1518 case RESPST_GET_REQ:
1519 state = get_req(qp, &pkt);
1520 break;
1521 case RESPST_CHK_PSN:
1522 state = check_psn(qp, pkt);
1523 break;
1524 case RESPST_CHK_OP_SEQ:
1525 state = check_op_seq(qp, pkt);
1526 break;
1527 case RESPST_CHK_OP_VALID:
1528 state = check_op_valid(qp, pkt);
1529 break;
1530 case RESPST_CHK_RESOURCE:
1531 state = check_resource(qp, pkt);
1532 break;
1533 case RESPST_CHK_LENGTH:
1534 state = rxe_resp_check_length(qp, pkt);
1535 break;
1536 case RESPST_CHK_RKEY:
1537 state = check_rkey(qp, pkt);
1538 break;
1539 case RESPST_EXECUTE:
1540 state = execute(qp, pkt);
1541 break;
1542 case RESPST_COMPLETE:
1543 state = do_complete(qp, pkt);
1544 break;
1545 case RESPST_READ_REPLY:
1546 state = read_reply(qp, pkt);
1547 break;
1548 case RESPST_ATOMIC_REPLY:
1549 state = atomic_reply(qp, pkt);
1550 break;
1551 case RESPST_ATOMIC_WRITE_REPLY:
1552 state = atomic_write_reply(qp, pkt);
1553 break;
1554 case RESPST_PROCESS_FLUSH:
1555 state = process_flush(qp, pkt);
1556 break;
1557 case RESPST_ACKNOWLEDGE:
1558 state = acknowledge(qp, pkt);
1559 break;
1560 case RESPST_CLEANUP:
1561 state = cleanup(qp, pkt);
1562 break;
1563 case RESPST_DUPLICATE_REQUEST:
1564 state = duplicate_request(qp, pkt);
1565 break;
1566 case RESPST_ERR_PSN_OUT_OF_SEQ:
1567 /* RC only - Class B. Drop packet. */
1568 send_ack(qp, AETH_NAK_PSN_SEQ_ERROR, qp->resp.psn);
1569 state = RESPST_CLEANUP;
1570 break;
1571
1572 case RESPST_ERR_TOO_MANY_RDMA_ATM_REQ:
1573 case RESPST_ERR_MISSING_OPCODE_FIRST:
1574 case RESPST_ERR_MISSING_OPCODE_LAST_C:
1575 case RESPST_ERR_UNSUPPORTED_OPCODE:
1576 case RESPST_ERR_MISALIGNED_ATOMIC:
1577 /* RC Only - Class C. */
1578 do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
1579 IB_WC_REM_INV_REQ_ERR);
1580 state = RESPST_COMPLETE;
1581 break;
1582
1583 case RESPST_ERR_MISSING_OPCODE_LAST_D1E:
1584 state = do_class_d1e_error(qp);
1585 break;
1586 case RESPST_ERR_RNR:
1587 if (qp_type(qp) == IB_QPT_RC) {
1588 rxe_counter_inc(rxe, RXE_CNT_SND_RNR);
1589 /* RC - class B */
1590 send_ack(qp, AETH_RNR_NAK |
1591 (~AETH_TYPE_MASK &
1592 qp->attr.min_rnr_timer),
1593 pkt->psn);
1594 } else {
1595 /* UD/UC - class D */
1596 qp->resp.drop_msg = 1;
1597 }
1598 state = RESPST_CLEANUP;
1599 break;
1600
1601 case RESPST_ERR_RKEY_VIOLATION:
1602 if (qp_type(qp) == IB_QPT_RC) {
1603 /* Class C */
1604 do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR,
1605 IB_WC_REM_ACCESS_ERR);
1606 state = RESPST_COMPLETE;
1607 } else {
1608 qp->resp.drop_msg = 1;
1609 if (qp->srq) {
1610 /* UC/SRQ Class D */
1611 qp->resp.status = IB_WC_REM_ACCESS_ERR;
1612 state = RESPST_COMPLETE;
1613 } else {
1614 /* UC/non-SRQ Class E. */
1615 state = RESPST_CLEANUP;
1616 }
1617 }
1618 break;
1619
1620 case RESPST_ERR_INVALIDATE_RKEY:
1621 /* RC - Class J. */
1622 qp->resp.goto_error = 1;
1623 qp->resp.status = IB_WC_REM_INV_REQ_ERR;
1624 state = RESPST_COMPLETE;
1625 break;
1626
1627 case RESPST_ERR_LENGTH:
1628 if (qp_type(qp) == IB_QPT_RC) {
1629 /* Class C */
1630 do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
1631 IB_WC_REM_INV_REQ_ERR);
1632 state = RESPST_COMPLETE;
1633 } else if (qp->srq) {
1634 /* UC/UD - class E */
1635 qp->resp.status = IB_WC_REM_INV_REQ_ERR;
1636 state = RESPST_COMPLETE;
1637 } else {
1638 /* UC/UD - class D */
1639 qp->resp.drop_msg = 1;
1640 state = RESPST_CLEANUP;
1641 }
1642 break;
1643
1644 case RESPST_ERR_MALFORMED_WQE:
1645 /* All, Class A. */
1646 do_class_ac_error(qp, AETH_NAK_REM_OP_ERR,
1647 IB_WC_LOC_QP_OP_ERR);
1648 state = RESPST_COMPLETE;
1649 break;
1650
1651 case RESPST_ERR_CQ_OVERFLOW:
1652 /* All - Class G */
1653 state = RESPST_ERROR;
1654 break;
1655
1656 case RESPST_DONE:
1657 if (qp->resp.goto_error) {
1658 state = RESPST_ERROR;
1659 break;
1660 }
1661
1662 goto done;
1663
1664 case RESPST_EXIT:
1665 if (qp->resp.goto_error) {
1666 state = RESPST_ERROR;
1667 break;
1668 }
1669
1670 goto exit;
1671
1672 case RESPST_ERROR:
1673 qp->resp.goto_error = 0;
1674 rxe_dbg_qp(qp, "moved to error state\n");
1675 rxe_qp_error(qp);
1676 goto exit;
1677
1678 default:
1679 WARN_ON_ONCE(1);
1680 }
1681 }
1682
1683 /* A non-zero return value will cause rxe_do_task to
1684 * exit its loop and end the work item. A zero return
1685 * will continue looping and return to rxe_responder
1686 */
1687 done:
1688 ret = 0;
1689 goto out;
1690 exit:
1691 ret = -EAGAIN;
1692 out:
1693 return ret;
1694 }
1695