1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #include <linux/skbuff.h> 8 9 #include "rxe.h" 10 #include "rxe_loc.h" 11 #include "rxe_queue.h" 12 13 static char *resp_state_name[] = { 14 [RESPST_NONE] = "NONE", 15 [RESPST_GET_REQ] = "GET_REQ", 16 [RESPST_CHK_PSN] = "CHK_PSN", 17 [RESPST_CHK_OP_SEQ] = "CHK_OP_SEQ", 18 [RESPST_CHK_OP_VALID] = "CHK_OP_VALID", 19 [RESPST_CHK_RESOURCE] = "CHK_RESOURCE", 20 [RESPST_CHK_LENGTH] = "CHK_LENGTH", 21 [RESPST_CHK_RKEY] = "CHK_RKEY", 22 [RESPST_EXECUTE] = "EXECUTE", 23 [RESPST_READ_REPLY] = "READ_REPLY", 24 [RESPST_ATOMIC_REPLY] = "ATOMIC_REPLY", 25 [RESPST_ATOMIC_WRITE_REPLY] = "ATOMIC_WRITE_REPLY", 26 [RESPST_PROCESS_FLUSH] = "PROCESS_FLUSH", 27 [RESPST_COMPLETE] = "COMPLETE", 28 [RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE", 29 [RESPST_CLEANUP] = "CLEANUP", 30 [RESPST_DUPLICATE_REQUEST] = "DUPLICATE_REQUEST", 31 [RESPST_ERR_MALFORMED_WQE] = "ERR_MALFORMED_WQE", 32 [RESPST_ERR_UNSUPPORTED_OPCODE] = "ERR_UNSUPPORTED_OPCODE", 33 [RESPST_ERR_MISALIGNED_ATOMIC] = "ERR_MISALIGNED_ATOMIC", 34 [RESPST_ERR_PSN_OUT_OF_SEQ] = "ERR_PSN_OUT_OF_SEQ", 35 [RESPST_ERR_MISSING_OPCODE_FIRST] = "ERR_MISSING_OPCODE_FIRST", 36 [RESPST_ERR_MISSING_OPCODE_LAST_C] = "ERR_MISSING_OPCODE_LAST_C", 37 [RESPST_ERR_MISSING_OPCODE_LAST_D1E] = "ERR_MISSING_OPCODE_LAST_D1E", 38 [RESPST_ERR_TOO_MANY_RDMA_ATM_REQ] = "ERR_TOO_MANY_RDMA_ATM_REQ", 39 [RESPST_ERR_RNR] = "ERR_RNR", 40 [RESPST_ERR_RKEY_VIOLATION] = "ERR_RKEY_VIOLATION", 41 [RESPST_ERR_INVALIDATE_RKEY] = "ERR_INVALIDATE_RKEY_VIOLATION", 42 [RESPST_ERR_LENGTH] = "ERR_LENGTH", 43 [RESPST_ERR_CQ_OVERFLOW] = "ERR_CQ_OVERFLOW", 44 [RESPST_ERROR] = "ERROR", 45 [RESPST_DONE] = "DONE", 46 [RESPST_EXIT] = "EXIT", 47 }; 48 49 /* rxe_recv calls here to add a request packet to the input queue */ 50 void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) 51 { 52 int must_sched; 53 struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); 54 55 skb_queue_tail(&qp->req_pkts, skb); 56 57 must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) || 58 (skb_queue_len(&qp->req_pkts) > 1); 59 60 if (must_sched) 61 rxe_sched_task(&qp->resp.task); 62 else 63 rxe_run_task(&qp->resp.task); 64 } 65 66 static inline enum resp_states get_req(struct rxe_qp *qp, 67 struct rxe_pkt_info **pkt_p) 68 { 69 struct sk_buff *skb; 70 71 skb = skb_peek(&qp->req_pkts); 72 if (!skb) 73 return RESPST_EXIT; 74 75 *pkt_p = SKB_TO_PKT(skb); 76 77 return (qp->resp.res) ? RESPST_READ_REPLY : RESPST_CHK_PSN; 78 } 79 80 static enum resp_states check_psn(struct rxe_qp *qp, 81 struct rxe_pkt_info *pkt) 82 { 83 int diff = psn_compare(pkt->psn, qp->resp.psn); 84 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 85 86 switch (qp_type(qp)) { 87 case IB_QPT_RC: 88 if (diff > 0) { 89 if (qp->resp.sent_psn_nak) 90 return RESPST_CLEANUP; 91 92 qp->resp.sent_psn_nak = 1; 93 rxe_counter_inc(rxe, RXE_CNT_OUT_OF_SEQ_REQ); 94 return RESPST_ERR_PSN_OUT_OF_SEQ; 95 96 } else if (diff < 0) { 97 rxe_counter_inc(rxe, RXE_CNT_DUP_REQ); 98 return RESPST_DUPLICATE_REQUEST; 99 } 100 101 if (qp->resp.sent_psn_nak) 102 qp->resp.sent_psn_nak = 0; 103 104 break; 105 106 case IB_QPT_UC: 107 if (qp->resp.drop_msg || diff != 0) { 108 if (pkt->mask & RXE_START_MASK) { 109 qp->resp.drop_msg = 0; 110 return RESPST_CHK_OP_SEQ; 111 } 112 113 qp->resp.drop_msg = 1; 114 return RESPST_CLEANUP; 115 } 116 break; 117 default: 118 break; 119 } 120 121 return RESPST_CHK_OP_SEQ; 122 } 123 124 static enum resp_states check_op_seq(struct rxe_qp *qp, 125 struct rxe_pkt_info *pkt) 126 { 127 switch (qp_type(qp)) { 128 case IB_QPT_RC: 129 switch (qp->resp.opcode) { 130 case IB_OPCODE_RC_SEND_FIRST: 131 case IB_OPCODE_RC_SEND_MIDDLE: 132 switch (pkt->opcode) { 133 case IB_OPCODE_RC_SEND_MIDDLE: 134 case IB_OPCODE_RC_SEND_LAST: 135 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE: 136 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE: 137 return RESPST_CHK_OP_VALID; 138 default: 139 return RESPST_ERR_MISSING_OPCODE_LAST_C; 140 } 141 142 case IB_OPCODE_RC_RDMA_WRITE_FIRST: 143 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE: 144 switch (pkt->opcode) { 145 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE: 146 case IB_OPCODE_RC_RDMA_WRITE_LAST: 147 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE: 148 return RESPST_CHK_OP_VALID; 149 default: 150 return RESPST_ERR_MISSING_OPCODE_LAST_C; 151 } 152 153 default: 154 switch (pkt->opcode) { 155 case IB_OPCODE_RC_SEND_MIDDLE: 156 case IB_OPCODE_RC_SEND_LAST: 157 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE: 158 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE: 159 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE: 160 case IB_OPCODE_RC_RDMA_WRITE_LAST: 161 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE: 162 return RESPST_ERR_MISSING_OPCODE_FIRST; 163 default: 164 return RESPST_CHK_OP_VALID; 165 } 166 } 167 break; 168 169 case IB_QPT_UC: 170 switch (qp->resp.opcode) { 171 case IB_OPCODE_UC_SEND_FIRST: 172 case IB_OPCODE_UC_SEND_MIDDLE: 173 switch (pkt->opcode) { 174 case IB_OPCODE_UC_SEND_MIDDLE: 175 case IB_OPCODE_UC_SEND_LAST: 176 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE: 177 return RESPST_CHK_OP_VALID; 178 default: 179 return RESPST_ERR_MISSING_OPCODE_LAST_D1E; 180 } 181 182 case IB_OPCODE_UC_RDMA_WRITE_FIRST: 183 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE: 184 switch (pkt->opcode) { 185 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE: 186 case IB_OPCODE_UC_RDMA_WRITE_LAST: 187 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE: 188 return RESPST_CHK_OP_VALID; 189 default: 190 return RESPST_ERR_MISSING_OPCODE_LAST_D1E; 191 } 192 193 default: 194 switch (pkt->opcode) { 195 case IB_OPCODE_UC_SEND_MIDDLE: 196 case IB_OPCODE_UC_SEND_LAST: 197 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE: 198 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE: 199 case IB_OPCODE_UC_RDMA_WRITE_LAST: 200 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE: 201 qp->resp.drop_msg = 1; 202 return RESPST_CLEANUP; 203 default: 204 return RESPST_CHK_OP_VALID; 205 } 206 } 207 break; 208 209 default: 210 return RESPST_CHK_OP_VALID; 211 } 212 } 213 214 static bool check_qp_attr_access(struct rxe_qp *qp, 215 struct rxe_pkt_info *pkt) 216 { 217 if (((pkt->mask & RXE_READ_MASK) && 218 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) || 219 ((pkt->mask & (RXE_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) && 220 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) || 221 ((pkt->mask & RXE_ATOMIC_MASK) && 222 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 223 return false; 224 225 if (pkt->mask & RXE_FLUSH_MASK) { 226 u32 flush_type = feth_plt(pkt); 227 228 if ((flush_type & IB_FLUSH_GLOBAL && 229 !(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_GLOBAL)) || 230 (flush_type & IB_FLUSH_PERSISTENT && 231 !(qp->attr.qp_access_flags & IB_ACCESS_FLUSH_PERSISTENT))) 232 return false; 233 } 234 235 return true; 236 } 237 238 static enum resp_states check_op_valid(struct rxe_qp *qp, 239 struct rxe_pkt_info *pkt) 240 { 241 switch (qp_type(qp)) { 242 case IB_QPT_RC: 243 if (!check_qp_attr_access(qp, pkt)) 244 return RESPST_ERR_UNSUPPORTED_OPCODE; 245 246 break; 247 248 case IB_QPT_UC: 249 if ((pkt->mask & RXE_WRITE_MASK) && 250 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) { 251 qp->resp.drop_msg = 1; 252 return RESPST_CLEANUP; 253 } 254 255 break; 256 257 case IB_QPT_UD: 258 case IB_QPT_GSI: 259 break; 260 261 default: 262 WARN_ON_ONCE(1); 263 break; 264 } 265 266 return RESPST_CHK_RESOURCE; 267 } 268 269 static enum resp_states get_srq_wqe(struct rxe_qp *qp) 270 { 271 struct rxe_srq *srq = qp->srq; 272 struct rxe_queue *q = srq->rq.queue; 273 struct rxe_recv_wqe *wqe; 274 struct ib_event ev; 275 unsigned int count; 276 size_t size; 277 unsigned long flags; 278 279 if (srq->error) 280 return RESPST_ERR_RNR; 281 282 spin_lock_irqsave(&srq->rq.consumer_lock, flags); 283 284 wqe = queue_head(q, QUEUE_TYPE_FROM_CLIENT); 285 if (!wqe) { 286 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags); 287 return RESPST_ERR_RNR; 288 } 289 290 /* don't trust user space data */ 291 if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) { 292 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags); 293 rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n"); 294 return RESPST_ERR_MALFORMED_WQE; 295 } 296 size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge); 297 memcpy(&qp->resp.srq_wqe, wqe, size); 298 299 qp->resp.wqe = &qp->resp.srq_wqe.wqe; 300 queue_advance_consumer(q, QUEUE_TYPE_FROM_CLIENT); 301 count = queue_count(q, QUEUE_TYPE_FROM_CLIENT); 302 303 if (srq->limit && srq->ibsrq.event_handler && (count < srq->limit)) { 304 srq->limit = 0; 305 goto event; 306 } 307 308 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags); 309 return RESPST_CHK_LENGTH; 310 311 event: 312 spin_unlock_irqrestore(&srq->rq.consumer_lock, flags); 313 ev.device = qp->ibqp.device; 314 ev.element.srq = qp->ibqp.srq; 315 ev.event = IB_EVENT_SRQ_LIMIT_REACHED; 316 srq->ibsrq.event_handler(&ev, srq->ibsrq.srq_context); 317 return RESPST_CHK_LENGTH; 318 } 319 320 static enum resp_states check_resource(struct rxe_qp *qp, 321 struct rxe_pkt_info *pkt) 322 { 323 struct rxe_srq *srq = qp->srq; 324 325 if (pkt->mask & (RXE_READ_OR_ATOMIC_MASK | RXE_ATOMIC_WRITE_MASK)) { 326 /* it is the requesters job to not send 327 * too many read/atomic ops, we just 328 * recycle the responder resource queue 329 */ 330 if (likely(qp->attr.max_dest_rd_atomic > 0)) 331 return RESPST_CHK_LENGTH; 332 else 333 return RESPST_ERR_TOO_MANY_RDMA_ATM_REQ; 334 } 335 336 if (pkt->mask & RXE_RWR_MASK) { 337 if (srq) 338 return get_srq_wqe(qp); 339 340 qp->resp.wqe = queue_head(qp->rq.queue, 341 QUEUE_TYPE_FROM_CLIENT); 342 return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR; 343 } 344 345 return RESPST_CHK_LENGTH; 346 } 347 348 static enum resp_states rxe_resp_check_length(struct rxe_qp *qp, 349 struct rxe_pkt_info *pkt) 350 { 351 /* 352 * See IBA C9-92 353 * For UD QPs we only check if the packet will fit in the 354 * receive buffer later. For rmda operations additional 355 * length checks are performed in check_rkey. 356 */ 357 if (pkt->mask & RXE_PAYLOAD_MASK && ((qp_type(qp) == IB_QPT_RC) || 358 (qp_type(qp) == IB_QPT_UC))) { 359 unsigned int mtu = qp->mtu; 360 unsigned int payload = payload_size(pkt); 361 362 if ((pkt->mask & RXE_START_MASK) && 363 (pkt->mask & RXE_END_MASK)) { 364 if (unlikely(payload > mtu)) { 365 rxe_dbg_qp(qp, "only packet too long"); 366 return RESPST_ERR_LENGTH; 367 } 368 } else if ((pkt->mask & RXE_START_MASK) || 369 (pkt->mask & RXE_MIDDLE_MASK)) { 370 if (unlikely(payload != mtu)) { 371 rxe_dbg_qp(qp, "first or middle packet not mtu"); 372 return RESPST_ERR_LENGTH; 373 } 374 } else if (pkt->mask & RXE_END_MASK) { 375 if (unlikely((payload == 0) || (payload > mtu))) { 376 rxe_dbg_qp(qp, "last packet zero or too long"); 377 return RESPST_ERR_LENGTH; 378 } 379 } 380 } 381 382 /* See IBA C9-94 */ 383 if (pkt->mask & RXE_RETH_MASK) { 384 if (reth_len(pkt) > (1U << 31)) { 385 rxe_dbg_qp(qp, "dma length too long"); 386 return RESPST_ERR_LENGTH; 387 } 388 } 389 390 if (pkt->mask & RXE_RDMA_OP_MASK) 391 return RESPST_CHK_RKEY; 392 else 393 return RESPST_EXECUTE; 394 } 395 396 /* if the reth length field is zero we can assume nothing 397 * about the rkey value and should not validate or use it. 398 * Instead set qp->resp.rkey to 0 which is an invalid rkey 399 * value since the minimum index part is 1. 400 */ 401 static void qp_resp_from_reth(struct rxe_qp *qp, struct rxe_pkt_info *pkt) 402 { 403 unsigned int length = reth_len(pkt); 404 405 qp->resp.va = reth_va(pkt); 406 qp->resp.offset = 0; 407 qp->resp.resid = length; 408 qp->resp.length = length; 409 if (pkt->mask & RXE_READ_OR_WRITE_MASK && length == 0) 410 qp->resp.rkey = 0; 411 else 412 qp->resp.rkey = reth_rkey(pkt); 413 } 414 415 static void qp_resp_from_atmeth(struct rxe_qp *qp, struct rxe_pkt_info *pkt) 416 { 417 qp->resp.va = atmeth_va(pkt); 418 qp->resp.offset = 0; 419 qp->resp.rkey = atmeth_rkey(pkt); 420 qp->resp.resid = sizeof(u64); 421 } 422 423 /* resolve the packet rkey to qp->resp.mr or set qp->resp.mr to NULL 424 * if an invalid rkey is received or the rdma length is zero. For middle 425 * or last packets use the stored value of mr. 426 */ 427 static enum resp_states check_rkey(struct rxe_qp *qp, 428 struct rxe_pkt_info *pkt) 429 { 430 struct rxe_mr *mr = NULL; 431 struct rxe_mw *mw = NULL; 432 u64 va; 433 u32 rkey; 434 u32 resid; 435 u32 pktlen; 436 int mtu = qp->mtu; 437 enum resp_states state; 438 int access = 0; 439 440 /* parse RETH or ATMETH header for first/only packets 441 * for va, length, rkey, etc. or use current value for 442 * middle/last packets. 443 */ 444 if (pkt->mask & (RXE_READ_OR_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) { 445 if (pkt->mask & RXE_RETH_MASK) 446 qp_resp_from_reth(qp, pkt); 447 448 access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ 449 : IB_ACCESS_REMOTE_WRITE; 450 } else if (pkt->mask & RXE_FLUSH_MASK) { 451 u32 flush_type = feth_plt(pkt); 452 453 if (pkt->mask & RXE_RETH_MASK) 454 qp_resp_from_reth(qp, pkt); 455 456 if (flush_type & IB_FLUSH_GLOBAL) 457 access |= IB_ACCESS_FLUSH_GLOBAL; 458 if (flush_type & IB_FLUSH_PERSISTENT) 459 access |= IB_ACCESS_FLUSH_PERSISTENT; 460 } else if (pkt->mask & RXE_ATOMIC_MASK) { 461 qp_resp_from_atmeth(qp, pkt); 462 access = IB_ACCESS_REMOTE_ATOMIC; 463 } else { 464 /* shouldn't happen */ 465 WARN_ON(1); 466 } 467 468 /* A zero-byte read or write op is not required to 469 * set an addr or rkey. See C9-88 470 */ 471 if ((pkt->mask & RXE_READ_OR_WRITE_MASK) && 472 (pkt->mask & RXE_RETH_MASK) && reth_len(pkt) == 0) { 473 qp->resp.mr = NULL; 474 return RESPST_EXECUTE; 475 } 476 477 va = qp->resp.va; 478 rkey = qp->resp.rkey; 479 resid = qp->resp.resid; 480 pktlen = payload_size(pkt); 481 482 if (rkey_is_mw(rkey)) { 483 mw = rxe_lookup_mw(qp, access, rkey); 484 if (!mw) { 485 rxe_dbg_qp(qp, "no MW matches rkey %#x\n", rkey); 486 state = RESPST_ERR_RKEY_VIOLATION; 487 goto err; 488 } 489 490 mr = mw->mr; 491 if (!mr) { 492 rxe_dbg_qp(qp, "MW doesn't have an MR\n"); 493 state = RESPST_ERR_RKEY_VIOLATION; 494 goto err; 495 } 496 497 if (mw->access & IB_ZERO_BASED) 498 qp->resp.offset = mw->addr; 499 500 rxe_get(mr); 501 rxe_put(mw); 502 mw = NULL; 503 } else { 504 mr = lookup_mr(qp->pd, access, rkey, RXE_LOOKUP_REMOTE); 505 if (!mr) { 506 rxe_dbg_qp(qp, "no MR matches rkey %#x\n", rkey); 507 state = RESPST_ERR_RKEY_VIOLATION; 508 goto err; 509 } 510 } 511 512 if (pkt->mask & RXE_FLUSH_MASK) { 513 /* FLUSH MR may not set va or resid 514 * no need to check range since we will flush whole mr 515 */ 516 if (feth_sel(pkt) == IB_FLUSH_MR) 517 goto skip_check_range; 518 } 519 520 if (mr_check_range(mr, va + qp->resp.offset, resid)) { 521 state = RESPST_ERR_RKEY_VIOLATION; 522 goto err; 523 } 524 525 skip_check_range: 526 if (pkt->mask & (RXE_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) { 527 if (resid > mtu) { 528 if (pktlen != mtu || bth_pad(pkt)) { 529 state = RESPST_ERR_LENGTH; 530 goto err; 531 } 532 } else { 533 if (pktlen != resid) { 534 state = RESPST_ERR_LENGTH; 535 goto err; 536 } 537 if ((bth_pad(pkt) != (0x3 & (-resid)))) { 538 /* This case may not be exactly that 539 * but nothing else fits. 540 */ 541 state = RESPST_ERR_LENGTH; 542 goto err; 543 } 544 } 545 } 546 547 WARN_ON_ONCE(qp->resp.mr); 548 549 qp->resp.mr = mr; 550 return RESPST_EXECUTE; 551 552 err: 553 qp->resp.mr = NULL; 554 if (mr) 555 rxe_put(mr); 556 if (mw) 557 rxe_put(mw); 558 559 return state; 560 } 561 562 static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr, 563 int data_len) 564 { 565 int err; 566 567 err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma, 568 data_addr, data_len, RXE_TO_MR_OBJ); 569 if (unlikely(err)) 570 return (err == -ENOSPC) ? RESPST_ERR_LENGTH 571 : RESPST_ERR_MALFORMED_WQE; 572 573 return RESPST_NONE; 574 } 575 576 static enum resp_states write_data_in(struct rxe_qp *qp, 577 struct rxe_pkt_info *pkt) 578 { 579 enum resp_states rc = RESPST_NONE; 580 int err; 581 int data_len = payload_size(pkt); 582 583 err = rxe_mr_copy(qp->resp.mr, qp->resp.va + qp->resp.offset, 584 payload_addr(pkt), data_len, RXE_TO_MR_OBJ); 585 if (err) { 586 rc = RESPST_ERR_RKEY_VIOLATION; 587 goto out; 588 } 589 590 qp->resp.va += data_len; 591 qp->resp.resid -= data_len; 592 593 out: 594 return rc; 595 } 596 597 static struct resp_res *rxe_prepare_res(struct rxe_qp *qp, 598 struct rxe_pkt_info *pkt, 599 int type) 600 { 601 struct resp_res *res; 602 u32 pkts; 603 604 res = &qp->resp.resources[qp->resp.res_head]; 605 rxe_advance_resp_resource(qp); 606 free_rd_atomic_resource(res); 607 608 res->type = type; 609 res->replay = 0; 610 611 switch (type) { 612 case RXE_READ_MASK: 613 res->read.va = qp->resp.va + qp->resp.offset; 614 res->read.va_org = qp->resp.va + qp->resp.offset; 615 res->read.resid = qp->resp.resid; 616 res->read.length = qp->resp.resid; 617 res->read.rkey = qp->resp.rkey; 618 619 pkts = max_t(u32, (reth_len(pkt) + qp->mtu - 1)/qp->mtu, 1); 620 res->first_psn = pkt->psn; 621 res->cur_psn = pkt->psn; 622 res->last_psn = (pkt->psn + pkts - 1) & BTH_PSN_MASK; 623 624 res->state = rdatm_res_state_new; 625 break; 626 case RXE_ATOMIC_MASK: 627 case RXE_ATOMIC_WRITE_MASK: 628 res->first_psn = pkt->psn; 629 res->last_psn = pkt->psn; 630 res->cur_psn = pkt->psn; 631 break; 632 case RXE_FLUSH_MASK: 633 res->flush.va = qp->resp.va + qp->resp.offset; 634 res->flush.length = qp->resp.length; 635 res->flush.type = feth_plt(pkt); 636 res->flush.level = feth_sel(pkt); 637 } 638 639 return res; 640 } 641 642 static enum resp_states process_flush(struct rxe_qp *qp, 643 struct rxe_pkt_info *pkt) 644 { 645 u64 length, start; 646 struct rxe_mr *mr = qp->resp.mr; 647 struct resp_res *res = qp->resp.res; 648 649 /* oA19-14, oA19-15 */ 650 if (res && res->replay) 651 return RESPST_ACKNOWLEDGE; 652 else if (!res) { 653 res = rxe_prepare_res(qp, pkt, RXE_FLUSH_MASK); 654 qp->resp.res = res; 655 } 656 657 if (res->flush.level == IB_FLUSH_RANGE) { 658 start = res->flush.va; 659 length = res->flush.length; 660 } else { /* level == IB_FLUSH_MR */ 661 start = mr->ibmr.iova; 662 length = mr->ibmr.length; 663 } 664 665 if (res->flush.type & IB_FLUSH_PERSISTENT) { 666 if (rxe_flush_pmem_iova(mr, start, length)) 667 return RESPST_ERR_RKEY_VIOLATION; 668 /* Make data persistent. */ 669 wmb(); 670 } else if (res->flush.type & IB_FLUSH_GLOBAL) { 671 /* Make data global visibility. */ 672 wmb(); 673 } 674 675 qp->resp.msn++; 676 677 /* next expected psn, read handles this separately */ 678 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK; 679 qp->resp.ack_psn = qp->resp.psn; 680 681 qp->resp.opcode = pkt->opcode; 682 qp->resp.status = IB_WC_SUCCESS; 683 684 return RESPST_ACKNOWLEDGE; 685 } 686 687 static enum resp_states atomic_reply(struct rxe_qp *qp, 688 struct rxe_pkt_info *pkt) 689 { 690 struct rxe_mr *mr = qp->resp.mr; 691 struct resp_res *res = qp->resp.res; 692 int err; 693 694 if (!res) { 695 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_MASK); 696 qp->resp.res = res; 697 } 698 699 if (!res->replay) { 700 u64 iova = qp->resp.va + qp->resp.offset; 701 702 err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode, 703 atmeth_comp(pkt), 704 atmeth_swap_add(pkt), 705 &res->atomic.orig_val); 706 if (err) 707 return err; 708 709 qp->resp.msn++; 710 711 /* next expected psn, read handles this separately */ 712 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK; 713 qp->resp.ack_psn = qp->resp.psn; 714 715 qp->resp.opcode = pkt->opcode; 716 qp->resp.status = IB_WC_SUCCESS; 717 } 718 719 return RESPST_ACKNOWLEDGE; 720 } 721 722 static enum resp_states atomic_write_reply(struct rxe_qp *qp, 723 struct rxe_pkt_info *pkt) 724 { 725 struct resp_res *res = qp->resp.res; 726 struct rxe_mr *mr; 727 u64 value; 728 u64 iova; 729 int err; 730 731 if (!res) { 732 res = rxe_prepare_res(qp, pkt, RXE_ATOMIC_WRITE_MASK); 733 qp->resp.res = res; 734 } 735 736 if (res->replay) 737 return RESPST_ACKNOWLEDGE; 738 739 mr = qp->resp.mr; 740 value = *(u64 *)payload_addr(pkt); 741 iova = qp->resp.va + qp->resp.offset; 742 743 err = rxe_mr_do_atomic_write(mr, iova, value); 744 if (err) 745 return err; 746 747 qp->resp.resid = 0; 748 qp->resp.msn++; 749 750 /* next expected psn, read handles this separately */ 751 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK; 752 qp->resp.ack_psn = qp->resp.psn; 753 754 qp->resp.opcode = pkt->opcode; 755 qp->resp.status = IB_WC_SUCCESS; 756 757 return RESPST_ACKNOWLEDGE; 758 } 759 760 static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, 761 struct rxe_pkt_info *ack, 762 int opcode, 763 int payload, 764 u32 psn, 765 u8 syndrome) 766 { 767 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 768 struct sk_buff *skb; 769 int paylen; 770 int pad; 771 int err; 772 773 /* 774 * allocate packet 775 */ 776 pad = (-payload) & 0x3; 777 paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE; 778 779 skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack); 780 if (!skb) 781 return NULL; 782 783 ack->qp = qp; 784 ack->opcode = opcode; 785 ack->mask = rxe_opcode[opcode].mask; 786 ack->paylen = paylen; 787 ack->psn = psn; 788 789 bth_init(ack, opcode, 0, 0, pad, IB_DEFAULT_PKEY_FULL, 790 qp->attr.dest_qp_num, 0, psn); 791 792 if (ack->mask & RXE_AETH_MASK) { 793 aeth_set_syn(ack, syndrome); 794 aeth_set_msn(ack, qp->resp.msn); 795 } 796 797 if (ack->mask & RXE_ATMACK_MASK) 798 atmack_set_orig(ack, qp->resp.res->atomic.orig_val); 799 800 err = rxe_prepare(&qp->pri_av, ack, skb); 801 if (err) { 802 kfree_skb(skb); 803 return NULL; 804 } 805 806 return skb; 807 } 808 809 /** 810 * rxe_recheck_mr - revalidate MR from rkey and get a reference 811 * @qp: the qp 812 * @rkey: the rkey 813 * 814 * This code allows the MR to be invalidated or deregistered or 815 * the MW if one was used to be invalidated or deallocated. 816 * It is assumed that the access permissions if originally good 817 * are OK and the mappings to be unchanged. 818 * 819 * TODO: If someone reregisters an MR to change its size or 820 * access permissions during the processing of an RDMA read 821 * we should kill the responder resource and complete the 822 * operation with an error. 823 * 824 * Return: mr on success else NULL 825 */ 826 static struct rxe_mr *rxe_recheck_mr(struct rxe_qp *qp, u32 rkey) 827 { 828 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 829 struct rxe_mr *mr; 830 struct rxe_mw *mw; 831 832 if (rkey_is_mw(rkey)) { 833 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8); 834 if (!mw) 835 return NULL; 836 837 mr = mw->mr; 838 if (mw->rkey != rkey || mw->state != RXE_MW_STATE_VALID || 839 !mr || mr->state != RXE_MR_STATE_VALID) { 840 rxe_put(mw); 841 return NULL; 842 } 843 844 rxe_get(mr); 845 rxe_put(mw); 846 847 return mr; 848 } 849 850 mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8); 851 if (!mr) 852 return NULL; 853 854 if (mr->rkey != rkey || mr->state != RXE_MR_STATE_VALID) { 855 rxe_put(mr); 856 return NULL; 857 } 858 859 return mr; 860 } 861 862 /* RDMA read response. If res is not NULL, then we have a current RDMA request 863 * being processed or replayed. 864 */ 865 static enum resp_states read_reply(struct rxe_qp *qp, 866 struct rxe_pkt_info *req_pkt) 867 { 868 struct rxe_pkt_info ack_pkt; 869 struct sk_buff *skb; 870 int mtu = qp->mtu; 871 enum resp_states state; 872 int payload; 873 int opcode; 874 int err; 875 struct resp_res *res = qp->resp.res; 876 struct rxe_mr *mr; 877 878 if (!res) { 879 res = rxe_prepare_res(qp, req_pkt, RXE_READ_MASK); 880 qp->resp.res = res; 881 } 882 883 if (res->state == rdatm_res_state_new) { 884 if (!res->replay || qp->resp.length == 0) { 885 /* if length == 0 mr will be NULL (is ok) 886 * otherwise qp->resp.mr holds a ref on mr 887 * which we transfer to mr and drop below. 888 */ 889 mr = qp->resp.mr; 890 qp->resp.mr = NULL; 891 } else { 892 mr = rxe_recheck_mr(qp, res->read.rkey); 893 if (!mr) 894 return RESPST_ERR_RKEY_VIOLATION; 895 } 896 897 if (res->read.resid <= mtu) 898 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY; 899 else 900 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST; 901 } else { 902 /* re-lookup mr from rkey on all later packets. 903 * length will be non-zero. This can fail if someone 904 * modifies or destroys the mr since the first packet. 905 */ 906 mr = rxe_recheck_mr(qp, res->read.rkey); 907 if (!mr) 908 return RESPST_ERR_RKEY_VIOLATION; 909 910 if (res->read.resid > mtu) 911 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE; 912 else 913 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST; 914 } 915 916 res->state = rdatm_res_state_next; 917 918 payload = min_t(int, res->read.resid, mtu); 919 920 skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload, 921 res->cur_psn, AETH_ACK_UNLIMITED); 922 if (!skb) { 923 state = RESPST_ERR_RNR; 924 goto err_out; 925 } 926 927 err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt), 928 payload, RXE_FROM_MR_OBJ); 929 if (err) { 930 kfree_skb(skb); 931 state = RESPST_ERR_RKEY_VIOLATION; 932 goto err_out; 933 } 934 935 if (bth_pad(&ack_pkt)) { 936 u8 *pad = payload_addr(&ack_pkt) + payload; 937 938 memset(pad, 0, bth_pad(&ack_pkt)); 939 } 940 941 /* rxe_xmit_packet always consumes the skb */ 942 err = rxe_xmit_packet(qp, &ack_pkt, skb); 943 if (err) { 944 state = RESPST_ERR_RNR; 945 goto err_out; 946 } 947 948 res->read.va += payload; 949 res->read.resid -= payload; 950 res->cur_psn = (res->cur_psn + 1) & BTH_PSN_MASK; 951 952 if (res->read.resid > 0) { 953 state = RESPST_DONE; 954 } else { 955 qp->resp.res = NULL; 956 if (!res->replay) 957 qp->resp.opcode = -1; 958 if (psn_compare(res->cur_psn, qp->resp.psn) >= 0) 959 qp->resp.psn = res->cur_psn; 960 state = RESPST_CLEANUP; 961 } 962 963 err_out: 964 if (mr) 965 rxe_put(mr); 966 return state; 967 } 968 969 static int invalidate_rkey(struct rxe_qp *qp, u32 rkey) 970 { 971 if (rkey_is_mw(rkey)) 972 return rxe_invalidate_mw(qp, rkey); 973 else 974 return rxe_invalidate_mr(qp, rkey); 975 } 976 977 /* Executes a new request. A retried request never reach that function (send 978 * and writes are discarded, and reads and atomics are retried elsewhere. 979 */ 980 static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt) 981 { 982 enum resp_states err; 983 struct sk_buff *skb = PKT_TO_SKB(pkt); 984 union rdma_network_hdr hdr; 985 986 if (pkt->mask & RXE_SEND_MASK) { 987 if (qp_type(qp) == IB_QPT_UD || 988 qp_type(qp) == IB_QPT_GSI) { 989 if (skb->protocol == htons(ETH_P_IP)) { 990 memset(&hdr.reserved, 0, 991 sizeof(hdr.reserved)); 992 memcpy(&hdr.roce4grh, ip_hdr(skb), 993 sizeof(hdr.roce4grh)); 994 err = send_data_in(qp, &hdr, sizeof(hdr)); 995 } else { 996 err = send_data_in(qp, ipv6_hdr(skb), 997 sizeof(hdr)); 998 } 999 if (err) 1000 return err; 1001 } 1002 err = send_data_in(qp, payload_addr(pkt), payload_size(pkt)); 1003 if (err) 1004 return err; 1005 } else if (pkt->mask & RXE_WRITE_MASK) { 1006 err = write_data_in(qp, pkt); 1007 if (err) 1008 return err; 1009 } else if (pkt->mask & RXE_READ_MASK) { 1010 /* For RDMA Read we can increment the msn now. See C9-148. */ 1011 qp->resp.msn++; 1012 return RESPST_READ_REPLY; 1013 } else if (pkt->mask & RXE_ATOMIC_MASK) { 1014 return RESPST_ATOMIC_REPLY; 1015 } else if (pkt->mask & RXE_ATOMIC_WRITE_MASK) { 1016 return RESPST_ATOMIC_WRITE_REPLY; 1017 } else if (pkt->mask & RXE_FLUSH_MASK) { 1018 return RESPST_PROCESS_FLUSH; 1019 } else { 1020 /* Unreachable */ 1021 WARN_ON_ONCE(1); 1022 } 1023 1024 if (pkt->mask & RXE_IETH_MASK) { 1025 u32 rkey = ieth_rkey(pkt); 1026 1027 err = invalidate_rkey(qp, rkey); 1028 if (err) 1029 return RESPST_ERR_INVALIDATE_RKEY; 1030 } 1031 1032 if (pkt->mask & RXE_END_MASK) 1033 /* We successfully processed this new request. */ 1034 qp->resp.msn++; 1035 1036 /* next expected psn, read handles this separately */ 1037 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK; 1038 qp->resp.ack_psn = qp->resp.psn; 1039 1040 qp->resp.opcode = pkt->opcode; 1041 qp->resp.status = IB_WC_SUCCESS; 1042 1043 if (pkt->mask & RXE_COMP_MASK) 1044 return RESPST_COMPLETE; 1045 else if (qp_type(qp) == IB_QPT_RC) 1046 return RESPST_ACKNOWLEDGE; 1047 else 1048 return RESPST_CLEANUP; 1049 } 1050 1051 static enum resp_states do_complete(struct rxe_qp *qp, 1052 struct rxe_pkt_info *pkt) 1053 { 1054 struct rxe_cqe cqe; 1055 struct ib_wc *wc = &cqe.ibwc; 1056 struct ib_uverbs_wc *uwc = &cqe.uibwc; 1057 struct rxe_recv_wqe *wqe = qp->resp.wqe; 1058 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 1059 unsigned long flags; 1060 1061 if (!wqe) 1062 goto finish; 1063 1064 memset(&cqe, 0, sizeof(cqe)); 1065 1066 if (qp->rcq->is_user) { 1067 uwc->status = qp->resp.status; 1068 uwc->qp_num = qp->ibqp.qp_num; 1069 uwc->wr_id = wqe->wr_id; 1070 } else { 1071 wc->status = qp->resp.status; 1072 wc->qp = &qp->ibqp; 1073 wc->wr_id = wqe->wr_id; 1074 } 1075 1076 if (wc->status == IB_WC_SUCCESS) { 1077 rxe_counter_inc(rxe, RXE_CNT_RDMA_RECV); 1078 wc->opcode = (pkt->mask & RXE_IMMDT_MASK && 1079 pkt->mask & RXE_WRITE_MASK) ? 1080 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; 1081 wc->byte_len = (pkt->mask & RXE_IMMDT_MASK && 1082 pkt->mask & RXE_WRITE_MASK) ? 1083 qp->resp.length : wqe->dma.length - wqe->dma.resid; 1084 1085 /* fields after byte_len are different between kernel and user 1086 * space 1087 */ 1088 if (qp->rcq->is_user) { 1089 uwc->wc_flags = IB_WC_GRH; 1090 1091 if (pkt->mask & RXE_IMMDT_MASK) { 1092 uwc->wc_flags |= IB_WC_WITH_IMM; 1093 uwc->ex.imm_data = immdt_imm(pkt); 1094 } 1095 1096 if (pkt->mask & RXE_IETH_MASK) { 1097 uwc->wc_flags |= IB_WC_WITH_INVALIDATE; 1098 uwc->ex.invalidate_rkey = ieth_rkey(pkt); 1099 } 1100 1101 if (pkt->mask & RXE_DETH_MASK) 1102 uwc->src_qp = deth_sqp(pkt); 1103 1104 uwc->port_num = qp->attr.port_num; 1105 } else { 1106 struct sk_buff *skb = PKT_TO_SKB(pkt); 1107 1108 wc->wc_flags = IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE; 1109 if (skb->protocol == htons(ETH_P_IP)) 1110 wc->network_hdr_type = RDMA_NETWORK_IPV4; 1111 else 1112 wc->network_hdr_type = RDMA_NETWORK_IPV6; 1113 1114 if (is_vlan_dev(skb->dev)) { 1115 wc->wc_flags |= IB_WC_WITH_VLAN; 1116 wc->vlan_id = vlan_dev_vlan_id(skb->dev); 1117 } 1118 1119 if (pkt->mask & RXE_IMMDT_MASK) { 1120 wc->wc_flags |= IB_WC_WITH_IMM; 1121 wc->ex.imm_data = immdt_imm(pkt); 1122 } 1123 1124 if (pkt->mask & RXE_IETH_MASK) { 1125 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 1126 wc->ex.invalidate_rkey = ieth_rkey(pkt); 1127 } 1128 1129 if (pkt->mask & RXE_DETH_MASK) 1130 wc->src_qp = deth_sqp(pkt); 1131 1132 wc->port_num = qp->attr.port_num; 1133 } 1134 } else { 1135 if (wc->status != IB_WC_WR_FLUSH_ERR) 1136 rxe_err_qp(qp, "non-flush error status = %d", 1137 wc->status); 1138 } 1139 1140 /* have copy for srq and reference for !srq */ 1141 if (!qp->srq) 1142 queue_advance_consumer(qp->rq.queue, QUEUE_TYPE_FROM_CLIENT); 1143 1144 qp->resp.wqe = NULL; 1145 1146 if (rxe_cq_post(qp->rcq, &cqe, pkt ? bth_se(pkt) : 1)) 1147 return RESPST_ERR_CQ_OVERFLOW; 1148 1149 finish: 1150 spin_lock_irqsave(&qp->state_lock, flags); 1151 if (unlikely(qp_state(qp) == IB_QPS_ERR)) { 1152 spin_unlock_irqrestore(&qp->state_lock, flags); 1153 return RESPST_CHK_RESOURCE; 1154 } 1155 spin_unlock_irqrestore(&qp->state_lock, flags); 1156 1157 if (unlikely(!pkt)) 1158 return RESPST_DONE; 1159 if (qp_type(qp) == IB_QPT_RC) 1160 return RESPST_ACKNOWLEDGE; 1161 else 1162 return RESPST_CLEANUP; 1163 } 1164 1165 1166 static int send_common_ack(struct rxe_qp *qp, u8 syndrome, u32 psn, 1167 int opcode, const char *msg) 1168 { 1169 int err; 1170 struct rxe_pkt_info ack_pkt; 1171 struct sk_buff *skb; 1172 1173 skb = prepare_ack_packet(qp, &ack_pkt, opcode, 0, psn, syndrome); 1174 if (!skb) 1175 return -ENOMEM; 1176 1177 err = rxe_xmit_packet(qp, &ack_pkt, skb); 1178 if (err) 1179 rxe_dbg_qp(qp, "Failed sending %s\n", msg); 1180 1181 return err; 1182 } 1183 1184 static int send_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) 1185 { 1186 return send_common_ack(qp, syndrome, psn, 1187 IB_OPCODE_RC_ACKNOWLEDGE, "ACK"); 1188 } 1189 1190 static int send_atomic_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) 1191 { 1192 int ret = send_common_ack(qp, syndrome, psn, 1193 IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE, "ATOMIC ACK"); 1194 1195 /* have to clear this since it is used to trigger 1196 * long read replies 1197 */ 1198 qp->resp.res = NULL; 1199 return ret; 1200 } 1201 1202 static int send_read_response_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) 1203 { 1204 int ret = send_common_ack(qp, syndrome, psn, 1205 IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY, 1206 "RDMA READ response of length zero ACK"); 1207 1208 /* have to clear this since it is used to trigger 1209 * long read replies 1210 */ 1211 qp->resp.res = NULL; 1212 return ret; 1213 } 1214 1215 static enum resp_states acknowledge(struct rxe_qp *qp, 1216 struct rxe_pkt_info *pkt) 1217 { 1218 if (qp_type(qp) != IB_QPT_RC) 1219 return RESPST_CLEANUP; 1220 1221 if (qp->resp.aeth_syndrome != AETH_ACK_UNLIMITED) 1222 send_ack(qp, qp->resp.aeth_syndrome, pkt->psn); 1223 else if (pkt->mask & RXE_ATOMIC_MASK) 1224 send_atomic_ack(qp, AETH_ACK_UNLIMITED, pkt->psn); 1225 else if (pkt->mask & (RXE_FLUSH_MASK | RXE_ATOMIC_WRITE_MASK)) 1226 send_read_response_ack(qp, AETH_ACK_UNLIMITED, pkt->psn); 1227 else if (bth_ack(pkt)) 1228 send_ack(qp, AETH_ACK_UNLIMITED, pkt->psn); 1229 1230 return RESPST_CLEANUP; 1231 } 1232 1233 static enum resp_states cleanup(struct rxe_qp *qp, 1234 struct rxe_pkt_info *pkt) 1235 { 1236 struct sk_buff *skb; 1237 1238 if (pkt) { 1239 skb = skb_dequeue(&qp->req_pkts); 1240 rxe_put(qp); 1241 kfree_skb(skb); 1242 ib_device_put(qp->ibqp.device); 1243 } 1244 1245 if (qp->resp.mr) { 1246 rxe_put(qp->resp.mr); 1247 qp->resp.mr = NULL; 1248 } 1249 1250 return RESPST_DONE; 1251 } 1252 1253 static struct resp_res *find_resource(struct rxe_qp *qp, u32 psn) 1254 { 1255 int i; 1256 1257 for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) { 1258 struct resp_res *res = &qp->resp.resources[i]; 1259 1260 if (res->type == 0) 1261 continue; 1262 1263 if (psn_compare(psn, res->first_psn) >= 0 && 1264 psn_compare(psn, res->last_psn) <= 0) { 1265 return res; 1266 } 1267 } 1268 1269 return NULL; 1270 } 1271 1272 static enum resp_states duplicate_request(struct rxe_qp *qp, 1273 struct rxe_pkt_info *pkt) 1274 { 1275 enum resp_states rc; 1276 u32 prev_psn = (qp->resp.ack_psn - 1) & BTH_PSN_MASK; 1277 1278 if (pkt->mask & RXE_SEND_MASK || 1279 pkt->mask & RXE_WRITE_MASK) { 1280 /* SEND. Ack again and cleanup. C9-105. */ 1281 send_ack(qp, AETH_ACK_UNLIMITED, prev_psn); 1282 return RESPST_CLEANUP; 1283 } else if (pkt->mask & RXE_FLUSH_MASK) { 1284 struct resp_res *res; 1285 1286 /* Find the operation in our list of responder resources. */ 1287 res = find_resource(qp, pkt->psn); 1288 if (res) { 1289 res->replay = 1; 1290 res->cur_psn = pkt->psn; 1291 qp->resp.res = res; 1292 rc = RESPST_PROCESS_FLUSH; 1293 goto out; 1294 } 1295 1296 /* Resource not found. Class D error. Drop the request. */ 1297 rc = RESPST_CLEANUP; 1298 goto out; 1299 } else if (pkt->mask & RXE_READ_MASK) { 1300 struct resp_res *res; 1301 1302 res = find_resource(qp, pkt->psn); 1303 if (!res) { 1304 /* Resource not found. Class D error. Drop the 1305 * request. 1306 */ 1307 rc = RESPST_CLEANUP; 1308 goto out; 1309 } else { 1310 /* Ensure this new request is the same as the previous 1311 * one or a subset of it. 1312 */ 1313 u64 iova = reth_va(pkt); 1314 u32 resid = reth_len(pkt); 1315 1316 if (iova < res->read.va_org || 1317 resid > res->read.length || 1318 (iova + resid) > (res->read.va_org + 1319 res->read.length)) { 1320 rc = RESPST_CLEANUP; 1321 goto out; 1322 } 1323 1324 if (reth_rkey(pkt) != res->read.rkey) { 1325 rc = RESPST_CLEANUP; 1326 goto out; 1327 } 1328 1329 res->cur_psn = pkt->psn; 1330 res->state = (pkt->psn == res->first_psn) ? 1331 rdatm_res_state_new : 1332 rdatm_res_state_replay; 1333 res->replay = 1; 1334 1335 /* Reset the resource, except length. */ 1336 res->read.va_org = iova; 1337 res->read.va = iova; 1338 res->read.resid = resid; 1339 1340 /* Replay the RDMA read reply. */ 1341 qp->resp.res = res; 1342 rc = RESPST_READ_REPLY; 1343 goto out; 1344 } 1345 } else { 1346 struct resp_res *res; 1347 1348 /* Find the operation in our list of responder resources. */ 1349 res = find_resource(qp, pkt->psn); 1350 if (res) { 1351 res->replay = 1; 1352 res->cur_psn = pkt->psn; 1353 qp->resp.res = res; 1354 rc = pkt->mask & RXE_ATOMIC_MASK ? 1355 RESPST_ATOMIC_REPLY : 1356 RESPST_ATOMIC_WRITE_REPLY; 1357 goto out; 1358 } 1359 1360 /* Resource not found. Class D error. Drop the request. */ 1361 rc = RESPST_CLEANUP; 1362 goto out; 1363 } 1364 out: 1365 return rc; 1366 } 1367 1368 /* Process a class A or C. Both are treated the same in this implementation. */ 1369 static void do_class_ac_error(struct rxe_qp *qp, u8 syndrome, 1370 enum ib_wc_status status) 1371 { 1372 qp->resp.aeth_syndrome = syndrome; 1373 qp->resp.status = status; 1374 1375 /* indicate that we should go through the ERROR state */ 1376 qp->resp.goto_error = 1; 1377 } 1378 1379 static enum resp_states do_class_d1e_error(struct rxe_qp *qp) 1380 { 1381 /* UC */ 1382 if (qp->srq) { 1383 /* Class E */ 1384 qp->resp.drop_msg = 1; 1385 if (qp->resp.wqe) { 1386 qp->resp.status = IB_WC_REM_INV_REQ_ERR; 1387 return RESPST_COMPLETE; 1388 } else { 1389 return RESPST_CLEANUP; 1390 } 1391 } else { 1392 /* Class D1. This packet may be the start of a 1393 * new message and could be valid. The previous 1394 * message is invalid and ignored. reset the 1395 * recv wr to its original state 1396 */ 1397 if (qp->resp.wqe) { 1398 qp->resp.wqe->dma.resid = qp->resp.wqe->dma.length; 1399 qp->resp.wqe->dma.cur_sge = 0; 1400 qp->resp.wqe->dma.sge_offset = 0; 1401 qp->resp.opcode = -1; 1402 } 1403 1404 if (qp->resp.mr) { 1405 rxe_put(qp->resp.mr); 1406 qp->resp.mr = NULL; 1407 } 1408 1409 return RESPST_CLEANUP; 1410 } 1411 } 1412 1413 /* drain incoming request packet queue */ 1414 static void drain_req_pkts(struct rxe_qp *qp) 1415 { 1416 struct sk_buff *skb; 1417 1418 while ((skb = skb_dequeue(&qp->req_pkts))) { 1419 rxe_put(qp); 1420 kfree_skb(skb); 1421 ib_device_put(qp->ibqp.device); 1422 } 1423 } 1424 1425 /* complete receive wqe with flush error */ 1426 static int flush_recv_wqe(struct rxe_qp *qp, struct rxe_recv_wqe *wqe) 1427 { 1428 struct rxe_cqe cqe = {}; 1429 struct ib_wc *wc = &cqe.ibwc; 1430 struct ib_uverbs_wc *uwc = &cqe.uibwc; 1431 int err; 1432 1433 if (qp->rcq->is_user) { 1434 uwc->wr_id = wqe->wr_id; 1435 uwc->status = IB_WC_WR_FLUSH_ERR; 1436 uwc->qp_num = qp_num(qp); 1437 } else { 1438 wc->wr_id = wqe->wr_id; 1439 wc->status = IB_WC_WR_FLUSH_ERR; 1440 wc->qp = &qp->ibqp; 1441 } 1442 1443 err = rxe_cq_post(qp->rcq, &cqe, 0); 1444 if (err) 1445 rxe_dbg_cq(qp->rcq, "post cq failed err = %d", err); 1446 1447 return err; 1448 } 1449 1450 /* drain and optionally complete the recive queue 1451 * if unable to complete a wqe stop completing and 1452 * just flush the remaining wqes 1453 */ 1454 static void flush_recv_queue(struct rxe_qp *qp, bool notify) 1455 { 1456 struct rxe_queue *q = qp->rq.queue; 1457 struct rxe_recv_wqe *wqe; 1458 int err; 1459 1460 if (qp->srq) { 1461 if (notify && qp->ibqp.event_handler) { 1462 struct ib_event ev; 1463 1464 ev.device = qp->ibqp.device; 1465 ev.element.qp = &qp->ibqp; 1466 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 1467 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1468 } 1469 return; 1470 } 1471 1472 /* recv queue not created. nothing to do. */ 1473 if (!qp->rq.queue) 1474 return; 1475 1476 while ((wqe = queue_head(q, q->type))) { 1477 if (notify) { 1478 err = flush_recv_wqe(qp, wqe); 1479 if (err) 1480 notify = 0; 1481 } 1482 queue_advance_consumer(q, q->type); 1483 } 1484 1485 qp->resp.wqe = NULL; 1486 } 1487 1488 int rxe_responder(struct rxe_qp *qp) 1489 { 1490 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 1491 enum resp_states state; 1492 struct rxe_pkt_info *pkt = NULL; 1493 int ret; 1494 unsigned long flags; 1495 1496 spin_lock_irqsave(&qp->state_lock, flags); 1497 if (!qp->valid || qp_state(qp) == IB_QPS_ERR || 1498 qp_state(qp) == IB_QPS_RESET) { 1499 bool notify = qp->valid && (qp_state(qp) == IB_QPS_ERR); 1500 1501 drain_req_pkts(qp); 1502 flush_recv_queue(qp, notify); 1503 spin_unlock_irqrestore(&qp->state_lock, flags); 1504 goto exit; 1505 } 1506 spin_unlock_irqrestore(&qp->state_lock, flags); 1507 1508 qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED; 1509 1510 state = RESPST_GET_REQ; 1511 1512 while (1) { 1513 rxe_dbg_qp(qp, "state = %s\n", resp_state_name[state]); 1514 switch (state) { 1515 case RESPST_GET_REQ: 1516 state = get_req(qp, &pkt); 1517 break; 1518 case RESPST_CHK_PSN: 1519 state = check_psn(qp, pkt); 1520 break; 1521 case RESPST_CHK_OP_SEQ: 1522 state = check_op_seq(qp, pkt); 1523 break; 1524 case RESPST_CHK_OP_VALID: 1525 state = check_op_valid(qp, pkt); 1526 break; 1527 case RESPST_CHK_RESOURCE: 1528 state = check_resource(qp, pkt); 1529 break; 1530 case RESPST_CHK_LENGTH: 1531 state = rxe_resp_check_length(qp, pkt); 1532 break; 1533 case RESPST_CHK_RKEY: 1534 state = check_rkey(qp, pkt); 1535 break; 1536 case RESPST_EXECUTE: 1537 state = execute(qp, pkt); 1538 break; 1539 case RESPST_COMPLETE: 1540 state = do_complete(qp, pkt); 1541 break; 1542 case RESPST_READ_REPLY: 1543 state = read_reply(qp, pkt); 1544 break; 1545 case RESPST_ATOMIC_REPLY: 1546 state = atomic_reply(qp, pkt); 1547 break; 1548 case RESPST_ATOMIC_WRITE_REPLY: 1549 state = atomic_write_reply(qp, pkt); 1550 break; 1551 case RESPST_PROCESS_FLUSH: 1552 state = process_flush(qp, pkt); 1553 break; 1554 case RESPST_ACKNOWLEDGE: 1555 state = acknowledge(qp, pkt); 1556 break; 1557 case RESPST_CLEANUP: 1558 state = cleanup(qp, pkt); 1559 break; 1560 case RESPST_DUPLICATE_REQUEST: 1561 state = duplicate_request(qp, pkt); 1562 break; 1563 case RESPST_ERR_PSN_OUT_OF_SEQ: 1564 /* RC only - Class B. Drop packet. */ 1565 send_ack(qp, AETH_NAK_PSN_SEQ_ERROR, qp->resp.psn); 1566 state = RESPST_CLEANUP; 1567 break; 1568 1569 case RESPST_ERR_TOO_MANY_RDMA_ATM_REQ: 1570 case RESPST_ERR_MISSING_OPCODE_FIRST: 1571 case RESPST_ERR_MISSING_OPCODE_LAST_C: 1572 case RESPST_ERR_UNSUPPORTED_OPCODE: 1573 case RESPST_ERR_MISALIGNED_ATOMIC: 1574 /* RC Only - Class C. */ 1575 do_class_ac_error(qp, AETH_NAK_INVALID_REQ, 1576 IB_WC_REM_INV_REQ_ERR); 1577 state = RESPST_COMPLETE; 1578 break; 1579 1580 case RESPST_ERR_MISSING_OPCODE_LAST_D1E: 1581 state = do_class_d1e_error(qp); 1582 break; 1583 case RESPST_ERR_RNR: 1584 if (qp_type(qp) == IB_QPT_RC) { 1585 rxe_counter_inc(rxe, RXE_CNT_SND_RNR); 1586 /* RC - class B */ 1587 send_ack(qp, AETH_RNR_NAK | 1588 (~AETH_TYPE_MASK & 1589 qp->attr.min_rnr_timer), 1590 pkt->psn); 1591 } else { 1592 /* UD/UC - class D */ 1593 qp->resp.drop_msg = 1; 1594 } 1595 state = RESPST_CLEANUP; 1596 break; 1597 1598 case RESPST_ERR_RKEY_VIOLATION: 1599 if (qp_type(qp) == IB_QPT_RC) { 1600 /* Class C */ 1601 do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR, 1602 IB_WC_REM_ACCESS_ERR); 1603 state = RESPST_COMPLETE; 1604 } else { 1605 qp->resp.drop_msg = 1; 1606 if (qp->srq) { 1607 /* UC/SRQ Class D */ 1608 qp->resp.status = IB_WC_REM_ACCESS_ERR; 1609 state = RESPST_COMPLETE; 1610 } else { 1611 /* UC/non-SRQ Class E. */ 1612 state = RESPST_CLEANUP; 1613 } 1614 } 1615 break; 1616 1617 case RESPST_ERR_INVALIDATE_RKEY: 1618 /* RC - Class J. */ 1619 qp->resp.goto_error = 1; 1620 qp->resp.status = IB_WC_REM_INV_REQ_ERR; 1621 state = RESPST_COMPLETE; 1622 break; 1623 1624 case RESPST_ERR_LENGTH: 1625 if (qp_type(qp) == IB_QPT_RC) { 1626 /* Class C */ 1627 do_class_ac_error(qp, AETH_NAK_INVALID_REQ, 1628 IB_WC_REM_INV_REQ_ERR); 1629 state = RESPST_COMPLETE; 1630 } else if (qp->srq) { 1631 /* UC/UD - class E */ 1632 qp->resp.status = IB_WC_REM_INV_REQ_ERR; 1633 state = RESPST_COMPLETE; 1634 } else { 1635 /* UC/UD - class D */ 1636 qp->resp.drop_msg = 1; 1637 state = RESPST_CLEANUP; 1638 } 1639 break; 1640 1641 case RESPST_ERR_MALFORMED_WQE: 1642 /* All, Class A. */ 1643 do_class_ac_error(qp, AETH_NAK_REM_OP_ERR, 1644 IB_WC_LOC_QP_OP_ERR); 1645 state = RESPST_COMPLETE; 1646 break; 1647 1648 case RESPST_ERR_CQ_OVERFLOW: 1649 /* All - Class G */ 1650 state = RESPST_ERROR; 1651 break; 1652 1653 case RESPST_DONE: 1654 if (qp->resp.goto_error) { 1655 state = RESPST_ERROR; 1656 break; 1657 } 1658 1659 goto done; 1660 1661 case RESPST_EXIT: 1662 if (qp->resp.goto_error) { 1663 state = RESPST_ERROR; 1664 break; 1665 } 1666 1667 goto exit; 1668 1669 case RESPST_ERROR: 1670 qp->resp.goto_error = 0; 1671 rxe_dbg_qp(qp, "moved to error state\n"); 1672 rxe_qp_error(qp); 1673 goto exit; 1674 1675 default: 1676 WARN_ON_ONCE(1); 1677 } 1678 } 1679 1680 /* A non-zero return value will cause rxe_do_task to 1681 * exit its loop and end the work item. A zero return 1682 * will continue looping and return to rxe_responder 1683 */ 1684 done: 1685 ret = 0; 1686 goto out; 1687 exit: 1688 ret = -EAGAIN; 1689 out: 1690 return ret; 1691 } 1692