1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2021, Alibaba Group */ 6 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 7 /* Copyright (c) 2008-2019, IBM Corporation */ 8 9 #include "erdma_cm.h" 10 #include "erdma_verbs.h" 11 12 void erdma_qp_llp_close(struct erdma_qp *qp) 13 { 14 struct erdma_qp_attrs qp_attrs; 15 16 down_write(&qp->state_lock); 17 18 switch (qp->attrs.state) { 19 case ERDMA_QP_STATE_RTS: 20 case ERDMA_QP_STATE_RTR: 21 case ERDMA_QP_STATE_IDLE: 22 case ERDMA_QP_STATE_TERMINATE: 23 qp_attrs.state = ERDMA_QP_STATE_CLOSING; 24 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE); 25 break; 26 case ERDMA_QP_STATE_CLOSING: 27 qp->attrs.state = ERDMA_QP_STATE_IDLE; 28 break; 29 default: 30 break; 31 } 32 33 if (qp->cep) { 34 erdma_cep_put(qp->cep); 35 qp->cep = NULL; 36 } 37 38 up_write(&qp->state_lock); 39 } 40 41 struct ib_qp *erdma_get_ibqp(struct ib_device *ibdev, int id) 42 { 43 struct erdma_qp *qp = find_qp_by_qpn(to_edev(ibdev), id); 44 45 if (qp) 46 return &qp->ibqp; 47 48 return NULL; 49 } 50 51 static int erdma_modify_qp_state_to_rts(struct erdma_qp *qp, 52 struct erdma_qp_attrs *attrs, 53 enum erdma_qp_attr_mask mask) 54 { 55 int ret; 56 struct erdma_dev *dev = qp->dev; 57 struct erdma_cmdq_modify_qp_req req; 58 struct tcp_sock *tp; 59 struct erdma_cep *cep = qp->cep; 60 struct sockaddr_storage local_addr, remote_addr; 61 62 if (!(mask & ERDMA_QP_ATTR_LLP_HANDLE)) 63 return -EINVAL; 64 65 if (!(mask & ERDMA_QP_ATTR_MPA)) 66 return -EINVAL; 67 68 ret = getname_local(cep->sock, &local_addr); 69 if (ret < 0) 70 return ret; 71 72 ret = getname_peer(cep->sock, &remote_addr); 73 if (ret < 0) 74 return ret; 75 76 qp->attrs.state = ERDMA_QP_STATE_RTS; 77 78 tp = tcp_sk(qp->cep->sock->sk); 79 80 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 81 CMDQ_OPCODE_MODIFY_QP); 82 83 req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, qp->attrs.state) | 84 FIELD_PREP(ERDMA_CMD_MODIFY_QP_CC_MASK, qp->attrs.cc) | 85 FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp)); 86 87 req.cookie = be32_to_cpu(qp->cep->mpa.ext_data.cookie); 88 req.dip = to_sockaddr_in(remote_addr).sin_addr.s_addr; 89 req.sip = to_sockaddr_in(local_addr).sin_addr.s_addr; 90 req.dport = to_sockaddr_in(remote_addr).sin_port; 91 req.sport = to_sockaddr_in(local_addr).sin_port; 92 93 req.send_nxt = tp->snd_nxt; 94 /* rsvd tcp seq for mpa-rsp in server. */ 95 if (qp->attrs.qp_type == ERDMA_QP_PASSIVE) 96 req.send_nxt += MPA_DEFAULT_HDR_LEN + qp->attrs.pd_len; 97 req.recv_nxt = tp->rcv_nxt; 98 99 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 100 } 101 102 static int erdma_modify_qp_state_to_stop(struct erdma_qp *qp, 103 struct erdma_qp_attrs *attrs, 104 enum erdma_qp_attr_mask mask) 105 { 106 struct erdma_dev *dev = qp->dev; 107 struct erdma_cmdq_modify_qp_req req; 108 109 qp->attrs.state = attrs->state; 110 111 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 112 CMDQ_OPCODE_MODIFY_QP); 113 114 req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, attrs->state) | 115 FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp)); 116 117 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 118 } 119 120 int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs, 121 enum erdma_qp_attr_mask mask) 122 { 123 bool need_reflush = false; 124 int drop_conn, ret = 0; 125 126 if (!mask) 127 return 0; 128 129 if (!(mask & ERDMA_QP_ATTR_STATE)) 130 return 0; 131 132 switch (qp->attrs.state) { 133 case ERDMA_QP_STATE_IDLE: 134 case ERDMA_QP_STATE_RTR: 135 if (attrs->state == ERDMA_QP_STATE_RTS) { 136 ret = erdma_modify_qp_state_to_rts(qp, attrs, mask); 137 } else if (attrs->state == ERDMA_QP_STATE_ERROR) { 138 qp->attrs.state = ERDMA_QP_STATE_ERROR; 139 need_reflush = true; 140 if (qp->cep) { 141 erdma_cep_put(qp->cep); 142 qp->cep = NULL; 143 } 144 ret = erdma_modify_qp_state_to_stop(qp, attrs, mask); 145 } 146 break; 147 case ERDMA_QP_STATE_RTS: 148 drop_conn = 0; 149 150 if (attrs->state == ERDMA_QP_STATE_CLOSING || 151 attrs->state == ERDMA_QP_STATE_TERMINATE || 152 attrs->state == ERDMA_QP_STATE_ERROR) { 153 ret = erdma_modify_qp_state_to_stop(qp, attrs, mask); 154 drop_conn = 1; 155 need_reflush = true; 156 } 157 158 if (drop_conn) 159 erdma_qp_cm_drop(qp); 160 161 break; 162 case ERDMA_QP_STATE_TERMINATE: 163 if (attrs->state == ERDMA_QP_STATE_ERROR) 164 qp->attrs.state = ERDMA_QP_STATE_ERROR; 165 break; 166 case ERDMA_QP_STATE_CLOSING: 167 if (attrs->state == ERDMA_QP_STATE_IDLE) { 168 qp->attrs.state = ERDMA_QP_STATE_IDLE; 169 } else if (attrs->state == ERDMA_QP_STATE_ERROR) { 170 ret = erdma_modify_qp_state_to_stop(qp, attrs, mask); 171 qp->attrs.state = ERDMA_QP_STATE_ERROR; 172 } else if (attrs->state != ERDMA_QP_STATE_CLOSING) { 173 return -ECONNABORTED; 174 } 175 break; 176 default: 177 break; 178 } 179 180 if (need_reflush && !ret && rdma_is_kernel_res(&qp->ibqp.res)) { 181 qp->flags |= ERDMA_QP_IN_FLUSHING; 182 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork, 183 usecs_to_jiffies(100)); 184 } 185 186 return ret; 187 } 188 189 static void erdma_qp_safe_free(struct kref *ref) 190 { 191 struct erdma_qp *qp = container_of(ref, struct erdma_qp, ref); 192 193 complete(&qp->safe_free); 194 } 195 196 void erdma_qp_put(struct erdma_qp *qp) 197 { 198 WARN_ON(kref_read(&qp->ref) < 1); 199 kref_put(&qp->ref, erdma_qp_safe_free); 200 } 201 202 void erdma_qp_get(struct erdma_qp *qp) 203 { 204 kref_get(&qp->ref); 205 } 206 207 static int fill_inline_data(struct erdma_qp *qp, 208 const struct ib_send_wr *send_wr, u16 wqe_idx, 209 u32 sgl_offset, __le32 *length_field) 210 { 211 u32 remain_size, copy_size, data_off, bytes = 0; 212 char *data; 213 int i = 0; 214 215 wqe_idx += (sgl_offset >> SQEBB_SHIFT); 216 sgl_offset &= (SQEBB_SIZE - 1); 217 data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size, 218 SQEBB_SHIFT); 219 220 while (i < send_wr->num_sge) { 221 bytes += send_wr->sg_list[i].length; 222 if (bytes > (int)ERDMA_MAX_INLINE) 223 return -EINVAL; 224 225 remain_size = send_wr->sg_list[i].length; 226 data_off = 0; 227 228 while (1) { 229 copy_size = min(remain_size, SQEBB_SIZE - sgl_offset); 230 231 memcpy(data + sgl_offset, 232 (void *)(uintptr_t)send_wr->sg_list[i].addr + 233 data_off, 234 copy_size); 235 remain_size -= copy_size; 236 data_off += copy_size; 237 sgl_offset += copy_size; 238 wqe_idx += (sgl_offset >> SQEBB_SHIFT); 239 sgl_offset &= (SQEBB_SIZE - 1); 240 241 data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, 242 qp->attrs.sq_size, SQEBB_SHIFT); 243 if (!remain_size) 244 break; 245 } 246 247 i++; 248 } 249 *length_field = cpu_to_le32(bytes); 250 251 return bytes; 252 } 253 254 static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr, 255 u16 wqe_idx, u32 sgl_offset, __le32 *length_field) 256 { 257 int i = 0; 258 u32 bytes = 0; 259 char *sgl; 260 261 if (send_wr->num_sge > qp->dev->attrs.max_send_sge) 262 return -EINVAL; 263 264 if (sgl_offset & 0xF) 265 return -EINVAL; 266 267 while (i < send_wr->num_sge) { 268 wqe_idx += (sgl_offset >> SQEBB_SHIFT); 269 sgl_offset &= (SQEBB_SIZE - 1); 270 sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, 271 qp->attrs.sq_size, SQEBB_SHIFT); 272 273 bytes += send_wr->sg_list[i].length; 274 memcpy(sgl + sgl_offset, &send_wr->sg_list[i], 275 sizeof(struct ib_sge)); 276 277 sgl_offset += sizeof(struct ib_sge); 278 i++; 279 } 280 281 *length_field = cpu_to_le32(bytes); 282 return 0; 283 } 284 285 static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, 286 const struct ib_send_wr *send_wr) 287 { 288 u32 wqe_size, wqebb_cnt, hw_op, flags, sgl_offset; 289 u32 idx = *pi & (qp->attrs.sq_size - 1); 290 enum ib_wr_opcode op = send_wr->opcode; 291 struct erdma_atomic_sqe *atomic_sqe; 292 struct erdma_readreq_sqe *read_sqe; 293 struct erdma_reg_mr_sqe *regmr_sge; 294 struct erdma_write_sqe *write_sqe; 295 struct erdma_send_sqe *send_sqe; 296 struct ib_rdma_wr *rdma_wr; 297 struct erdma_sge *sge; 298 __le32 *length_field; 299 struct erdma_mr *mr; 300 u64 wqe_hdr, *entry; 301 u32 attrs; 302 int ret; 303 304 entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size, 305 SQEBB_SHIFT); 306 307 /* Clear the SQE header section. */ 308 *entry = 0; 309 310 qp->kern_qp.swr_tbl[idx] = send_wr->wr_id; 311 flags = send_wr->send_flags; 312 wqe_hdr = FIELD_PREP( 313 ERDMA_SQE_HDR_CE_MASK, 314 ((flags & IB_SEND_SIGNALED) || qp->kern_qp.sig_all) ? 1 : 0); 315 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SE_MASK, 316 flags & IB_SEND_SOLICITED ? 1 : 0); 317 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_FENCE_MASK, 318 flags & IB_SEND_FENCE ? 1 : 0); 319 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_INLINE_MASK, 320 flags & IB_SEND_INLINE ? 1 : 0); 321 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)); 322 323 switch (op) { 324 case IB_WR_RDMA_WRITE: 325 case IB_WR_RDMA_WRITE_WITH_IMM: 326 hw_op = ERDMA_OP_WRITE; 327 if (op == IB_WR_RDMA_WRITE_WITH_IMM) 328 hw_op = ERDMA_OP_WRITE_WITH_IMM; 329 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op); 330 rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr); 331 write_sqe = (struct erdma_write_sqe *)entry; 332 333 write_sqe->imm_data = send_wr->ex.imm_data; 334 write_sqe->sink_stag = cpu_to_le32(rdma_wr->rkey); 335 write_sqe->sink_to_h = 336 cpu_to_le32(upper_32_bits(rdma_wr->remote_addr)); 337 write_sqe->sink_to_l = 338 cpu_to_le32(lower_32_bits(rdma_wr->remote_addr)); 339 340 length_field = &write_sqe->length; 341 wqe_size = sizeof(struct erdma_write_sqe); 342 sgl_offset = wqe_size; 343 break; 344 case IB_WR_RDMA_READ: 345 case IB_WR_RDMA_READ_WITH_INV: 346 read_sqe = (struct erdma_readreq_sqe *)entry; 347 if (unlikely(send_wr->num_sge != 1)) 348 return -EINVAL; 349 hw_op = ERDMA_OP_READ; 350 if (op == IB_WR_RDMA_READ_WITH_INV) { 351 hw_op = ERDMA_OP_READ_WITH_INV; 352 read_sqe->invalid_stag = 353 cpu_to_le32(send_wr->ex.invalidate_rkey); 354 } 355 356 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op); 357 rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr); 358 read_sqe->length = cpu_to_le32(send_wr->sg_list[0].length); 359 read_sqe->sink_stag = cpu_to_le32(send_wr->sg_list[0].lkey); 360 read_sqe->sink_to_l = 361 cpu_to_le32(lower_32_bits(send_wr->sg_list[0].addr)); 362 read_sqe->sink_to_h = 363 cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr)); 364 365 sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, 366 qp->attrs.sq_size, SQEBB_SHIFT); 367 sge->addr = cpu_to_le64(rdma_wr->remote_addr); 368 sge->key = cpu_to_le32(rdma_wr->rkey); 369 sge->length = cpu_to_le32(send_wr->sg_list[0].length); 370 wqe_size = sizeof(struct erdma_readreq_sqe) + 371 send_wr->num_sge * sizeof(struct ib_sge); 372 373 goto out; 374 case IB_WR_SEND: 375 case IB_WR_SEND_WITH_IMM: 376 case IB_WR_SEND_WITH_INV: 377 send_sqe = (struct erdma_send_sqe *)entry; 378 hw_op = ERDMA_OP_SEND; 379 if (op == IB_WR_SEND_WITH_IMM) { 380 hw_op = ERDMA_OP_SEND_WITH_IMM; 381 send_sqe->imm_data = send_wr->ex.imm_data; 382 } else if (op == IB_WR_SEND_WITH_INV) { 383 hw_op = ERDMA_OP_SEND_WITH_INV; 384 send_sqe->invalid_stag = 385 cpu_to_le32(send_wr->ex.invalidate_rkey); 386 } 387 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op); 388 length_field = &send_sqe->length; 389 wqe_size = sizeof(struct erdma_send_sqe); 390 sgl_offset = wqe_size; 391 392 break; 393 case IB_WR_REG_MR: 394 wqe_hdr |= 395 FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, ERDMA_OP_REG_MR); 396 regmr_sge = (struct erdma_reg_mr_sqe *)entry; 397 mr = to_emr(reg_wr(send_wr)->mr); 398 399 mr->access = ERDMA_MR_ACC_LR | 400 to_erdma_access_flags(reg_wr(send_wr)->access); 401 regmr_sge->addr = cpu_to_le64(mr->ibmr.iova); 402 regmr_sge->length = cpu_to_le32(mr->ibmr.length); 403 regmr_sge->stag = cpu_to_le32(reg_wr(send_wr)->key); 404 attrs = FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) | 405 FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK, 406 mr->mem.mtt_nents); 407 408 if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) { 409 attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0); 410 /* Copy SGLs to SQE content to accelerate */ 411 memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1, 412 qp->attrs.sq_size, SQEBB_SHIFT), 413 mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents)); 414 wqe_size = sizeof(struct erdma_reg_mr_sqe) + 415 MTT_SIZE(mr->mem.mtt_nents); 416 } else { 417 attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 1); 418 wqe_size = sizeof(struct erdma_reg_mr_sqe); 419 } 420 421 regmr_sge->attrs = cpu_to_le32(attrs); 422 goto out; 423 case IB_WR_LOCAL_INV: 424 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, 425 ERDMA_OP_LOCAL_INV); 426 regmr_sge = (struct erdma_reg_mr_sqe *)entry; 427 regmr_sge->stag = cpu_to_le32(send_wr->ex.invalidate_rkey); 428 wqe_size = sizeof(struct erdma_reg_mr_sqe); 429 goto out; 430 case IB_WR_ATOMIC_CMP_AND_SWP: 431 case IB_WR_ATOMIC_FETCH_AND_ADD: 432 atomic_sqe = (struct erdma_atomic_sqe *)entry; 433 if (op == IB_WR_ATOMIC_CMP_AND_SWP) { 434 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, 435 ERDMA_OP_ATOMIC_CAS); 436 atomic_sqe->fetchadd_swap_data = 437 cpu_to_le64(atomic_wr(send_wr)->swap); 438 atomic_sqe->cmp_data = 439 cpu_to_le64(atomic_wr(send_wr)->compare_add); 440 } else { 441 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, 442 ERDMA_OP_ATOMIC_FAA); 443 atomic_sqe->fetchadd_swap_data = 444 cpu_to_le64(atomic_wr(send_wr)->compare_add); 445 } 446 447 sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, 448 qp->attrs.sq_size, SQEBB_SHIFT); 449 sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr); 450 sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey); 451 sge++; 452 453 sge->addr = cpu_to_le64(send_wr->sg_list[0].addr); 454 sge->key = cpu_to_le32(send_wr->sg_list[0].lkey); 455 sge->length = cpu_to_le32(send_wr->sg_list[0].length); 456 457 wqe_size = sizeof(*atomic_sqe); 458 goto out; 459 default: 460 return -EOPNOTSUPP; 461 } 462 463 if (flags & IB_SEND_INLINE) { 464 ret = fill_inline_data(qp, send_wr, idx, sgl_offset, 465 length_field); 466 if (ret < 0) 467 return -EINVAL; 468 wqe_size += ret; 469 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK, ret); 470 } else { 471 ret = fill_sgl(qp, send_wr, idx, sgl_offset, length_field); 472 if (ret) 473 return -EINVAL; 474 wqe_size += send_wr->num_sge * sizeof(struct ib_sge); 475 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK, 476 send_wr->num_sge); 477 } 478 479 out: 480 wqebb_cnt = SQEBB_COUNT(wqe_size); 481 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_CNT_MASK, wqebb_cnt - 1); 482 *pi += wqebb_cnt; 483 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, *pi); 484 485 *entry = wqe_hdr; 486 487 return 0; 488 } 489 490 static void kick_sq_db(struct erdma_qp *qp, u16 pi) 491 { 492 u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) | 493 FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi); 494 495 *(u64 *)qp->kern_qp.sq_dbrec = db_data; 496 writeq(db_data, qp->kern_qp.hw_sq_db); 497 } 498 499 int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr, 500 const struct ib_send_wr **bad_send_wr) 501 { 502 struct erdma_qp *qp = to_eqp(ibqp); 503 int ret = 0; 504 const struct ib_send_wr *wr = send_wr; 505 unsigned long flags; 506 u16 sq_pi; 507 508 if (!send_wr) 509 return -EINVAL; 510 511 spin_lock_irqsave(&qp->lock, flags); 512 sq_pi = qp->kern_qp.sq_pi; 513 514 while (wr) { 515 if ((u16)(sq_pi - qp->kern_qp.sq_ci) >= qp->attrs.sq_size) { 516 ret = -ENOMEM; 517 *bad_send_wr = send_wr; 518 break; 519 } 520 521 ret = erdma_push_one_sqe(qp, &sq_pi, wr); 522 if (ret) { 523 *bad_send_wr = wr; 524 break; 525 } 526 qp->kern_qp.sq_pi = sq_pi; 527 kick_sq_db(qp, sq_pi); 528 529 wr = wr->next; 530 } 531 spin_unlock_irqrestore(&qp->lock, flags); 532 533 if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING)) 534 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork, 535 usecs_to_jiffies(100)); 536 537 return ret; 538 } 539 540 static int erdma_post_recv_one(struct erdma_qp *qp, 541 const struct ib_recv_wr *recv_wr) 542 { 543 struct erdma_rqe *rqe = 544 get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi, 545 qp->attrs.rq_size, RQE_SHIFT); 546 547 rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1); 548 rqe->qpn = cpu_to_le32(QP_ID(qp)); 549 550 if (recv_wr->num_sge == 0) { 551 rqe->length = 0; 552 } else if (recv_wr->num_sge == 1) { 553 rqe->stag = cpu_to_le32(recv_wr->sg_list[0].lkey); 554 rqe->to = cpu_to_le64(recv_wr->sg_list[0].addr); 555 rqe->length = cpu_to_le32(recv_wr->sg_list[0].length); 556 } else { 557 return -EINVAL; 558 } 559 560 *(u64 *)qp->kern_qp.rq_dbrec = *(u64 *)rqe; 561 writeq(*(u64 *)rqe, qp->kern_qp.hw_rq_db); 562 563 qp->kern_qp.rwr_tbl[qp->kern_qp.rq_pi & (qp->attrs.rq_size - 1)] = 564 recv_wr->wr_id; 565 qp->kern_qp.rq_pi++; 566 567 return 0; 568 } 569 570 int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr, 571 const struct ib_recv_wr **bad_recv_wr) 572 { 573 const struct ib_recv_wr *wr = recv_wr; 574 struct erdma_qp *qp = to_eqp(ibqp); 575 unsigned long flags; 576 int ret; 577 578 spin_lock_irqsave(&qp->lock, flags); 579 580 while (wr) { 581 ret = erdma_post_recv_one(qp, wr); 582 if (ret) { 583 *bad_recv_wr = wr; 584 break; 585 } 586 wr = wr->next; 587 } 588 589 spin_unlock_irqrestore(&qp->lock, flags); 590 591 if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING)) 592 mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork, 593 usecs_to_jiffies(100)); 594 595 return ret; 596 } 597