1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Work Requests exploiting Infiniband API 6 * 7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively 8 * are submitted to either RC SQ or RC RQ respectively 9 * (reliably connected send/receive queue) 10 * and become work queue entries (WQEs). 11 * While an SQ WR/WQE is pending, we track it until transmission completion. 12 * Through a send or receive completion queue (CQ) respectively, 13 * we get completion queue entries (CQEs) [aka work completions (WCs)]. 14 * Since the CQ callback is called from IRQ context, we split work by using 15 * bottom halves implemented by tasklets. 16 * 17 * SMC uses this to exchange LLC (link layer control) 18 * and CDC (connection data control) messages. 19 * 20 * Copyright IBM Corp. 2016 21 * 22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> 23 */ 24 25 #include <linux/atomic.h> 26 #include <linux/hashtable.h> 27 #include <linux/wait.h> 28 #include <rdma/ib_verbs.h> 29 #include <asm/div64.h> 30 31 #include "smc.h" 32 #include "smc_wr.h" 33 34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ 35 36 #define SMC_WR_RX_HASH_BITS 4 37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); 38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); 39 40 struct smc_wr_tx_pend { /* control data for a pending send request */ 41 u64 wr_id; /* work request id sent */ 42 smc_wr_tx_handler handler; 43 enum ib_wc_status wc_status; /* CQE status */ 44 struct smc_link *link; 45 u32 idx; 46 struct smc_wr_tx_pend_priv priv; 47 u8 compl_requested; 48 }; 49 50 /******************************** send queue *********************************/ 51 52 /*------------------------------- completion --------------------------------*/ 53 54 /* returns true if at least one tx work request is pending on the given link */ 55 static inline bool smc_wr_is_tx_pend(struct smc_link *link) 56 { 57 return !bitmap_empty(link->wr_tx_mask, link->wr_tx_cnt); 58 } 59 60 /* wait till all pending tx work requests on the given link are completed */ 61 void smc_wr_tx_wait_no_pending_sends(struct smc_link *link) 62 { 63 wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link)); 64 } 65 66 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) 67 { 68 u32 i; 69 70 for (i = 0; i < link->wr_tx_cnt; i++) { 71 if (link->wr_tx_pends[i].wr_id == wr_id) 72 return i; 73 } 74 return link->wr_tx_cnt; 75 } 76 77 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) 78 { 79 struct smc_wr_tx_pend pnd_snd; 80 struct smc_link *link; 81 u32 pnd_snd_idx; 82 83 link = wc->qp->qp_context; 84 85 if (wc->opcode == IB_WC_REG_MR) { 86 if (wc->status) 87 link->wr_reg_state = FAILED; 88 else 89 link->wr_reg_state = CONFIRMED; 90 smc_wr_wakeup_reg_wait(link); 91 return; 92 } 93 94 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); 95 if (pnd_snd_idx == link->wr_tx_cnt) { 96 if (link->lgr->smc_version != SMC_V2 || 97 link->wr_tx_v2_pend->wr_id != wc->wr_id) 98 return; 99 link->wr_tx_v2_pend->wc_status = wc->status; 100 memcpy(&pnd_snd, link->wr_tx_v2_pend, sizeof(pnd_snd)); 101 /* clear the full struct smc_wr_tx_pend including .priv */ 102 memset(link->wr_tx_v2_pend, 0, 103 sizeof(*link->wr_tx_v2_pend)); 104 memset(link->lgr->wr_tx_buf_v2, 0, 105 sizeof(*link->lgr->wr_tx_buf_v2)); 106 } else { 107 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; 108 if (link->wr_tx_pends[pnd_snd_idx].compl_requested) 109 complete(&link->wr_tx_compl[pnd_snd_idx]); 110 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], 111 sizeof(pnd_snd)); 112 /* clear the full struct smc_wr_tx_pend including .priv */ 113 memset(&link->wr_tx_pends[pnd_snd_idx], 0, 114 sizeof(link->wr_tx_pends[pnd_snd_idx])); 115 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, 116 sizeof(link->wr_tx_bufs[pnd_snd_idx])); 117 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) 118 return; 119 } 120 121 if (wc->status) { 122 if (link->lgr->smc_version == SMC_V2) { 123 memset(link->wr_tx_v2_pend, 0, 124 sizeof(*link->wr_tx_v2_pend)); 125 memset(link->lgr->wr_tx_buf_v2, 0, 126 sizeof(*link->lgr->wr_tx_buf_v2)); 127 } 128 /* terminate link */ 129 smcr_link_down_cond_sched(link); 130 } 131 if (pnd_snd.handler) 132 pnd_snd.handler(&pnd_snd.priv, link, wc->status); 133 wake_up(&link->wr_tx_wait); 134 } 135 136 static void smc_wr_tx_tasklet_fn(struct tasklet_struct *t) 137 { 138 struct smc_ib_device *dev = from_tasklet(dev, t, send_tasklet); 139 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 140 int i, rc; 141 142 again: 143 do { 144 memset(&wc, 0, sizeof(wc)); 145 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); 146 for (i = 0; i < rc; i++) 147 smc_wr_tx_process_cqe(&wc[i]); 148 if (rc < SMC_WR_MAX_POLL_CQE) 149 /* If < SMC_WR_MAX_POLL_CQE, the CQ should have been 150 * drained, no need to poll again. --Guangguan Wang 151 */ 152 break; 153 } while (rc > 0); 154 155 /* IB_CQ_REPORT_MISSED_EVENTS make sure if ib_req_notify_cq() returns 156 * 0, it is safe to wait for the next event. 157 * Else we must poll the CQ again to make sure we won't miss any event 158 */ 159 if (ib_req_notify_cq(dev->roce_cq_send, 160 IB_CQ_NEXT_COMP | 161 IB_CQ_REPORT_MISSED_EVENTS)) 162 goto again; 163 } 164 165 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 166 { 167 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 168 169 tasklet_schedule(&dev->send_tasklet); 170 } 171 172 /*---------------------------- request submission ---------------------------*/ 173 174 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) 175 { 176 *idx = link->wr_tx_cnt; 177 if (!smc_link_sendable(link)) 178 return -ENOLINK; 179 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { 180 if (!test_and_set_bit(*idx, link->wr_tx_mask)) 181 return 0; 182 } 183 *idx = link->wr_tx_cnt; 184 return -EBUSY; 185 } 186 187 /** 188 * smc_wr_tx_get_free_slot() - returns buffer for message assembly, 189 * and sets info for pending transmit tracking 190 * @link: Pointer to smc_link used to later send the message. 191 * @handler: Send completion handler function pointer. 192 * @wr_buf: Out value returns pointer to message buffer. 193 * @wr_rdma_buf: Out value returns pointer to rdma work request. 194 * @wr_pend_priv: Out value returns pointer serving as handler context. 195 * 196 * Return: 0 on success, or -errno on error. 197 */ 198 int smc_wr_tx_get_free_slot(struct smc_link *link, 199 smc_wr_tx_handler handler, 200 struct smc_wr_buf **wr_buf, 201 struct smc_rdma_wr **wr_rdma_buf, 202 struct smc_wr_tx_pend_priv **wr_pend_priv) 203 { 204 struct smc_link_group *lgr = smc_get_lgr(link); 205 struct smc_wr_tx_pend *wr_pend; 206 u32 idx = link->wr_tx_cnt; 207 struct ib_send_wr *wr_ib; 208 u64 wr_id; 209 int rc; 210 211 *wr_buf = NULL; 212 *wr_pend_priv = NULL; 213 if (in_softirq() || lgr->terminating) { 214 rc = smc_wr_tx_get_free_slot_index(link, &idx); 215 if (rc) 216 return rc; 217 } else { 218 rc = wait_event_interruptible_timeout( 219 link->wr_tx_wait, 220 !smc_link_sendable(link) || 221 lgr->terminating || 222 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), 223 SMC_WR_TX_WAIT_FREE_SLOT_TIME); 224 if (!rc) { 225 /* timeout - terminate link */ 226 smcr_link_down_cond_sched(link); 227 return -EPIPE; 228 } 229 if (idx == link->wr_tx_cnt) 230 return -EPIPE; 231 } 232 wr_id = smc_wr_tx_get_next_wr_id(link); 233 wr_pend = &link->wr_tx_pends[idx]; 234 wr_pend->wr_id = wr_id; 235 wr_pend->handler = handler; 236 wr_pend->link = link; 237 wr_pend->idx = idx; 238 wr_ib = &link->wr_tx_ibs[idx]; 239 wr_ib->wr_id = wr_id; 240 *wr_buf = &link->wr_tx_bufs[idx]; 241 if (wr_rdma_buf) 242 *wr_rdma_buf = &link->wr_tx_rdmas[idx]; 243 *wr_pend_priv = &wr_pend->priv; 244 return 0; 245 } 246 247 int smc_wr_tx_get_v2_slot(struct smc_link *link, 248 smc_wr_tx_handler handler, 249 struct smc_wr_v2_buf **wr_buf, 250 struct smc_wr_tx_pend_priv **wr_pend_priv) 251 { 252 struct smc_wr_tx_pend *wr_pend; 253 struct ib_send_wr *wr_ib; 254 u64 wr_id; 255 256 if (link->wr_tx_v2_pend->idx == link->wr_tx_cnt) 257 return -EBUSY; 258 259 *wr_buf = NULL; 260 *wr_pend_priv = NULL; 261 wr_id = smc_wr_tx_get_next_wr_id(link); 262 wr_pend = link->wr_tx_v2_pend; 263 wr_pend->wr_id = wr_id; 264 wr_pend->handler = handler; 265 wr_pend->link = link; 266 wr_pend->idx = link->wr_tx_cnt; 267 wr_ib = link->wr_tx_v2_ib; 268 wr_ib->wr_id = wr_id; 269 *wr_buf = link->lgr->wr_tx_buf_v2; 270 *wr_pend_priv = &wr_pend->priv; 271 return 0; 272 } 273 274 int smc_wr_tx_put_slot(struct smc_link *link, 275 struct smc_wr_tx_pend_priv *wr_pend_priv) 276 { 277 struct smc_wr_tx_pend *pend; 278 279 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); 280 if (pend->idx < link->wr_tx_cnt) { 281 u32 idx = pend->idx; 282 283 /* clear the full struct smc_wr_tx_pend including .priv */ 284 memset(&link->wr_tx_pends[idx], 0, 285 sizeof(link->wr_tx_pends[idx])); 286 memset(&link->wr_tx_bufs[idx], 0, 287 sizeof(link->wr_tx_bufs[idx])); 288 test_and_clear_bit(idx, link->wr_tx_mask); 289 wake_up(&link->wr_tx_wait); 290 return 1; 291 } else if (link->lgr->smc_version == SMC_V2 && 292 pend->idx == link->wr_tx_cnt) { 293 /* Large v2 buffer */ 294 memset(&link->wr_tx_v2_pend, 0, 295 sizeof(link->wr_tx_v2_pend)); 296 memset(&link->lgr->wr_tx_buf_v2, 0, 297 sizeof(link->lgr->wr_tx_buf_v2)); 298 return 1; 299 } 300 301 return 0; 302 } 303 304 /* Send prepared WR slot via ib_post_send. 305 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 306 */ 307 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) 308 { 309 struct smc_wr_tx_pend *pend; 310 int rc; 311 312 ib_req_notify_cq(link->smcibdev->roce_cq_send, 313 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 314 pend = container_of(priv, struct smc_wr_tx_pend, priv); 315 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL); 316 if (rc) { 317 smc_wr_tx_put_slot(link, priv); 318 smcr_link_down_cond_sched(link); 319 } 320 return rc; 321 } 322 323 int smc_wr_tx_v2_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, 324 int len) 325 { 326 int rc; 327 328 link->wr_tx_v2_ib->sg_list[0].length = len; 329 ib_req_notify_cq(link->smcibdev->roce_cq_send, 330 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 331 rc = ib_post_send(link->roce_qp, link->wr_tx_v2_ib, NULL); 332 if (rc) { 333 smc_wr_tx_put_slot(link, priv); 334 smcr_link_down_cond_sched(link); 335 } 336 return rc; 337 } 338 339 /* Send prepared WR slot via ib_post_send and wait for send completion 340 * notification. 341 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 342 */ 343 int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, 344 unsigned long timeout) 345 { 346 struct smc_wr_tx_pend *pend; 347 u32 pnd_idx; 348 int rc; 349 350 pend = container_of(priv, struct smc_wr_tx_pend, priv); 351 pend->compl_requested = 1; 352 pnd_idx = pend->idx; 353 init_completion(&link->wr_tx_compl[pnd_idx]); 354 355 rc = smc_wr_tx_send(link, priv); 356 if (rc) 357 return rc; 358 /* wait for completion by smc_wr_tx_process_cqe() */ 359 rc = wait_for_completion_interruptible_timeout( 360 &link->wr_tx_compl[pnd_idx], timeout); 361 if (rc <= 0) 362 rc = -ENODATA; 363 if (rc > 0) 364 rc = 0; 365 return rc; 366 } 367 368 /* Register a memory region and wait for result. */ 369 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) 370 { 371 int rc; 372 373 ib_req_notify_cq(link->smcibdev->roce_cq_send, 374 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 375 link->wr_reg_state = POSTED; 376 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; 377 link->wr_reg.mr = mr; 378 link->wr_reg.key = mr->rkey; 379 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL); 380 if (rc) 381 return rc; 382 383 atomic_inc(&link->wr_reg_refcnt); 384 rc = wait_event_interruptible_timeout(link->wr_reg_wait, 385 (link->wr_reg_state != POSTED), 386 SMC_WR_REG_MR_WAIT_TIME); 387 if (atomic_dec_and_test(&link->wr_reg_refcnt)) 388 wake_up_all(&link->wr_reg_wait); 389 if (!rc) { 390 /* timeout - terminate link */ 391 smcr_link_down_cond_sched(link); 392 return -EPIPE; 393 } 394 if (rc == -ERESTARTSYS) 395 return -EINTR; 396 switch (link->wr_reg_state) { 397 case CONFIRMED: 398 rc = 0; 399 break; 400 case FAILED: 401 rc = -EIO; 402 break; 403 case POSTED: 404 rc = -EPIPE; 405 break; 406 } 407 return rc; 408 } 409 410 /****************************** receive queue ********************************/ 411 412 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) 413 { 414 struct smc_wr_rx_handler *h_iter; 415 int rc = 0; 416 417 spin_lock(&smc_wr_rx_hash_lock); 418 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { 419 if (h_iter->type == handler->type) { 420 rc = -EEXIST; 421 goto out_unlock; 422 } 423 } 424 hash_add(smc_wr_rx_hash, &handler->list, handler->type); 425 out_unlock: 426 spin_unlock(&smc_wr_rx_hash_lock); 427 return rc; 428 } 429 430 /* Demultiplex a received work request based on the message type to its handler. 431 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, 432 * and not being modified any more afterwards so we don't need to lock it. 433 */ 434 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) 435 { 436 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 437 struct smc_wr_rx_handler *handler; 438 struct smc_wr_rx_hdr *wr_rx; 439 u64 temp_wr_id; 440 u32 index; 441 442 if (wc->byte_len < sizeof(*wr_rx)) 443 return; /* short message */ 444 temp_wr_id = wc->wr_id; 445 index = do_div(temp_wr_id, link->wr_rx_cnt); 446 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; 447 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { 448 if (handler->type == wr_rx->type) 449 handler->handler(wc, wr_rx); 450 } 451 } 452 453 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) 454 { 455 struct smc_link *link; 456 int i; 457 458 for (i = 0; i < num; i++) { 459 link = wc[i].qp->qp_context; 460 if (wc[i].status == IB_WC_SUCCESS) { 461 link->wr_rx_tstamp = jiffies; 462 smc_wr_rx_demultiplex(&wc[i]); 463 smc_wr_rx_post(link); /* refill WR RX */ 464 } else { 465 /* handle status errors */ 466 switch (wc[i].status) { 467 case IB_WC_RETRY_EXC_ERR: 468 case IB_WC_RNR_RETRY_EXC_ERR: 469 case IB_WC_WR_FLUSH_ERR: 470 smcr_link_down_cond_sched(link); 471 break; 472 default: 473 smc_wr_rx_post(link); /* refill WR RX */ 474 break; 475 } 476 } 477 } 478 } 479 480 static void smc_wr_rx_tasklet_fn(struct tasklet_struct *t) 481 { 482 struct smc_ib_device *dev = from_tasklet(dev, t, recv_tasklet); 483 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 484 int rc; 485 486 again: 487 do { 488 memset(&wc, 0, sizeof(wc)); 489 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); 490 if (rc > 0) 491 smc_wr_rx_process_cqes(&wc[0], rc); 492 if (rc < SMC_WR_MAX_POLL_CQE) 493 /* If < SMC_WR_MAX_POLL_CQE, the CQ should have been 494 * drained, no need to poll again. --Guangguan Wang 495 */ 496 break; 497 } while (rc > 0); 498 499 /* IB_CQ_REPORT_MISSED_EVENTS make sure if ib_req_notify_cq() returns 500 * 0, it is safe to wait for the next event. 501 * Else we must poll the CQ again to make sure we won't miss any event 502 */ 503 if (ib_req_notify_cq(dev->roce_cq_recv, 504 IB_CQ_SOLICITED_MASK | 505 IB_CQ_REPORT_MISSED_EVENTS)) 506 goto again; 507 } 508 509 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 510 { 511 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 512 513 tasklet_schedule(&dev->recv_tasklet); 514 } 515 516 int smc_wr_rx_post_init(struct smc_link *link) 517 { 518 u32 i; 519 int rc = 0; 520 521 for (i = 0; i < link->wr_rx_cnt; i++) 522 rc = smc_wr_rx_post(link); 523 return rc; 524 } 525 526 /***************************** init, exit, misc ******************************/ 527 528 void smc_wr_remember_qp_attr(struct smc_link *lnk) 529 { 530 struct ib_qp_attr *attr = &lnk->qp_attr; 531 struct ib_qp_init_attr init_attr; 532 533 memset(attr, 0, sizeof(*attr)); 534 memset(&init_attr, 0, sizeof(init_attr)); 535 ib_query_qp(lnk->roce_qp, attr, 536 IB_QP_STATE | 537 IB_QP_CUR_STATE | 538 IB_QP_PKEY_INDEX | 539 IB_QP_PORT | 540 IB_QP_QKEY | 541 IB_QP_AV | 542 IB_QP_PATH_MTU | 543 IB_QP_TIMEOUT | 544 IB_QP_RETRY_CNT | 545 IB_QP_RNR_RETRY | 546 IB_QP_RQ_PSN | 547 IB_QP_ALT_PATH | 548 IB_QP_MIN_RNR_TIMER | 549 IB_QP_SQ_PSN | 550 IB_QP_PATH_MIG_STATE | 551 IB_QP_CAP | 552 IB_QP_DEST_QPN, 553 &init_attr); 554 555 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, 556 lnk->qp_attr.cap.max_send_wr); 557 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, 558 lnk->qp_attr.cap.max_recv_wr); 559 } 560 561 static void smc_wr_init_sge(struct smc_link *lnk) 562 { 563 int sges_per_buf = (lnk->lgr->smc_version == SMC_V2) ? 2 : 1; 564 u32 i; 565 566 for (i = 0; i < lnk->wr_tx_cnt; i++) { 567 lnk->wr_tx_sges[i].addr = 568 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; 569 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; 570 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 571 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey = 572 lnk->roce_pd->local_dma_lkey; 573 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey = 574 lnk->roce_pd->local_dma_lkey; 575 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey = 576 lnk->roce_pd->local_dma_lkey; 577 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey = 578 lnk->roce_pd->local_dma_lkey; 579 lnk->wr_tx_ibs[i].next = NULL; 580 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; 581 lnk->wr_tx_ibs[i].num_sge = 1; 582 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; 583 lnk->wr_tx_ibs[i].send_flags = 584 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 585 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE; 586 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE; 587 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list = 588 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge; 589 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list = 590 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge; 591 } 592 593 if (lnk->lgr->smc_version == SMC_V2) { 594 lnk->wr_tx_v2_sge->addr = lnk->wr_tx_v2_dma_addr; 595 lnk->wr_tx_v2_sge->length = SMC_WR_BUF_V2_SIZE; 596 lnk->wr_tx_v2_sge->lkey = lnk->roce_pd->local_dma_lkey; 597 598 lnk->wr_tx_v2_ib->next = NULL; 599 lnk->wr_tx_v2_ib->sg_list = lnk->wr_tx_v2_sge; 600 lnk->wr_tx_v2_ib->num_sge = 1; 601 lnk->wr_tx_v2_ib->opcode = IB_WR_SEND; 602 lnk->wr_tx_v2_ib->send_flags = 603 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 604 } 605 606 /* With SMC-Rv2 there can be messages larger than SMC_WR_TX_SIZE. 607 * Each ib_recv_wr gets 2 sges, the second one is a spillover buffer 608 * and the same buffer for all sges. When a larger message arrived then 609 * the content of the first small sge is copied to the beginning of 610 * the larger spillover buffer, allowing easy data mapping. 611 */ 612 for (i = 0; i < lnk->wr_rx_cnt; i++) { 613 int x = i * sges_per_buf; 614 615 lnk->wr_rx_sges[x].addr = 616 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; 617 lnk->wr_rx_sges[x].length = SMC_WR_TX_SIZE; 618 lnk->wr_rx_sges[x].lkey = lnk->roce_pd->local_dma_lkey; 619 if (lnk->lgr->smc_version == SMC_V2) { 620 lnk->wr_rx_sges[x + 1].addr = 621 lnk->wr_rx_v2_dma_addr + SMC_WR_TX_SIZE; 622 lnk->wr_rx_sges[x + 1].length = 623 SMC_WR_BUF_V2_SIZE - SMC_WR_TX_SIZE; 624 lnk->wr_rx_sges[x + 1].lkey = 625 lnk->roce_pd->local_dma_lkey; 626 } 627 lnk->wr_rx_ibs[i].next = NULL; 628 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[x]; 629 lnk->wr_rx_ibs[i].num_sge = sges_per_buf; 630 } 631 lnk->wr_reg.wr.next = NULL; 632 lnk->wr_reg.wr.num_sge = 0; 633 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; 634 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; 635 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; 636 } 637 638 void smc_wr_free_link(struct smc_link *lnk) 639 { 640 struct ib_device *ibdev; 641 642 if (!lnk->smcibdev) 643 return; 644 ibdev = lnk->smcibdev->ibdev; 645 646 smc_wr_wakeup_reg_wait(lnk); 647 smc_wr_wakeup_tx_wait(lnk); 648 649 smc_wr_tx_wait_no_pending_sends(lnk); 650 wait_event(lnk->wr_reg_wait, (!atomic_read(&lnk->wr_reg_refcnt))); 651 wait_event(lnk->wr_tx_wait, (!atomic_read(&lnk->wr_tx_refcnt))); 652 653 if (lnk->wr_rx_dma_addr) { 654 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 655 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 656 DMA_FROM_DEVICE); 657 lnk->wr_rx_dma_addr = 0; 658 } 659 if (lnk->wr_rx_v2_dma_addr) { 660 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, 661 SMC_WR_BUF_V2_SIZE, 662 DMA_FROM_DEVICE); 663 lnk->wr_rx_v2_dma_addr = 0; 664 } 665 if (lnk->wr_tx_dma_addr) { 666 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, 667 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 668 DMA_TO_DEVICE); 669 lnk->wr_tx_dma_addr = 0; 670 } 671 if (lnk->wr_tx_v2_dma_addr) { 672 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, 673 SMC_WR_BUF_V2_SIZE, 674 DMA_TO_DEVICE); 675 lnk->wr_tx_v2_dma_addr = 0; 676 } 677 } 678 679 void smc_wr_free_lgr_mem(struct smc_link_group *lgr) 680 { 681 if (lgr->smc_version < SMC_V2) 682 return; 683 684 kfree(lgr->wr_rx_buf_v2); 685 lgr->wr_rx_buf_v2 = NULL; 686 kfree(lgr->wr_tx_buf_v2); 687 lgr->wr_tx_buf_v2 = NULL; 688 } 689 690 void smc_wr_free_link_mem(struct smc_link *lnk) 691 { 692 kfree(lnk->wr_tx_v2_ib); 693 lnk->wr_tx_v2_ib = NULL; 694 kfree(lnk->wr_tx_v2_sge); 695 lnk->wr_tx_v2_sge = NULL; 696 kfree(lnk->wr_tx_v2_pend); 697 lnk->wr_tx_v2_pend = NULL; 698 kfree(lnk->wr_tx_compl); 699 lnk->wr_tx_compl = NULL; 700 kfree(lnk->wr_tx_pends); 701 lnk->wr_tx_pends = NULL; 702 bitmap_free(lnk->wr_tx_mask); 703 lnk->wr_tx_mask = NULL; 704 kfree(lnk->wr_tx_sges); 705 lnk->wr_tx_sges = NULL; 706 kfree(lnk->wr_tx_rdma_sges); 707 lnk->wr_tx_rdma_sges = NULL; 708 kfree(lnk->wr_rx_sges); 709 lnk->wr_rx_sges = NULL; 710 kfree(lnk->wr_tx_rdmas); 711 lnk->wr_tx_rdmas = NULL; 712 kfree(lnk->wr_rx_ibs); 713 lnk->wr_rx_ibs = NULL; 714 kfree(lnk->wr_tx_ibs); 715 lnk->wr_tx_ibs = NULL; 716 kfree(lnk->wr_tx_bufs); 717 lnk->wr_tx_bufs = NULL; 718 kfree(lnk->wr_rx_bufs); 719 lnk->wr_rx_bufs = NULL; 720 } 721 722 int smc_wr_alloc_lgr_mem(struct smc_link_group *lgr) 723 { 724 if (lgr->smc_version < SMC_V2) 725 return 0; 726 727 lgr->wr_rx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL); 728 if (!lgr->wr_rx_buf_v2) 729 return -ENOMEM; 730 lgr->wr_tx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL); 731 if (!lgr->wr_tx_buf_v2) { 732 kfree(lgr->wr_rx_buf_v2); 733 return -ENOMEM; 734 } 735 return 0; 736 } 737 738 int smc_wr_alloc_link_mem(struct smc_link *link) 739 { 740 int sges_per_buf = link->lgr->smc_version == SMC_V2 ? 2 : 1; 741 742 /* allocate link related memory */ 743 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); 744 if (!link->wr_tx_bufs) 745 goto no_mem; 746 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, 747 GFP_KERNEL); 748 if (!link->wr_rx_bufs) 749 goto no_mem_wr_tx_bufs; 750 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), 751 GFP_KERNEL); 752 if (!link->wr_tx_ibs) 753 goto no_mem_wr_rx_bufs; 754 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, 755 sizeof(link->wr_rx_ibs[0]), 756 GFP_KERNEL); 757 if (!link->wr_rx_ibs) 758 goto no_mem_wr_tx_ibs; 759 link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT, 760 sizeof(link->wr_tx_rdmas[0]), 761 GFP_KERNEL); 762 if (!link->wr_tx_rdmas) 763 goto no_mem_wr_rx_ibs; 764 link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT, 765 sizeof(link->wr_tx_rdma_sges[0]), 766 GFP_KERNEL); 767 if (!link->wr_tx_rdma_sges) 768 goto no_mem_wr_tx_rdmas; 769 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), 770 GFP_KERNEL); 771 if (!link->wr_tx_sges) 772 goto no_mem_wr_tx_rdma_sges; 773 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, 774 sizeof(link->wr_rx_sges[0]) * sges_per_buf, 775 GFP_KERNEL); 776 if (!link->wr_rx_sges) 777 goto no_mem_wr_tx_sges; 778 link->wr_tx_mask = bitmap_zalloc(SMC_WR_BUF_CNT, GFP_KERNEL); 779 if (!link->wr_tx_mask) 780 goto no_mem_wr_rx_sges; 781 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, 782 sizeof(link->wr_tx_pends[0]), 783 GFP_KERNEL); 784 if (!link->wr_tx_pends) 785 goto no_mem_wr_tx_mask; 786 link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT, 787 sizeof(link->wr_tx_compl[0]), 788 GFP_KERNEL); 789 if (!link->wr_tx_compl) 790 goto no_mem_wr_tx_pends; 791 792 if (link->lgr->smc_version == SMC_V2) { 793 link->wr_tx_v2_ib = kzalloc(sizeof(*link->wr_tx_v2_ib), 794 GFP_KERNEL); 795 if (!link->wr_tx_v2_ib) 796 goto no_mem_tx_compl; 797 link->wr_tx_v2_sge = kzalloc(sizeof(*link->wr_tx_v2_sge), 798 GFP_KERNEL); 799 if (!link->wr_tx_v2_sge) 800 goto no_mem_v2_ib; 801 link->wr_tx_v2_pend = kzalloc(sizeof(*link->wr_tx_v2_pend), 802 GFP_KERNEL); 803 if (!link->wr_tx_v2_pend) 804 goto no_mem_v2_sge; 805 } 806 return 0; 807 808 no_mem_v2_sge: 809 kfree(link->wr_tx_v2_sge); 810 no_mem_v2_ib: 811 kfree(link->wr_tx_v2_ib); 812 no_mem_tx_compl: 813 kfree(link->wr_tx_compl); 814 no_mem_wr_tx_pends: 815 kfree(link->wr_tx_pends); 816 no_mem_wr_tx_mask: 817 kfree(link->wr_tx_mask); 818 no_mem_wr_rx_sges: 819 kfree(link->wr_rx_sges); 820 no_mem_wr_tx_sges: 821 kfree(link->wr_tx_sges); 822 no_mem_wr_tx_rdma_sges: 823 kfree(link->wr_tx_rdma_sges); 824 no_mem_wr_tx_rdmas: 825 kfree(link->wr_tx_rdmas); 826 no_mem_wr_rx_ibs: 827 kfree(link->wr_rx_ibs); 828 no_mem_wr_tx_ibs: 829 kfree(link->wr_tx_ibs); 830 no_mem_wr_rx_bufs: 831 kfree(link->wr_rx_bufs); 832 no_mem_wr_tx_bufs: 833 kfree(link->wr_tx_bufs); 834 no_mem: 835 return -ENOMEM; 836 } 837 838 void smc_wr_remove_dev(struct smc_ib_device *smcibdev) 839 { 840 tasklet_kill(&smcibdev->recv_tasklet); 841 tasklet_kill(&smcibdev->send_tasklet); 842 } 843 844 void smc_wr_add_dev(struct smc_ib_device *smcibdev) 845 { 846 tasklet_setup(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn); 847 tasklet_setup(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn); 848 } 849 850 int smc_wr_create_link(struct smc_link *lnk) 851 { 852 struct ib_device *ibdev = lnk->smcibdev->ibdev; 853 int rc = 0; 854 855 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); 856 lnk->wr_rx_id = 0; 857 lnk->wr_rx_dma_addr = ib_dma_map_single( 858 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 859 DMA_FROM_DEVICE); 860 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { 861 lnk->wr_rx_dma_addr = 0; 862 rc = -EIO; 863 goto out; 864 } 865 if (lnk->lgr->smc_version == SMC_V2) { 866 lnk->wr_rx_v2_dma_addr = ib_dma_map_single(ibdev, 867 lnk->lgr->wr_rx_buf_v2, SMC_WR_BUF_V2_SIZE, 868 DMA_FROM_DEVICE); 869 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_v2_dma_addr)) { 870 lnk->wr_rx_v2_dma_addr = 0; 871 rc = -EIO; 872 goto dma_unmap; 873 } 874 lnk->wr_tx_v2_dma_addr = ib_dma_map_single(ibdev, 875 lnk->lgr->wr_tx_buf_v2, SMC_WR_BUF_V2_SIZE, 876 DMA_TO_DEVICE); 877 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_v2_dma_addr)) { 878 lnk->wr_tx_v2_dma_addr = 0; 879 rc = -EIO; 880 goto dma_unmap; 881 } 882 } 883 lnk->wr_tx_dma_addr = ib_dma_map_single( 884 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 885 DMA_TO_DEVICE); 886 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { 887 rc = -EIO; 888 goto dma_unmap; 889 } 890 smc_wr_init_sge(lnk); 891 bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT); 892 init_waitqueue_head(&lnk->wr_tx_wait); 893 atomic_set(&lnk->wr_tx_refcnt, 0); 894 init_waitqueue_head(&lnk->wr_reg_wait); 895 atomic_set(&lnk->wr_reg_refcnt, 0); 896 return rc; 897 898 dma_unmap: 899 if (lnk->wr_rx_v2_dma_addr) { 900 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, 901 SMC_WR_BUF_V2_SIZE, 902 DMA_FROM_DEVICE); 903 lnk->wr_rx_v2_dma_addr = 0; 904 } 905 if (lnk->wr_tx_v2_dma_addr) { 906 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, 907 SMC_WR_BUF_V2_SIZE, 908 DMA_TO_DEVICE); 909 lnk->wr_tx_v2_dma_addr = 0; 910 } 911 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 912 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 913 DMA_FROM_DEVICE); 914 lnk->wr_rx_dma_addr = 0; 915 out: 916 return rc; 917 } 918