1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Link Layer Control (LLC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 10 * Ursula Braun <ubraun@linux.vnet.ibm.com> 11 */ 12 13 #include <net/tcp.h> 14 #include <rdma/ib_verbs.h> 15 16 #include "smc.h" 17 #include "smc_core.h" 18 #include "smc_clc.h" 19 #include "smc_llc.h" 20 #include "smc_pnet.h" 21 22 #define SMC_LLC_DATA_LEN 40 23 24 struct smc_llc_hdr { 25 struct smc_wr_rx_hdr common; 26 u8 length; /* 44 */ 27 #if defined(__BIG_ENDIAN_BITFIELD) 28 u8 reserved:4, 29 add_link_rej_rsn:4; 30 #elif defined(__LITTLE_ENDIAN_BITFIELD) 31 u8 add_link_rej_rsn:4, 32 reserved:4; 33 #endif 34 u8 flags; 35 }; 36 37 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 38 39 struct smc_llc_msg_confirm_link { /* type 0x01 */ 40 struct smc_llc_hdr hd; 41 u8 sender_mac[ETH_ALEN]; 42 u8 sender_gid[SMC_GID_SIZE]; 43 u8 sender_qp_num[3]; 44 u8 link_num; 45 u8 link_uid[SMC_LGR_ID_SIZE]; 46 u8 max_links; 47 u8 reserved[9]; 48 }; 49 50 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 51 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 52 53 #define SMC_LLC_ADD_LNK_MAX_LINKS 2 54 55 struct smc_llc_msg_add_link { /* type 0x02 */ 56 struct smc_llc_hdr hd; 57 u8 sender_mac[ETH_ALEN]; 58 u8 reserved2[2]; 59 u8 sender_gid[SMC_GID_SIZE]; 60 u8 sender_qp_num[3]; 61 u8 link_num; 62 #if defined(__BIG_ENDIAN_BITFIELD) 63 u8 reserved3 : 4, 64 qp_mtu : 4; 65 #elif defined(__LITTLE_ENDIAN_BITFIELD) 66 u8 qp_mtu : 4, 67 reserved3 : 4; 68 #endif 69 u8 initial_psn[3]; 70 u8 reserved[8]; 71 }; 72 73 struct smc_llc_msg_add_link_cont_rt { 74 __be32 rmb_key; 75 __be32 rmb_key_new; 76 __be64 rmb_vaddr_new; 77 }; 78 79 #define SMC_LLC_RKEYS_PER_CONT_MSG 2 80 81 struct smc_llc_msg_add_link_cont { /* type 0x03 */ 82 struct smc_llc_hdr hd; 83 u8 link_num; 84 u8 num_rkeys; 85 u8 reserved2[2]; 86 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 87 u8 reserved[4]; 88 } __packed; /* format defined in RFC7609 */ 89 90 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 91 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 92 93 struct smc_llc_msg_del_link { /* type 0x04 */ 94 struct smc_llc_hdr hd; 95 u8 link_num; 96 __be32 reason; 97 u8 reserved[35]; 98 } __packed; /* format defined in RFC7609 */ 99 100 struct smc_llc_msg_test_link { /* type 0x07 */ 101 struct smc_llc_hdr hd; 102 u8 user_data[16]; 103 u8 reserved[24]; 104 }; 105 106 struct smc_rmb_rtoken { 107 union { 108 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 109 /* is actually the num of rtokens, first */ 110 /* rtoken is always for the current link */ 111 u8 link_id; /* link id of the rtoken */ 112 }; 113 __be32 rmb_key; 114 __be64 rmb_vaddr; 115 } __packed; /* format defined in RFC7609 */ 116 117 #define SMC_LLC_RKEYS_PER_MSG 3 118 119 struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 120 struct smc_llc_hdr hd; 121 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 122 u8 reserved; 123 }; 124 125 #define SMC_LLC_DEL_RKEY_MAX 8 126 #define SMC_LLC_FLAG_RKEY_RETRY 0x10 127 #define SMC_LLC_FLAG_RKEY_NEG 0x20 128 129 struct smc_llc_msg_delete_rkey { /* type 0x09 */ 130 struct smc_llc_hdr hd; 131 u8 num_rkeys; 132 u8 err_mask; 133 u8 reserved[2]; 134 __be32 rkey[8]; 135 u8 reserved2[4]; 136 }; 137 138 union smc_llc_msg { 139 struct smc_llc_msg_confirm_link confirm_link; 140 struct smc_llc_msg_add_link add_link; 141 struct smc_llc_msg_add_link_cont add_link_cont; 142 struct smc_llc_msg_del_link delete_link; 143 144 struct smc_llc_msg_confirm_rkey confirm_rkey; 145 struct smc_llc_msg_delete_rkey delete_rkey; 146 147 struct smc_llc_msg_test_link test_link; 148 struct { 149 struct smc_llc_hdr hdr; 150 u8 data[SMC_LLC_DATA_LEN]; 151 } raw; 152 }; 153 154 #define SMC_LLC_FLAG_RESP 0x80 155 156 struct smc_llc_qentry { 157 struct list_head list; 158 struct smc_link *link; 159 union smc_llc_msg msg; 160 }; 161 162 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); 163 164 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 165 { 166 struct smc_llc_qentry *qentry = flow->qentry; 167 168 flow->qentry = NULL; 169 return qentry; 170 } 171 172 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 173 { 174 struct smc_llc_qentry *qentry; 175 176 if (flow->qentry) { 177 qentry = flow->qentry; 178 flow->qentry = NULL; 179 kfree(qentry); 180 } 181 } 182 183 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 184 struct smc_llc_qentry *qentry) 185 { 186 flow->qentry = qentry; 187 } 188 189 /* try to start a new llc flow, initiated by an incoming llc msg */ 190 static bool smc_llc_flow_start(struct smc_llc_flow *flow, 191 struct smc_llc_qentry *qentry) 192 { 193 struct smc_link_group *lgr = qentry->link->lgr; 194 195 spin_lock_bh(&lgr->llc_flow_lock); 196 if (flow->type) { 197 /* a flow is already active */ 198 if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK || 199 qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) && 200 !lgr->delayed_event) { 201 lgr->delayed_event = qentry; 202 } else { 203 /* forget this llc request */ 204 kfree(qentry); 205 } 206 spin_unlock_bh(&lgr->llc_flow_lock); 207 return false; 208 } 209 switch (qentry->msg.raw.hdr.common.type) { 210 case SMC_LLC_ADD_LINK: 211 flow->type = SMC_LLC_FLOW_ADD_LINK; 212 break; 213 case SMC_LLC_DELETE_LINK: 214 flow->type = SMC_LLC_FLOW_DEL_LINK; 215 break; 216 case SMC_LLC_CONFIRM_RKEY: 217 case SMC_LLC_DELETE_RKEY: 218 flow->type = SMC_LLC_FLOW_RKEY; 219 break; 220 default: 221 flow->type = SMC_LLC_FLOW_NONE; 222 } 223 if (qentry == lgr->delayed_event) 224 lgr->delayed_event = NULL; 225 spin_unlock_bh(&lgr->llc_flow_lock); 226 smc_llc_flow_qentry_set(flow, qentry); 227 return true; 228 } 229 230 /* start a new local llc flow, wait till current flow finished */ 231 int smc_llc_flow_initiate(struct smc_link_group *lgr, 232 enum smc_llc_flowtype type) 233 { 234 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 235 int rc; 236 237 /* all flows except confirm_rkey and delete_rkey are exclusive, 238 * confirm/delete rkey flows can run concurrently (local and remote) 239 */ 240 if (type == SMC_LLC_FLOW_RKEY) 241 allowed_remote = SMC_LLC_FLOW_RKEY; 242 again: 243 if (list_empty(&lgr->list)) 244 return -ENODEV; 245 spin_lock_bh(&lgr->llc_flow_lock); 246 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 247 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 248 lgr->llc_flow_rmt.type == allowed_remote)) { 249 lgr->llc_flow_lcl.type = type; 250 spin_unlock_bh(&lgr->llc_flow_lock); 251 return 0; 252 } 253 spin_unlock_bh(&lgr->llc_flow_lock); 254 rc = wait_event_interruptible_timeout(lgr->llc_waiter, 255 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 256 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 257 lgr->llc_flow_rmt.type == allowed_remote)), 258 SMC_LLC_WAIT_TIME); 259 if (!rc) 260 return -ETIMEDOUT; 261 goto again; 262 } 263 264 /* finish the current llc flow */ 265 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 266 { 267 spin_lock_bh(&lgr->llc_flow_lock); 268 memset(flow, 0, sizeof(*flow)); 269 flow->type = SMC_LLC_FLOW_NONE; 270 spin_unlock_bh(&lgr->llc_flow_lock); 271 if (!list_empty(&lgr->list) && lgr->delayed_event && 272 flow == &lgr->llc_flow_lcl) 273 schedule_work(&lgr->llc_event_work); 274 else 275 wake_up_interruptible(&lgr->llc_waiter); 276 } 277 278 /* lnk is optional and used for early wakeup when link goes down, useful in 279 * cases where we wait for a response on the link after we sent a request 280 */ 281 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 282 struct smc_link *lnk, 283 int time_out, u8 exp_msg) 284 { 285 struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 286 287 wait_event_interruptible_timeout(lgr->llc_waiter, 288 (flow->qentry || 289 (lnk && !smc_link_usable(lnk)) || 290 list_empty(&lgr->list)), 291 time_out); 292 if (!flow->qentry || 293 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 294 smc_llc_flow_qentry_del(flow); 295 goto out; 296 } 297 if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) { 298 if (exp_msg == SMC_LLC_ADD_LINK && 299 flow->qentry->msg.raw.hdr.common.type == 300 SMC_LLC_DELETE_LINK) { 301 /* flow_start will delay the unexpected msg */ 302 smc_llc_flow_start(&lgr->llc_flow_lcl, 303 smc_llc_flow_qentry_clr(flow)); 304 return NULL; 305 } 306 smc_llc_flow_qentry_del(flow); 307 } 308 out: 309 return flow->qentry; 310 } 311 312 /********************************** send *************************************/ 313 314 struct smc_llc_tx_pend { 315 }; 316 317 /* handler for send/transmission completion of an LLC msg */ 318 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 319 struct smc_link *link, 320 enum ib_wc_status wc_status) 321 { 322 /* future work: handle wc_status error for recovery and failover */ 323 } 324 325 /** 326 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 327 * @link: Pointer to SMC link used for sending LLC control message. 328 * @wr_buf: Out variable returning pointer to work request payload buffer. 329 * @pend: Out variable returning pointer to private pending WR tracking. 330 * It's the context the transmit complete handler will get. 331 * 332 * Reserves and pre-fills an entry for a pending work request send/tx. 333 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 334 * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 335 * 336 * Return: 0 on success, otherwise an error value. 337 */ 338 static int smc_llc_add_pending_send(struct smc_link *link, 339 struct smc_wr_buf **wr_buf, 340 struct smc_wr_tx_pend_priv **pend) 341 { 342 int rc; 343 344 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 345 pend); 346 if (rc < 0) 347 return rc; 348 BUILD_BUG_ON_MSG( 349 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 350 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 351 BUILD_BUG_ON_MSG( 352 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 353 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 354 BUILD_BUG_ON_MSG( 355 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 356 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 357 return 0; 358 } 359 360 /* high-level API to send LLC confirm link */ 361 int smc_llc_send_confirm_link(struct smc_link *link, 362 enum smc_llc_reqresp reqresp) 363 { 364 struct smc_llc_msg_confirm_link *confllc; 365 struct smc_wr_tx_pend_priv *pend; 366 struct smc_wr_buf *wr_buf; 367 int rc; 368 369 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 370 if (rc) 371 return rc; 372 confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 373 memset(confllc, 0, sizeof(*confllc)); 374 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 375 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 376 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 377 if (reqresp == SMC_LLC_RESP) 378 confllc->hd.flags |= SMC_LLC_FLAG_RESP; 379 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 380 ETH_ALEN); 381 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 382 hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 383 confllc->link_num = link->link_id; 384 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 385 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; 386 /* send llc message */ 387 rc = smc_wr_tx_send(link, pend); 388 return rc; 389 } 390 391 /* send LLC confirm rkey request */ 392 static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 393 struct smc_buf_desc *rmb_desc) 394 { 395 struct smc_llc_msg_confirm_rkey *rkeyllc; 396 struct smc_wr_tx_pend_priv *pend; 397 struct smc_wr_buf *wr_buf; 398 struct smc_link *link; 399 int i, rc, rtok_ix; 400 401 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 402 if (rc) 403 return rc; 404 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 405 memset(rkeyllc, 0, sizeof(*rkeyllc)); 406 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 407 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 408 409 rtok_ix = 1; 410 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 411 link = &send_link->lgr->lnk[i]; 412 if (link->state == SMC_LNK_ACTIVE && link != send_link) { 413 rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 414 rkeyllc->rtoken[rtok_ix].rmb_key = 415 htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 416 rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 417 (u64)sg_dma_address( 418 rmb_desc->sgt[link->link_idx].sgl)); 419 rtok_ix++; 420 } 421 } 422 /* rkey of send_link is in rtoken[0] */ 423 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 424 rkeyllc->rtoken[0].rmb_key = 425 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 426 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 427 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 428 /* send llc message */ 429 rc = smc_wr_tx_send(send_link, pend); 430 return rc; 431 } 432 433 /* send LLC delete rkey request */ 434 static int smc_llc_send_delete_rkey(struct smc_link *link, 435 struct smc_buf_desc *rmb_desc) 436 { 437 struct smc_llc_msg_delete_rkey *rkeyllc; 438 struct smc_wr_tx_pend_priv *pend; 439 struct smc_wr_buf *wr_buf; 440 int rc; 441 442 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 443 if (rc) 444 return rc; 445 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 446 memset(rkeyllc, 0, sizeof(*rkeyllc)); 447 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 448 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 449 rkeyllc->num_rkeys = 1; 450 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 451 /* send llc message */ 452 rc = smc_wr_tx_send(link, pend); 453 return rc; 454 } 455 456 /* send ADD LINK request or response */ 457 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 458 struct smc_link *link_new, 459 enum smc_llc_reqresp reqresp) 460 { 461 struct smc_llc_msg_add_link *addllc; 462 struct smc_wr_tx_pend_priv *pend; 463 struct smc_wr_buf *wr_buf; 464 int rc; 465 466 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 467 if (rc) 468 return rc; 469 addllc = (struct smc_llc_msg_add_link *)wr_buf; 470 471 memset(addllc, 0, sizeof(*addllc)); 472 addllc->hd.common.type = SMC_LLC_ADD_LINK; 473 addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 474 if (reqresp == SMC_LLC_RESP) 475 addllc->hd.flags |= SMC_LLC_FLAG_RESP; 476 memcpy(addllc->sender_mac, mac, ETH_ALEN); 477 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 478 if (link_new) { 479 addllc->link_num = link_new->link_id; 480 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 481 hton24(addllc->initial_psn, link_new->psn_initial); 482 if (reqresp == SMC_LLC_REQ) 483 addllc->qp_mtu = link_new->path_mtu; 484 else 485 addllc->qp_mtu = min(link_new->path_mtu, 486 link_new->peer_mtu); 487 } 488 /* send llc message */ 489 rc = smc_wr_tx_send(link, pend); 490 return rc; 491 } 492 493 /* send DELETE LINK request or response */ 494 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 495 enum smc_llc_reqresp reqresp, bool orderly, 496 u32 reason) 497 { 498 struct smc_llc_msg_del_link *delllc; 499 struct smc_wr_tx_pend_priv *pend; 500 struct smc_wr_buf *wr_buf; 501 int rc; 502 503 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 504 if (rc) 505 return rc; 506 delllc = (struct smc_llc_msg_del_link *)wr_buf; 507 508 memset(delllc, 0, sizeof(*delllc)); 509 delllc->hd.common.type = SMC_LLC_DELETE_LINK; 510 delllc->hd.length = sizeof(struct smc_llc_msg_del_link); 511 if (reqresp == SMC_LLC_RESP) 512 delllc->hd.flags |= SMC_LLC_FLAG_RESP; 513 if (orderly) 514 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 515 if (link_del_id) 516 delllc->link_num = link_del_id; 517 else 518 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 519 delllc->reason = htonl(reason); 520 /* send llc message */ 521 rc = smc_wr_tx_send(link, pend); 522 return rc; 523 } 524 525 /* send LLC test link request */ 526 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 527 { 528 struct smc_llc_msg_test_link *testllc; 529 struct smc_wr_tx_pend_priv *pend; 530 struct smc_wr_buf *wr_buf; 531 int rc; 532 533 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 534 if (rc) 535 return rc; 536 testllc = (struct smc_llc_msg_test_link *)wr_buf; 537 memset(testllc, 0, sizeof(*testllc)); 538 testllc->hd.common.type = SMC_LLC_TEST_LINK; 539 testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 540 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 541 /* send llc message */ 542 rc = smc_wr_tx_send(link, pend); 543 return rc; 544 } 545 546 /* schedule an llc send on link, may wait for buffers */ 547 static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 548 { 549 struct smc_wr_tx_pend_priv *pend; 550 struct smc_wr_buf *wr_buf; 551 int rc; 552 553 if (!smc_link_usable(link)) 554 return -ENOLINK; 555 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 556 if (rc) 557 return rc; 558 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 559 return smc_wr_tx_send(link, pend); 560 } 561 562 /* schedule an llc send on link, may wait for buffers, 563 * and wait for send completion notification. 564 * @return 0 on success 565 */ 566 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 567 { 568 struct smc_wr_tx_pend_priv *pend; 569 struct smc_wr_buf *wr_buf; 570 int rc; 571 572 if (!smc_link_usable(link)) 573 return -ENOLINK; 574 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 575 if (rc) 576 return rc; 577 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 578 return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 579 } 580 581 /********************************* receive ***********************************/ 582 583 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 584 enum smc_lgr_type lgr_new_t) 585 { 586 int i; 587 588 if (lgr->type == SMC_LGR_SYMMETRIC || 589 (lgr->type != SMC_LGR_SINGLE && 590 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 591 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 592 return -EMLINK; 593 594 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 595 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 596 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 597 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 598 return i; 599 } else { 600 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 601 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 602 return i; 603 } 604 return -EMLINK; 605 } 606 607 /* return first buffer from any of the next buf lists */ 608 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 609 int *buf_lst) 610 { 611 struct smc_buf_desc *buf_pos; 612 613 while (*buf_lst < SMC_RMBE_SIZES) { 614 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 615 struct smc_buf_desc, list); 616 if (buf_pos) 617 return buf_pos; 618 (*buf_lst)++; 619 } 620 return NULL; 621 } 622 623 /* return next rmb from buffer lists */ 624 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 625 int *buf_lst, 626 struct smc_buf_desc *buf_pos) 627 { 628 struct smc_buf_desc *buf_next; 629 630 if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 631 (*buf_lst)++; 632 return _smc_llc_get_next_rmb(lgr, buf_lst); 633 } 634 buf_next = list_next_entry(buf_pos, list); 635 return buf_next; 636 } 637 638 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 639 int *buf_lst) 640 { 641 *buf_lst = 0; 642 return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 643 } 644 645 /* send one add_link_continue msg */ 646 static int smc_llc_add_link_cont(struct smc_link *link, 647 struct smc_link *link_new, u8 *num_rkeys_todo, 648 int *buf_lst, struct smc_buf_desc **buf_pos) 649 { 650 struct smc_llc_msg_add_link_cont *addc_llc; 651 struct smc_link_group *lgr = link->lgr; 652 int prim_lnk_idx, lnk_idx, i, rc; 653 struct smc_wr_tx_pend_priv *pend; 654 struct smc_wr_buf *wr_buf; 655 struct smc_buf_desc *rmb; 656 u8 n; 657 658 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 659 if (rc) 660 return rc; 661 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 662 memset(addc_llc, 0, sizeof(*addc_llc)); 663 664 prim_lnk_idx = link->link_idx; 665 lnk_idx = link_new->link_idx; 666 addc_llc->link_num = link_new->link_id; 667 addc_llc->num_rkeys = *num_rkeys_todo; 668 n = *num_rkeys_todo; 669 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 670 if (!*buf_pos) { 671 addc_llc->num_rkeys = addc_llc->num_rkeys - 672 *num_rkeys_todo; 673 *num_rkeys_todo = 0; 674 break; 675 } 676 rmb = *buf_pos; 677 678 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); 679 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); 680 addc_llc->rt[i].rmb_vaddr_new = 681 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 682 683 (*num_rkeys_todo)--; 684 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 685 while (*buf_pos && !(*buf_pos)->used) 686 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 687 } 688 addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; 689 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 690 if (lgr->role == SMC_CLNT) 691 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 692 return smc_wr_tx_send(link, pend); 693 } 694 695 static int smc_llc_cli_rkey_exchange(struct smc_link *link, 696 struct smc_link *link_new) 697 { 698 struct smc_llc_msg_add_link_cont *addc_llc; 699 struct smc_link_group *lgr = link->lgr; 700 u8 max, num_rkeys_send, num_rkeys_recv; 701 struct smc_llc_qentry *qentry; 702 struct smc_buf_desc *buf_pos; 703 int buf_lst; 704 int rc = 0; 705 int i; 706 707 mutex_lock(&lgr->rmbs_lock); 708 num_rkeys_send = lgr->conns_num; 709 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 710 do { 711 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 712 SMC_LLC_ADD_LINK_CONT); 713 if (!qentry) { 714 rc = -ETIMEDOUT; 715 break; 716 } 717 addc_llc = &qentry->msg.add_link_cont; 718 num_rkeys_recv = addc_llc->num_rkeys; 719 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 720 for (i = 0; i < max; i++) { 721 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 722 addc_llc->rt[i].rmb_key, 723 addc_llc->rt[i].rmb_vaddr_new, 724 addc_llc->rt[i].rmb_key_new); 725 num_rkeys_recv--; 726 } 727 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 728 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 729 &buf_lst, &buf_pos); 730 if (rc) 731 break; 732 } while (num_rkeys_send || num_rkeys_recv); 733 734 mutex_unlock(&lgr->rmbs_lock); 735 return rc; 736 } 737 738 /* prepare and send an add link reject response */ 739 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 740 { 741 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 742 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 743 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 744 return smc_llc_send_message(qentry->link, &qentry->msg); 745 } 746 747 static int smc_llc_cli_conf_link(struct smc_link *link, 748 struct smc_init_info *ini, 749 struct smc_link *link_new, 750 enum smc_lgr_type lgr_new_t) 751 { 752 struct smc_link_group *lgr = link->lgr; 753 struct smc_llc_qentry *qentry = NULL; 754 int rc = 0; 755 756 /* receive CONFIRM LINK request over RoCE fabric */ 757 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 758 if (!qentry) { 759 rc = smc_llc_send_delete_link(link, link_new->link_id, 760 SMC_LLC_REQ, false, 761 SMC_LLC_DEL_LOST_PATH); 762 return -ENOLINK; 763 } 764 if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { 765 /* received DELETE_LINK instead */ 766 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 767 smc_llc_send_message(link, &qentry->msg); 768 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 769 return -ENOLINK; 770 } 771 smc_llc_save_peer_uid(qentry); 772 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 773 774 rc = smc_ib_modify_qp_rts(link_new); 775 if (rc) { 776 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 777 false, SMC_LLC_DEL_LOST_PATH); 778 return -ENOLINK; 779 } 780 smc_wr_remember_qp_attr(link_new); 781 782 rc = smcr_buf_reg_lgr(link_new); 783 if (rc) { 784 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 785 false, SMC_LLC_DEL_LOST_PATH); 786 return -ENOLINK; 787 } 788 789 /* send CONFIRM LINK response over RoCE fabric */ 790 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 791 if (rc) { 792 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 793 false, SMC_LLC_DEL_LOST_PATH); 794 return -ENOLINK; 795 } 796 smc_llc_link_active(link_new); 797 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 798 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 799 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 800 else 801 smcr_lgr_set_type(lgr, lgr_new_t); 802 return 0; 803 } 804 805 static void smc_llc_save_add_link_info(struct smc_link *link, 806 struct smc_llc_msg_add_link *add_llc) 807 { 808 link->peer_qpn = ntoh24(add_llc->sender_qp_num); 809 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 810 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 811 link->peer_psn = ntoh24(add_llc->initial_psn); 812 link->peer_mtu = add_llc->qp_mtu; 813 } 814 815 /* as an SMC client, process an add link request */ 816 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 817 { 818 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 819 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 820 struct smc_link_group *lgr = smc_get_lgr(link); 821 struct smc_link *lnk_new = NULL; 822 struct smc_init_info ini; 823 int lnk_idx, rc = 0; 824 825 ini.vlan_id = lgr->vlan_id; 826 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 827 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 828 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { 829 if (!ini.ib_dev) 830 goto out_reject; 831 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 832 } 833 if (!ini.ib_dev) { 834 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 835 ini.ib_dev = link->smcibdev; 836 ini.ib_port = link->ibport; 837 } 838 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 839 if (lnk_idx < 0) 840 goto out_reject; 841 lnk_new = &lgr->lnk[lnk_idx]; 842 rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); 843 if (rc) 844 goto out_reject; 845 smc_llc_save_add_link_info(lnk_new, llc); 846 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 847 smc_llc_link_set_uid(lnk_new); 848 849 rc = smc_ib_ready_link(lnk_new); 850 if (rc) 851 goto out_clear_lnk; 852 853 rc = smcr_buf_map_lgr(lnk_new); 854 if (rc) 855 goto out_clear_lnk; 856 857 rc = smc_llc_send_add_link(link, 858 lnk_new->smcibdev->mac[ini.ib_port - 1], 859 lnk_new->gid, lnk_new, SMC_LLC_RESP); 860 if (rc) 861 goto out_clear_lnk; 862 rc = smc_llc_cli_rkey_exchange(link, lnk_new); 863 if (rc) { 864 rc = 0; 865 goto out_clear_lnk; 866 } 867 rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); 868 if (!rc) 869 goto out; 870 out_clear_lnk: 871 smcr_link_clear(lnk_new, false); 872 out_reject: 873 smc_llc_cli_add_link_reject(qentry); 874 out: 875 kfree(qentry); 876 return rc; 877 } 878 879 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 880 { 881 struct smc_llc_qentry *qentry; 882 883 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 884 885 mutex_lock(&lgr->llc_conf_mutex); 886 smc_llc_cli_add_link(qentry->link, qentry); 887 mutex_unlock(&lgr->llc_conf_mutex); 888 } 889 890 static int smc_llc_active_link_count(struct smc_link_group *lgr) 891 { 892 int i, link_count = 0; 893 894 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 895 if (!smc_link_usable(&lgr->lnk[i])) 896 continue; 897 link_count++; 898 } 899 return link_count; 900 } 901 902 /* find the asymmetric link when 3 links are established */ 903 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 904 { 905 int asym_idx = -ENOENT; 906 int i, j, k; 907 bool found; 908 909 /* determine asymmetric link */ 910 found = false; 911 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 912 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 913 if (!smc_link_usable(&lgr->lnk[i]) || 914 !smc_link_usable(&lgr->lnk[j])) 915 continue; 916 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 917 SMC_GID_SIZE)) { 918 found = true; /* asym_lnk is i or j */ 919 break; 920 } 921 } 922 if (found) 923 break; 924 } 925 if (!found) 926 goto out; /* no asymmetric link */ 927 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 928 if (!smc_link_usable(&lgr->lnk[k])) 929 continue; 930 if (k != i && 931 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 932 SMC_GID_SIZE)) { 933 asym_idx = i; 934 break; 935 } 936 if (k != j && 937 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 938 SMC_GID_SIZE)) { 939 asym_idx = j; 940 break; 941 } 942 } 943 out: 944 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 945 } 946 947 static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 948 { 949 struct smc_link *lnk_new = NULL, *lnk_asym; 950 struct smc_llc_qentry *qentry; 951 int rc; 952 953 lnk_asym = smc_llc_find_asym_link(lgr); 954 if (!lnk_asym) 955 return; /* no asymmetric link */ 956 if (!smc_link_downing(&lnk_asym->state)) 957 return; 958 lnk_new = smc_switch_conns(lgr, lnk_asym, false); 959 smc_wr_tx_wait_no_pending_sends(lnk_asym); 960 if (!lnk_new) 961 goto out_free; 962 /* change flow type from ADD_LINK into DEL_LINK */ 963 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 964 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 965 true, SMC_LLC_DEL_NO_ASYM_NEEDED); 966 if (rc) { 967 smcr_link_down_cond(lnk_new); 968 goto out_free; 969 } 970 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 971 SMC_LLC_DELETE_LINK); 972 if (!qentry) { 973 smcr_link_down_cond(lnk_new); 974 goto out_free; 975 } 976 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 977 out_free: 978 smcr_link_clear(lnk_asym, true); 979 } 980 981 static int smc_llc_srv_rkey_exchange(struct smc_link *link, 982 struct smc_link *link_new) 983 { 984 struct smc_llc_msg_add_link_cont *addc_llc; 985 struct smc_link_group *lgr = link->lgr; 986 u8 max, num_rkeys_send, num_rkeys_recv; 987 struct smc_llc_qentry *qentry = NULL; 988 struct smc_buf_desc *buf_pos; 989 int buf_lst; 990 int rc = 0; 991 int i; 992 993 mutex_lock(&lgr->rmbs_lock); 994 num_rkeys_send = lgr->conns_num; 995 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 996 do { 997 smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 998 &buf_lst, &buf_pos); 999 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 1000 SMC_LLC_ADD_LINK_CONT); 1001 if (!qentry) { 1002 rc = -ETIMEDOUT; 1003 goto out; 1004 } 1005 addc_llc = &qentry->msg.add_link_cont; 1006 num_rkeys_recv = addc_llc->num_rkeys; 1007 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 1008 for (i = 0; i < max; i++) { 1009 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1010 addc_llc->rt[i].rmb_key, 1011 addc_llc->rt[i].rmb_vaddr_new, 1012 addc_llc->rt[i].rmb_key_new); 1013 num_rkeys_recv--; 1014 } 1015 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1016 } while (num_rkeys_send || num_rkeys_recv); 1017 out: 1018 mutex_unlock(&lgr->rmbs_lock); 1019 return rc; 1020 } 1021 1022 static int smc_llc_srv_conf_link(struct smc_link *link, 1023 struct smc_link *link_new, 1024 enum smc_lgr_type lgr_new_t) 1025 { 1026 struct smc_link_group *lgr = link->lgr; 1027 struct smc_llc_qentry *qentry = NULL; 1028 int rc; 1029 1030 /* send CONFIRM LINK request over the RoCE fabric */ 1031 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 1032 if (rc) 1033 return -ENOLINK; 1034 /* receive CONFIRM LINK response over the RoCE fabric */ 1035 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 1036 SMC_LLC_CONFIRM_LINK); 1037 if (!qentry) { 1038 /* send DELETE LINK */ 1039 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 1040 false, SMC_LLC_DEL_LOST_PATH); 1041 return -ENOLINK; 1042 } 1043 smc_llc_save_peer_uid(qentry); 1044 smc_llc_link_active(link_new); 1045 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1046 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1047 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1048 else 1049 smcr_lgr_set_type(lgr, lgr_new_t); 1050 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1051 return 0; 1052 } 1053 1054 int smc_llc_srv_add_link(struct smc_link *link) 1055 { 1056 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1057 struct smc_link_group *lgr = link->lgr; 1058 struct smc_llc_msg_add_link *add_llc; 1059 struct smc_llc_qentry *qentry = NULL; 1060 struct smc_link *link_new; 1061 struct smc_init_info ini; 1062 int lnk_idx, rc = 0; 1063 1064 /* ignore client add link recommendation, start new flow */ 1065 ini.vlan_id = lgr->vlan_id; 1066 smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 1067 if (!ini.ib_dev) { 1068 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1069 ini.ib_dev = link->smcibdev; 1070 ini.ib_port = link->ibport; 1071 } 1072 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1073 if (lnk_idx < 0) 1074 return 0; 1075 1076 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); 1077 if (rc) 1078 return rc; 1079 link_new = &lgr->lnk[lnk_idx]; 1080 rc = smc_llc_send_add_link(link, 1081 link_new->smcibdev->mac[ini.ib_port - 1], 1082 link_new->gid, link_new, SMC_LLC_REQ); 1083 if (rc) 1084 goto out_err; 1085 /* receive ADD LINK response over the RoCE fabric */ 1086 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 1087 if (!qentry) { 1088 rc = -ETIMEDOUT; 1089 goto out_err; 1090 } 1091 add_llc = &qentry->msg.add_link; 1092 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 1093 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1094 rc = -ENOLINK; 1095 goto out_err; 1096 } 1097 if (lgr->type == SMC_LGR_SINGLE && 1098 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1099 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { 1100 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1101 } 1102 smc_llc_save_add_link_info(link_new, add_llc); 1103 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1104 1105 rc = smc_ib_ready_link(link_new); 1106 if (rc) 1107 goto out_err; 1108 rc = smcr_buf_map_lgr(link_new); 1109 if (rc) 1110 goto out_err; 1111 rc = smcr_buf_reg_lgr(link_new); 1112 if (rc) 1113 goto out_err; 1114 rc = smc_llc_srv_rkey_exchange(link, link_new); 1115 if (rc) 1116 goto out_err; 1117 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 1118 if (rc) 1119 goto out_err; 1120 return 0; 1121 out_err: 1122 smcr_link_clear(link_new, false); 1123 return rc; 1124 } 1125 1126 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 1127 { 1128 struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 1129 int rc; 1130 1131 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1132 1133 mutex_lock(&lgr->llc_conf_mutex); 1134 rc = smc_llc_srv_add_link(link); 1135 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 1136 /* delete any asymmetric link */ 1137 smc_llc_delete_asym_link(lgr); 1138 } 1139 mutex_unlock(&lgr->llc_conf_mutex); 1140 } 1141 1142 /* enqueue a local add_link req to trigger a new add_link flow, only as SERV */ 1143 void smc_llc_srv_add_link_local(struct smc_link *link) 1144 { 1145 struct smc_llc_msg_add_link add_llc = {0}; 1146 1147 add_llc.hd.length = sizeof(add_llc); 1148 add_llc.hd.common.type = SMC_LLC_ADD_LINK; 1149 /* no dev and port needed, we as server ignore client data anyway */ 1150 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); 1151 } 1152 1153 /* worker to process an add link message */ 1154 static void smc_llc_add_link_work(struct work_struct *work) 1155 { 1156 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1157 llc_add_link_work); 1158 1159 if (list_empty(&lgr->list)) { 1160 /* link group is terminating */ 1161 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1162 goto out; 1163 } 1164 1165 if (lgr->role == SMC_CLNT) 1166 smc_llc_process_cli_add_link(lgr); 1167 else 1168 smc_llc_process_srv_add_link(lgr); 1169 out: 1170 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1171 } 1172 1173 /* enqueue a local del_link msg to trigger a new del_link flow, 1174 * called only for role SMC_SERV 1175 */ 1176 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 1177 { 1178 struct smc_llc_msg_del_link del_llc = {0}; 1179 1180 del_llc.hd.length = sizeof(del_llc); 1181 del_llc.hd.common.type = SMC_LLC_DELETE_LINK; 1182 del_llc.link_num = del_link_id; 1183 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 1184 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1185 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); 1186 } 1187 1188 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 1189 { 1190 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 1191 struct smc_llc_msg_del_link *del_llc; 1192 struct smc_llc_qentry *qentry; 1193 int active_links; 1194 int lnk_idx; 1195 1196 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1197 lnk = qentry->link; 1198 del_llc = &qentry->msg.delete_link; 1199 1200 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1201 smc_lgr_terminate_sched(lgr); 1202 goto out; 1203 } 1204 mutex_lock(&lgr->llc_conf_mutex); 1205 /* delete single link */ 1206 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 1207 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 1208 continue; 1209 lnk_del = &lgr->lnk[lnk_idx]; 1210 break; 1211 } 1212 del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 1213 if (!lnk_del) { 1214 /* link was not found */ 1215 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 1216 smc_llc_send_message(lnk, &qentry->msg); 1217 goto out_unlock; 1218 } 1219 lnk_asym = smc_llc_find_asym_link(lgr); 1220 1221 del_llc->reason = 0; 1222 smc_llc_send_message(lnk, &qentry->msg); /* response */ 1223 1224 if (smc_link_downing(&lnk_del->state)) { 1225 smc_switch_conns(lgr, lnk_del, false); 1226 smc_wr_tx_wait_no_pending_sends(lnk_del); 1227 } 1228 smcr_link_clear(lnk_del, true); 1229 1230 active_links = smc_llc_active_link_count(lgr); 1231 if (lnk_del == lnk_asym) { 1232 /* expected deletion of asym link, don't change lgr state */ 1233 } else if (active_links == 1) { 1234 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1235 } else if (!active_links) { 1236 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1237 smc_lgr_terminate_sched(lgr); 1238 } 1239 out_unlock: 1240 mutex_unlock(&lgr->llc_conf_mutex); 1241 out: 1242 kfree(qentry); 1243 } 1244 1245 /* try to send a DELETE LINK ALL request on any active link, 1246 * waiting for send completion 1247 */ 1248 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1249 { 1250 struct smc_llc_msg_del_link delllc = {0}; 1251 int i; 1252 1253 delllc.hd.common.type = SMC_LLC_DELETE_LINK; 1254 delllc.hd.length = sizeof(delllc); 1255 if (ord) 1256 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1257 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1258 delllc.reason = htonl(rsn); 1259 1260 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1261 if (!smc_link_usable(&lgr->lnk[i])) 1262 continue; 1263 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1264 break; 1265 } 1266 } 1267 1268 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 1269 { 1270 struct smc_llc_msg_del_link *del_llc; 1271 struct smc_link *lnk, *lnk_del; 1272 struct smc_llc_qentry *qentry; 1273 int active_links; 1274 int i; 1275 1276 mutex_lock(&lgr->llc_conf_mutex); 1277 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1278 lnk = qentry->link; 1279 del_llc = &qentry->msg.delete_link; 1280 1281 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1282 /* delete entire lgr */ 1283 smc_llc_send_link_delete_all(lgr, true, ntohl( 1284 qentry->msg.delete_link.reason)); 1285 smc_lgr_terminate_sched(lgr); 1286 goto out; 1287 } 1288 /* delete single link */ 1289 lnk_del = NULL; 1290 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1291 if (lgr->lnk[i].link_id == del_llc->link_num) { 1292 lnk_del = &lgr->lnk[i]; 1293 break; 1294 } 1295 } 1296 if (!lnk_del) 1297 goto out; /* asymmetric link already deleted */ 1298 1299 if (smc_link_downing(&lnk_del->state)) { 1300 smc_switch_conns(lgr, lnk_del, false); 1301 smc_wr_tx_wait_no_pending_sends(lnk_del); 1302 } 1303 if (!list_empty(&lgr->list)) { 1304 /* qentry is either a request from peer (send it back to 1305 * initiate the DELETE_LINK processing), or a locally 1306 * enqueued DELETE_LINK request (forward it) 1307 */ 1308 if (!smc_llc_send_message(lnk, &qentry->msg)) { 1309 struct smc_llc_qentry *qentry2; 1310 1311 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 1312 SMC_LLC_DELETE_LINK); 1313 if (qentry2) 1314 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1315 } 1316 } 1317 smcr_link_clear(lnk_del, true); 1318 1319 active_links = smc_llc_active_link_count(lgr); 1320 if (active_links == 1) { 1321 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1322 } else if (!active_links) { 1323 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1324 smc_lgr_terminate_sched(lgr); 1325 } 1326 1327 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 1328 /* trigger setup of asymm alt link */ 1329 smc_llc_srv_add_link_local(lnk); 1330 } 1331 out: 1332 mutex_unlock(&lgr->llc_conf_mutex); 1333 kfree(qentry); 1334 } 1335 1336 static void smc_llc_delete_link_work(struct work_struct *work) 1337 { 1338 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1339 llc_del_link_work); 1340 1341 if (list_empty(&lgr->list)) { 1342 /* link group is terminating */ 1343 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1344 goto out; 1345 } 1346 1347 if (lgr->role == SMC_CLNT) 1348 smc_llc_process_cli_delete_link(lgr); 1349 else 1350 smc_llc_process_srv_delete_link(lgr); 1351 out: 1352 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1353 } 1354 1355 /* process a confirm_rkey request from peer, remote flow */ 1356 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 1357 { 1358 struct smc_llc_msg_confirm_rkey *llc; 1359 struct smc_llc_qentry *qentry; 1360 struct smc_link *link; 1361 int num_entries; 1362 int rk_idx; 1363 int i; 1364 1365 qentry = lgr->llc_flow_rmt.qentry; 1366 llc = &qentry->msg.confirm_rkey; 1367 link = qentry->link; 1368 1369 num_entries = llc->rtoken[0].num_rkeys; 1370 /* first rkey entry is for receiving link */ 1371 rk_idx = smc_rtoken_add(link, 1372 llc->rtoken[0].rmb_vaddr, 1373 llc->rtoken[0].rmb_key); 1374 if (rk_idx < 0) 1375 goto out_err; 1376 1377 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 1378 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 1379 llc->rtoken[i].rmb_vaddr, 1380 llc->rtoken[i].rmb_key); 1381 /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 1382 goto out; 1383 out_err: 1384 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1385 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 1386 out: 1387 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1388 smc_llc_send_message(link, &qentry->msg); 1389 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1390 } 1391 1392 /* process a delete_rkey request from peer, remote flow */ 1393 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 1394 { 1395 struct smc_llc_msg_delete_rkey *llc; 1396 struct smc_llc_qentry *qentry; 1397 struct smc_link *link; 1398 u8 err_mask = 0; 1399 int i, max; 1400 1401 qentry = lgr->llc_flow_rmt.qentry; 1402 llc = &qentry->msg.delete_rkey; 1403 link = qentry->link; 1404 1405 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 1406 for (i = 0; i < max; i++) { 1407 if (smc_rtoken_delete(link, llc->rkey[i])) 1408 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 1409 } 1410 if (err_mask) { 1411 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1412 llc->err_mask = err_mask; 1413 } 1414 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1415 smc_llc_send_message(link, &qentry->msg); 1416 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1417 } 1418 1419 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 1420 { 1421 pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: " 1422 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type); 1423 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 1424 smc_lgr_terminate_sched(lgr); 1425 } 1426 1427 /* flush the llc event queue */ 1428 static void smc_llc_event_flush(struct smc_link_group *lgr) 1429 { 1430 struct smc_llc_qentry *qentry, *q; 1431 1432 spin_lock_bh(&lgr->llc_event_q_lock); 1433 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 1434 list_del_init(&qentry->list); 1435 kfree(qentry); 1436 } 1437 spin_unlock_bh(&lgr->llc_event_q_lock); 1438 } 1439 1440 static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 1441 { 1442 union smc_llc_msg *llc = &qentry->msg; 1443 struct smc_link *link = qentry->link; 1444 struct smc_link_group *lgr = link->lgr; 1445 1446 if (!smc_link_usable(link)) 1447 goto out; 1448 1449 switch (llc->raw.hdr.common.type) { 1450 case SMC_LLC_TEST_LINK: 1451 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 1452 smc_llc_send_message(link, llc); 1453 break; 1454 case SMC_LLC_ADD_LINK: 1455 if (list_empty(&lgr->list)) 1456 goto out; /* lgr is terminating */ 1457 if (lgr->role == SMC_CLNT) { 1458 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { 1459 /* a flow is waiting for this message */ 1460 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1461 qentry); 1462 wake_up_interruptible(&lgr->llc_waiter); 1463 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1464 qentry)) { 1465 schedule_work(&lgr->llc_add_link_work); 1466 } 1467 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1468 /* as smc server, handle client suggestion */ 1469 schedule_work(&lgr->llc_add_link_work); 1470 } 1471 return; 1472 case SMC_LLC_CONFIRM_LINK: 1473 case SMC_LLC_ADD_LINK_CONT: 1474 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 1475 /* a flow is waiting for this message */ 1476 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1477 wake_up_interruptible(&lgr->llc_waiter); 1478 return; 1479 } 1480 break; 1481 case SMC_LLC_DELETE_LINK: 1482 if (lgr->role == SMC_CLNT) { 1483 /* server requests to delete this link, send response */ 1484 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 1485 /* DEL LINK REQ during ADD LINK SEQ */ 1486 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1487 qentry); 1488 wake_up_interruptible(&lgr->llc_waiter); 1489 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1490 qentry)) { 1491 schedule_work(&lgr->llc_del_link_work); 1492 } 1493 } else { 1494 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1495 !lgr->llc_flow_lcl.qentry) { 1496 /* DEL LINK REQ during ADD LINK SEQ */ 1497 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1498 qentry); 1499 wake_up_interruptible(&lgr->llc_waiter); 1500 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1501 qentry)) { 1502 schedule_work(&lgr->llc_del_link_work); 1503 } 1504 } 1505 return; 1506 case SMC_LLC_CONFIRM_RKEY: 1507 /* new request from remote, assign to remote flow */ 1508 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1509 /* process here, does not wait for more llc msgs */ 1510 smc_llc_rmt_conf_rkey(lgr); 1511 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1512 } 1513 return; 1514 case SMC_LLC_CONFIRM_RKEY_CONT: 1515 /* not used because max links is 3, and 3 rkeys fit into 1516 * one CONFIRM_RKEY message 1517 */ 1518 break; 1519 case SMC_LLC_DELETE_RKEY: 1520 /* new request from remote, assign to remote flow */ 1521 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1522 /* process here, does not wait for more llc msgs */ 1523 smc_llc_rmt_delete_rkey(lgr); 1524 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1525 } 1526 return; 1527 default: 1528 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 1529 break; 1530 } 1531 out: 1532 kfree(qentry); 1533 } 1534 1535 /* worker to process llc messages on the event queue */ 1536 static void smc_llc_event_work(struct work_struct *work) 1537 { 1538 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1539 llc_event_work); 1540 struct smc_llc_qentry *qentry; 1541 1542 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 1543 if (smc_link_usable(lgr->delayed_event->link)) { 1544 smc_llc_event_handler(lgr->delayed_event); 1545 } else { 1546 qentry = lgr->delayed_event; 1547 lgr->delayed_event = NULL; 1548 kfree(qentry); 1549 } 1550 } 1551 1552 again: 1553 spin_lock_bh(&lgr->llc_event_q_lock); 1554 if (!list_empty(&lgr->llc_event_q)) { 1555 qentry = list_first_entry(&lgr->llc_event_q, 1556 struct smc_llc_qentry, list); 1557 list_del_init(&qentry->list); 1558 spin_unlock_bh(&lgr->llc_event_q_lock); 1559 smc_llc_event_handler(qentry); 1560 goto again; 1561 } 1562 spin_unlock_bh(&lgr->llc_event_q_lock); 1563 } 1564 1565 /* process llc responses in tasklet context */ 1566 static void smc_llc_rx_response(struct smc_link *link, 1567 struct smc_llc_qentry *qentry) 1568 { 1569 u8 llc_type = qentry->msg.raw.hdr.common.type; 1570 1571 switch (llc_type) { 1572 case SMC_LLC_TEST_LINK: 1573 if (link->state == SMC_LNK_ACTIVE) 1574 complete(&link->llc_testlink_resp); 1575 break; 1576 case SMC_LLC_ADD_LINK: 1577 case SMC_LLC_DELETE_LINK: 1578 case SMC_LLC_CONFIRM_LINK: 1579 case SMC_LLC_ADD_LINK_CONT: 1580 case SMC_LLC_CONFIRM_RKEY: 1581 case SMC_LLC_DELETE_RKEY: 1582 /* assign responses to the local flow, we requested them */ 1583 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 1584 wake_up_interruptible(&link->lgr->llc_waiter); 1585 return; 1586 case SMC_LLC_CONFIRM_RKEY_CONT: 1587 /* not used because max links is 3 */ 1588 break; 1589 default: 1590 smc_llc_protocol_violation(link->lgr, llc_type); 1591 break; 1592 } 1593 kfree(qentry); 1594 } 1595 1596 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 1597 { 1598 struct smc_link_group *lgr = link->lgr; 1599 struct smc_llc_qentry *qentry; 1600 unsigned long flags; 1601 1602 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 1603 if (!qentry) 1604 return; 1605 qentry->link = link; 1606 INIT_LIST_HEAD(&qentry->list); 1607 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 1608 1609 /* process responses immediately */ 1610 if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 1611 smc_llc_rx_response(link, qentry); 1612 return; 1613 } 1614 1615 /* add requests to event queue */ 1616 spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 1617 list_add_tail(&qentry->list, &lgr->llc_event_q); 1618 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 1619 schedule_work(&link->lgr->llc_event_work); 1620 } 1621 1622 /* copy received msg and add it to the event queue */ 1623 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 1624 { 1625 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 1626 union smc_llc_msg *llc = buf; 1627 1628 if (wc->byte_len < sizeof(*llc)) 1629 return; /* short message */ 1630 if (llc->raw.hdr.length != sizeof(*llc)) 1631 return; /* invalid message */ 1632 1633 smc_llc_enqueue(link, llc); 1634 } 1635 1636 /***************************** worker, utils *********************************/ 1637 1638 static void smc_llc_testlink_work(struct work_struct *work) 1639 { 1640 struct smc_link *link = container_of(to_delayed_work(work), 1641 struct smc_link, llc_testlink_wrk); 1642 unsigned long next_interval; 1643 unsigned long expire_time; 1644 u8 user_data[16] = { 0 }; 1645 int rc; 1646 1647 if (link->state != SMC_LNK_ACTIVE) 1648 return; /* don't reschedule worker */ 1649 expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 1650 if (time_is_after_jiffies(expire_time)) { 1651 next_interval = expire_time - jiffies; 1652 goto out; 1653 } 1654 reinit_completion(&link->llc_testlink_resp); 1655 smc_llc_send_test_link(link, user_data); 1656 /* receive TEST LINK response over RoCE fabric */ 1657 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 1658 SMC_LLC_WAIT_TIME); 1659 if (link->state != SMC_LNK_ACTIVE) 1660 return; /* link state changed */ 1661 if (rc <= 0) { 1662 smcr_link_down_cond_sched(link); 1663 return; 1664 } 1665 next_interval = link->llc_testlink_time; 1666 out: 1667 schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 1668 } 1669 1670 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 1671 { 1672 struct net *net = sock_net(smc->clcsock->sk); 1673 1674 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 1675 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 1676 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 1677 INIT_LIST_HEAD(&lgr->llc_event_q); 1678 spin_lock_init(&lgr->llc_event_q_lock); 1679 spin_lock_init(&lgr->llc_flow_lock); 1680 init_waitqueue_head(&lgr->llc_waiter); 1681 mutex_init(&lgr->llc_conf_mutex); 1682 lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 1683 } 1684 1685 /* called after lgr was removed from lgr_list */ 1686 void smc_llc_lgr_clear(struct smc_link_group *lgr) 1687 { 1688 smc_llc_event_flush(lgr); 1689 wake_up_interruptible_all(&lgr->llc_waiter); 1690 cancel_work_sync(&lgr->llc_event_work); 1691 cancel_work_sync(&lgr->llc_add_link_work); 1692 cancel_work_sync(&lgr->llc_del_link_work); 1693 if (lgr->delayed_event) { 1694 kfree(lgr->delayed_event); 1695 lgr->delayed_event = NULL; 1696 } 1697 } 1698 1699 int smc_llc_link_init(struct smc_link *link) 1700 { 1701 init_completion(&link->llc_testlink_resp); 1702 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 1703 return 0; 1704 } 1705 1706 void smc_llc_link_active(struct smc_link *link) 1707 { 1708 pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, " 1709 "peerid %*phN, ibdev %s, ibport %d\n", 1710 SMC_LGR_ID_SIZE, &link->lgr->id, 1711 SMC_LGR_ID_SIZE, &link->link_uid, 1712 SMC_LGR_ID_SIZE, &link->peer_link_uid, 1713 link->smcibdev->ibdev->name, link->ibport); 1714 link->state = SMC_LNK_ACTIVE; 1715 if (link->lgr->llc_testlink_time) { 1716 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; 1717 schedule_delayed_work(&link->llc_testlink_wrk, 1718 link->llc_testlink_time); 1719 } 1720 } 1721 1722 /* called in worker context */ 1723 void smc_llc_link_clear(struct smc_link *link, bool log) 1724 { 1725 if (log) 1726 pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN" 1727 ", peerid %*phN, ibdev %s, ibport %d\n", 1728 SMC_LGR_ID_SIZE, &link->lgr->id, 1729 SMC_LGR_ID_SIZE, &link->link_uid, 1730 SMC_LGR_ID_SIZE, &link->peer_link_uid, 1731 link->smcibdev->ibdev->name, link->ibport); 1732 complete(&link->llc_testlink_resp); 1733 cancel_delayed_work_sync(&link->llc_testlink_wrk); 1734 smc_wr_wakeup_reg_wait(link); 1735 smc_wr_wakeup_tx_wait(link); 1736 } 1737 1738 /* register a new rtoken at the remote peer (for all links) */ 1739 int smc_llc_do_confirm_rkey(struct smc_link *send_link, 1740 struct smc_buf_desc *rmb_desc) 1741 { 1742 struct smc_link_group *lgr = send_link->lgr; 1743 struct smc_llc_qentry *qentry = NULL; 1744 int rc = 0; 1745 1746 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 1747 if (rc) 1748 goto out; 1749 /* receive CONFIRM RKEY response from server over RoCE fabric */ 1750 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 1751 SMC_LLC_CONFIRM_RKEY); 1752 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 1753 rc = -EFAULT; 1754 out: 1755 if (qentry) 1756 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1757 return rc; 1758 } 1759 1760 /* unregister an rtoken at the remote peer */ 1761 int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 1762 struct smc_buf_desc *rmb_desc) 1763 { 1764 struct smc_llc_qentry *qentry = NULL; 1765 struct smc_link *send_link; 1766 int rc = 0; 1767 1768 send_link = smc_llc_usable_link(lgr); 1769 if (!send_link) 1770 return -ENOLINK; 1771 1772 /* protected by llc_flow control */ 1773 rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 1774 if (rc) 1775 goto out; 1776 /* receive DELETE RKEY response from server over RoCE fabric */ 1777 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 1778 SMC_LLC_DELETE_RKEY); 1779 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 1780 rc = -EFAULT; 1781 out: 1782 if (qentry) 1783 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1784 return rc; 1785 } 1786 1787 void smc_llc_link_set_uid(struct smc_link *link) 1788 { 1789 __be32 link_uid; 1790 1791 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 1792 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 1793 } 1794 1795 /* save peers link user id, used for debug purposes */ 1796 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 1797 { 1798 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 1799 SMC_LGR_ID_SIZE); 1800 } 1801 1802 /* evaluate confirm link request or response */ 1803 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 1804 enum smc_llc_reqresp type) 1805 { 1806 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 1807 qentry->link->link_id = qentry->msg.confirm_link.link_num; 1808 smc_llc_link_set_uid(qentry->link); 1809 } 1810 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 1811 return -ENOTSUPP; 1812 return 0; 1813 } 1814 1815 /***************************** init, exit, misc ******************************/ 1816 1817 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 1818 { 1819 .handler = smc_llc_rx_handler, 1820 .type = SMC_LLC_CONFIRM_LINK 1821 }, 1822 { 1823 .handler = smc_llc_rx_handler, 1824 .type = SMC_LLC_TEST_LINK 1825 }, 1826 { 1827 .handler = smc_llc_rx_handler, 1828 .type = SMC_LLC_ADD_LINK 1829 }, 1830 { 1831 .handler = smc_llc_rx_handler, 1832 .type = SMC_LLC_ADD_LINK_CONT 1833 }, 1834 { 1835 .handler = smc_llc_rx_handler, 1836 .type = SMC_LLC_DELETE_LINK 1837 }, 1838 { 1839 .handler = smc_llc_rx_handler, 1840 .type = SMC_LLC_CONFIRM_RKEY 1841 }, 1842 { 1843 .handler = smc_llc_rx_handler, 1844 .type = SMC_LLC_CONFIRM_RKEY_CONT 1845 }, 1846 { 1847 .handler = smc_llc_rx_handler, 1848 .type = SMC_LLC_DELETE_RKEY 1849 }, 1850 { 1851 .handler = NULL, 1852 } 1853 }; 1854 1855 int __init smc_llc_init(void) 1856 { 1857 struct smc_wr_rx_handler *handler; 1858 int rc = 0; 1859 1860 for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 1861 INIT_HLIST_NODE(&handler->list); 1862 rc = smc_wr_rx_register_handler(handler); 1863 if (rc) 1864 break; 1865 } 1866 return rc; 1867 } 1868