1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Link Layer Control (LLC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 10 * Ursula Braun <ubraun@linux.vnet.ibm.com> 11 */ 12 13 #include <net/tcp.h> 14 #include <rdma/ib_verbs.h> 15 16 #include "smc.h" 17 #include "smc_core.h" 18 #include "smc_clc.h" 19 #include "smc_llc.h" 20 #include "smc_pnet.h" 21 22 #define SMC_LLC_DATA_LEN 40 23 24 struct smc_llc_hdr { 25 struct smc_wr_rx_hdr common; 26 union { 27 struct { 28 u8 length; /* 44 */ 29 #if defined(__BIG_ENDIAN_BITFIELD) 30 u8 reserved:4, 31 add_link_rej_rsn:4; 32 #elif defined(__LITTLE_ENDIAN_BITFIELD) 33 u8 add_link_rej_rsn:4, 34 reserved:4; 35 #endif 36 }; 37 u16 length_v2; /* 44 - 8192*/ 38 }; 39 u8 flags; 40 } __packed; /* format defined in 41 * IBM Shared Memory Communications Version 2 42 * (https://www.ibm.com/support/pages/node/6326337) 43 */ 44 45 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 46 47 struct smc_llc_msg_confirm_link { /* type 0x01 */ 48 struct smc_llc_hdr hd; 49 u8 sender_mac[ETH_ALEN]; 50 u8 sender_gid[SMC_GID_SIZE]; 51 u8 sender_qp_num[3]; 52 u8 link_num; 53 u8 link_uid[SMC_LGR_ID_SIZE]; 54 u8 max_links; 55 u8 max_conns; 56 u8 reserved[8]; 57 }; 58 59 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 60 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 61 62 struct smc_llc_msg_add_link { /* type 0x02 */ 63 struct smc_llc_hdr hd; 64 u8 sender_mac[ETH_ALEN]; 65 u8 reserved2[2]; 66 u8 sender_gid[SMC_GID_SIZE]; 67 u8 sender_qp_num[3]; 68 u8 link_num; 69 #if defined(__BIG_ENDIAN_BITFIELD) 70 u8 reserved3 : 4, 71 qp_mtu : 4; 72 #elif defined(__LITTLE_ENDIAN_BITFIELD) 73 u8 qp_mtu : 4, 74 reserved3 : 4; 75 #endif 76 u8 initial_psn[3]; 77 u8 reserved[8]; 78 }; 79 80 struct smc_llc_msg_add_link_cont_rt { 81 __be32 rmb_key; 82 __be32 rmb_key_new; 83 __be64 rmb_vaddr_new; 84 }; 85 86 struct smc_llc_msg_add_link_v2_ext { 87 #if defined(__BIG_ENDIAN_BITFIELD) 88 u8 v2_direct : 1, 89 reserved : 7; 90 #elif defined(__LITTLE_ENDIAN_BITFIELD) 91 u8 reserved : 7, 92 v2_direct : 1; 93 #endif 94 u8 reserved2; 95 u8 client_target_gid[SMC_GID_SIZE]; 96 u8 reserved3[8]; 97 u16 num_rkeys; 98 struct smc_llc_msg_add_link_cont_rt rt[]; 99 } __packed; /* format defined in 100 * IBM Shared Memory Communications Version 2 101 * (https://www.ibm.com/support/pages/node/6326337) 102 */ 103 104 struct smc_llc_msg_req_add_link_v2 { 105 struct smc_llc_hdr hd; 106 u8 reserved[20]; 107 u8 gid_cnt; 108 u8 reserved2[3]; 109 u8 gid[][SMC_GID_SIZE]; 110 }; 111 112 #define SMC_LLC_RKEYS_PER_CONT_MSG 2 113 114 struct smc_llc_msg_add_link_cont { /* type 0x03 */ 115 struct smc_llc_hdr hd; 116 u8 link_num; 117 u8 num_rkeys; 118 u8 reserved2[2]; 119 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 120 u8 reserved[4]; 121 } __packed; /* format defined in RFC7609 */ 122 123 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 124 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 125 126 struct smc_llc_msg_del_link { /* type 0x04 */ 127 struct smc_llc_hdr hd; 128 u8 link_num; 129 __be32 reason; 130 u8 reserved[35]; 131 } __packed; /* format defined in RFC7609 */ 132 133 struct smc_llc_msg_test_link { /* type 0x07 */ 134 struct smc_llc_hdr hd; 135 u8 user_data[16]; 136 u8 reserved[24]; 137 }; 138 139 struct smc_rmb_rtoken { 140 union { 141 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 142 /* is actually the num of rtokens, first */ 143 /* rtoken is always for the current link */ 144 u8 link_id; /* link id of the rtoken */ 145 }; 146 __be32 rmb_key; 147 __be64 rmb_vaddr; 148 } __packed; /* format defined in RFC7609 */ 149 150 #define SMC_LLC_RKEYS_PER_MSG 3 151 #define SMC_LLC_RKEYS_PER_MSG_V2 255 152 153 struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 154 struct smc_llc_hdr hd; 155 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 156 u8 reserved; 157 }; 158 159 #define SMC_LLC_DEL_RKEY_MAX 8 160 #define SMC_LLC_DEL_RKEY_V2_INLINE 9 161 #define SMC_LLC_FLAG_RKEY_RETRY 0x10 162 #define SMC_LLC_FLAG_RKEY_NEG 0x20 163 164 struct smc_llc_msg_delete_rkey { /* type 0x09 */ 165 struct smc_llc_hdr hd; 166 u8 num_rkeys; 167 u8 err_mask; 168 u8 reserved[2]; 169 __be32 rkey[8]; 170 u8 reserved2[4]; 171 }; 172 173 struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */ 174 struct smc_llc_hdr hd; 175 u8 num_rkeys; 176 u8 num_inval_rkeys; 177 u8 reserved[2]; 178 __be32 rkey[]; 179 }; 180 181 /* the leading rkeys of a DELETE_RKEY_V2 fit into union smc_llc_msg */ 182 struct smc_llc_msg_delete_rkey_v2_inline { /* type 0x29 */ 183 struct smc_llc_hdr hd; 184 u8 num_rkeys; 185 u8 num_inval_rkeys; 186 u8 reserved[2]; 187 __be32 rkey[SMC_LLC_DEL_RKEY_V2_INLINE]; 188 }; 189 190 union smc_llc_msg { 191 struct smc_llc_msg_confirm_link confirm_link; 192 struct smc_llc_msg_add_link add_link; 193 struct smc_llc_msg_req_add_link_v2 req_add_link; 194 struct smc_llc_msg_add_link_cont add_link_cont; 195 struct smc_llc_msg_del_link delete_link; 196 197 struct smc_llc_msg_confirm_rkey confirm_rkey; 198 struct smc_llc_msg_delete_rkey delete_rkey; 199 struct smc_llc_msg_delete_rkey_v2_inline delete_rkey_v2; 200 201 struct smc_llc_msg_test_link test_link; 202 struct { 203 struct smc_llc_hdr hdr; 204 u8 data[SMC_LLC_DATA_LEN]; 205 } raw; 206 }; 207 208 static_assert(SMC_LLC_DEL_RKEY_V2_INLINE == 209 (sizeof(union smc_llc_msg) - 210 offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)) / 211 sizeof(__be32)); 212 static_assert(offsetof(struct smc_llc_msg_delete_rkey_v2_inline, rkey) == 213 offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)); 214 215 #define SMC_LLC_FLAG_RESP 0x80 216 217 struct smc_llc_qentry { 218 struct list_head list; 219 struct smc_link *link; 220 u16 body_len; 221 union smc_llc_msg msg; 222 u8 body[] __counted_by(body_len); 223 }; 224 225 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, 226 u32 byte_len); 227 228 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 229 { 230 struct smc_llc_qentry *qentry = flow->qentry; 231 232 flow->qentry = NULL; 233 return qentry; 234 } 235 236 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 237 { 238 struct smc_llc_qentry *qentry; 239 240 if (flow->qentry) { 241 qentry = flow->qentry; 242 flow->qentry = NULL; 243 kfree(qentry); 244 } 245 } 246 247 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 248 struct smc_llc_qentry *qentry) 249 { 250 flow->qentry = qentry; 251 } 252 253 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, 254 struct smc_llc_qentry *qentry) 255 { 256 u8 msg_type = qentry->msg.raw.hdr.common.llc_type; 257 258 if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && 259 flow_type != msg_type && !lgr->delayed_event) { 260 lgr->delayed_event = qentry; 261 return; 262 } 263 /* drop parallel or already-in-progress llc requests */ 264 if (flow_type != msg_type) 265 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel " 266 "LLC msg: msg %d flow %d role %d\n", 267 SMC_LGR_ID_SIZE, &lgr->id, 268 lgr->net->net_cookie, 269 qentry->msg.raw.hdr.common.type, 270 flow_type, lgr->role); 271 kfree(qentry); 272 } 273 274 /* try to start a new llc flow, initiated by an incoming llc msg */ 275 static bool smc_llc_flow_start(struct smc_llc_flow *flow, 276 struct smc_llc_qentry *qentry) 277 { 278 struct smc_link_group *lgr = qentry->link->lgr; 279 280 spin_lock_bh(&lgr->llc_flow_lock); 281 if (flow->type) { 282 /* a flow is already active */ 283 smc_llc_flow_parallel(lgr, flow->type, qentry); 284 spin_unlock_bh(&lgr->llc_flow_lock); 285 return false; 286 } 287 switch (qentry->msg.raw.hdr.common.llc_type) { 288 case SMC_LLC_ADD_LINK: 289 flow->type = SMC_LLC_FLOW_ADD_LINK; 290 break; 291 case SMC_LLC_DELETE_LINK: 292 flow->type = SMC_LLC_FLOW_DEL_LINK; 293 break; 294 case SMC_LLC_CONFIRM_RKEY: 295 case SMC_LLC_DELETE_RKEY: 296 flow->type = SMC_LLC_FLOW_RKEY; 297 break; 298 default: 299 flow->type = SMC_LLC_FLOW_NONE; 300 } 301 smc_llc_flow_qentry_set(flow, qentry); 302 spin_unlock_bh(&lgr->llc_flow_lock); 303 return true; 304 } 305 306 /* start a new local llc flow, wait till current flow finished */ 307 int smc_llc_flow_initiate(struct smc_link_group *lgr, 308 enum smc_llc_flowtype type) 309 { 310 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 311 int rc; 312 313 /* all flows except confirm_rkey and delete_rkey are exclusive, 314 * confirm/delete rkey flows can run concurrently (local and remote) 315 */ 316 if (type == SMC_LLC_FLOW_RKEY) 317 allowed_remote = SMC_LLC_FLOW_RKEY; 318 again: 319 if (list_empty(&lgr->list)) 320 return -ENODEV; 321 spin_lock_bh(&lgr->llc_flow_lock); 322 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 323 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 324 lgr->llc_flow_rmt.type == allowed_remote)) { 325 lgr->llc_flow_lcl.type = type; 326 spin_unlock_bh(&lgr->llc_flow_lock); 327 return 0; 328 } 329 spin_unlock_bh(&lgr->llc_flow_lock); 330 rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || 331 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 332 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 333 lgr->llc_flow_rmt.type == allowed_remote))), 334 SMC_LLC_WAIT_TIME * 10); 335 if (!rc) 336 return -ETIMEDOUT; 337 goto again; 338 } 339 340 /* finish the current llc flow */ 341 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 342 { 343 spin_lock_bh(&lgr->llc_flow_lock); 344 smc_llc_flow_qentry_del(flow); 345 memset(flow, 0, sizeof(*flow)); 346 flow->type = SMC_LLC_FLOW_NONE; 347 spin_unlock_bh(&lgr->llc_flow_lock); 348 if (!list_empty(&lgr->list) && lgr->delayed_event && 349 flow == &lgr->llc_flow_lcl) 350 schedule_work(&lgr->llc_event_work); 351 else 352 wake_up(&lgr->llc_flow_waiter); 353 } 354 355 /* lnk is optional and used for early wakeup when link goes down, useful in 356 * cases where we wait for a response on the link after we sent a request 357 */ 358 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 359 struct smc_link *lnk, 360 int time_out, u8 exp_msg) 361 { 362 struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 363 u8 rcv_msg; 364 365 wait_event_timeout(lgr->llc_msg_waiter, 366 (flow->qentry || 367 (lnk && !smc_link_usable(lnk)) || 368 list_empty(&lgr->list)), 369 time_out); 370 if (!flow->qentry || 371 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 372 smc_llc_flow_qentry_del(flow); 373 goto out; 374 } 375 rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type; 376 if (exp_msg && rcv_msg != exp_msg) { 377 if (exp_msg == SMC_LLC_ADD_LINK && 378 rcv_msg == SMC_LLC_DELETE_LINK) { 379 /* flow_start will delay the unexpected msg */ 380 smc_llc_flow_start(&lgr->llc_flow_lcl, 381 smc_llc_flow_qentry_clr(flow)); 382 return NULL; 383 } 384 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: " 385 "msg %d exp %d flow %d role %d flags %x\n", 386 SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie, 387 rcv_msg, exp_msg, 388 flow->type, lgr->role, 389 flow->qentry->msg.raw.hdr.flags); 390 smc_llc_flow_qentry_del(flow); 391 } 392 out: 393 return flow->qentry; 394 } 395 396 /********************************** send *************************************/ 397 398 struct smc_llc_tx_pend { 399 }; 400 401 /* handler for send/transmission completion of an LLC msg */ 402 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 403 struct smc_link *link, 404 enum ib_wc_status wc_status) 405 { 406 /* future work: handle wc_status error for recovery and failover */ 407 } 408 409 /** 410 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 411 * @link: Pointer to SMC link used for sending LLC control message. 412 * @wr_buf: Out variable returning pointer to work request payload buffer. 413 * @pend: Out variable returning pointer to private pending WR tracking. 414 * It's the context the transmit complete handler will get. 415 * 416 * Reserves and pre-fills an entry for a pending work request send/tx. 417 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 418 * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 419 * 420 * Return: 0 on success, otherwise an error value. 421 */ 422 static int smc_llc_add_pending_send(struct smc_link *link, 423 struct smc_wr_buf **wr_buf, 424 struct smc_wr_tx_pend_priv **pend) 425 { 426 int rc; 427 428 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 429 pend); 430 if (rc < 0) 431 return rc; 432 BUILD_BUG_ON_MSG( 433 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 434 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 435 BUILD_BUG_ON_MSG( 436 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 437 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 438 BUILD_BUG_ON_MSG( 439 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 440 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 441 return 0; 442 } 443 444 static int smc_llc_add_pending_send_v2(struct smc_link *link, 445 struct smc_wr_v2_buf **wr_buf, 446 struct smc_wr_tx_pend_priv **pend) 447 { 448 int rc; 449 450 rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend); 451 if (rc < 0) 452 return rc; 453 return 0; 454 } 455 456 static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr, 457 struct smc_link_group *lgr, size_t len) 458 { 459 if (lgr->smc_version == SMC_V2) { 460 hdr->common.llc_version = SMC_V2; 461 hdr->length_v2 = len; 462 } else { 463 hdr->common.llc_version = 0; 464 hdr->length = len; 465 } 466 } 467 468 /* high-level API to send LLC confirm link */ 469 int smc_llc_send_confirm_link(struct smc_link *link, 470 enum smc_llc_reqresp reqresp) 471 { 472 struct smc_llc_msg_confirm_link *confllc; 473 struct smc_wr_tx_pend_priv *pend; 474 struct smc_wr_buf *wr_buf; 475 int rc; 476 477 if (!smc_wr_tx_link_hold(link)) 478 return -ENOLINK; 479 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 480 if (rc) 481 goto put_out; 482 confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 483 memset(confllc, 0, sizeof(*confllc)); 484 confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK; 485 smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc)); 486 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 487 if (reqresp == SMC_LLC_RESP) 488 confllc->hd.flags |= SMC_LLC_FLAG_RESP; 489 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 490 ETH_ALEN); 491 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 492 hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 493 confllc->link_num = link->link_id; 494 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 495 confllc->max_links = SMC_LINKS_ADD_LNK_MAX; 496 if (link->lgr->smc_version == SMC_V2 && 497 link->lgr->peer_smc_release >= SMC_RELEASE_1) { 498 confllc->max_conns = link->lgr->max_conns; 499 confllc->max_links = link->lgr->max_links; 500 } 501 /* send llc message */ 502 rc = smc_wr_tx_send(link, pend); 503 put_out: 504 smc_wr_tx_link_put(link); 505 return rc; 506 } 507 508 /* send LLC confirm rkey request */ 509 static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 510 struct smc_buf_desc *rmb_desc) 511 { 512 struct smc_llc_msg_confirm_rkey *rkeyllc; 513 struct smc_wr_tx_pend_priv *pend; 514 struct smc_wr_buf *wr_buf; 515 struct smc_link *link; 516 int i, rc, rtok_ix; 517 518 if (!smc_wr_tx_link_hold(send_link)) 519 return -ENOLINK; 520 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 521 if (rc) 522 goto put_out; 523 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 524 memset(rkeyllc, 0, sizeof(*rkeyllc)); 525 rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY; 526 smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc)); 527 528 rtok_ix = 1; 529 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 530 link = &send_link->lgr->lnk[i]; 531 if (smc_link_active(link) && link != send_link) { 532 rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 533 rkeyllc->rtoken[rtok_ix].rmb_key = 534 htonl(rmb_desc->mr[link->link_idx]->rkey); 535 rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ? 536 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) : 537 cpu_to_be64((u64)sg_dma_address 538 (rmb_desc->sgt[link->link_idx].sgl)); 539 rtok_ix++; 540 } 541 } 542 /* rkey of send_link is in rtoken[0] */ 543 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 544 rkeyllc->rtoken[0].rmb_key = 545 htonl(rmb_desc->mr[send_link->link_idx]->rkey); 546 rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ? 547 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) : 548 cpu_to_be64((u64)sg_dma_address 549 (rmb_desc->sgt[send_link->link_idx].sgl)); 550 /* send llc message */ 551 rc = smc_wr_tx_send(send_link, pend); 552 put_out: 553 smc_wr_tx_link_put(send_link); 554 return rc; 555 } 556 557 /* send LLC delete rkey request */ 558 static int smc_llc_send_delete_rkey(struct smc_link *link, 559 struct smc_buf_desc *rmb_desc) 560 { 561 struct smc_llc_msg_delete_rkey *rkeyllc; 562 struct smc_wr_tx_pend_priv *pend; 563 struct smc_wr_buf *wr_buf; 564 int rc; 565 566 if (!smc_wr_tx_link_hold(link)) 567 return -ENOLINK; 568 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 569 if (rc) 570 goto put_out; 571 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 572 memset(rkeyllc, 0, sizeof(*rkeyllc)); 573 rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY; 574 smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc)); 575 rkeyllc->num_rkeys = 1; 576 rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey); 577 /* send llc message */ 578 rc = smc_wr_tx_send(link, pend); 579 put_out: 580 smc_wr_tx_link_put(link); 581 return rc; 582 } 583 584 /* return first buffer from any of the next buf lists */ 585 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 586 int *buf_lst) 587 { 588 struct smc_buf_desc *buf_pos; 589 590 while (*buf_lst < SMC_RMBE_SIZES) { 591 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 592 struct smc_buf_desc, list); 593 if (buf_pos) 594 return buf_pos; 595 (*buf_lst)++; 596 } 597 return NULL; 598 } 599 600 /* return next rmb from buffer lists */ 601 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 602 int *buf_lst, 603 struct smc_buf_desc *buf_pos) 604 { 605 struct smc_buf_desc *buf_next; 606 607 if (!buf_pos) 608 return _smc_llc_get_next_rmb(lgr, buf_lst); 609 610 if (list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 611 (*buf_lst)++; 612 return _smc_llc_get_next_rmb(lgr, buf_lst); 613 } 614 buf_next = list_next_entry(buf_pos, list); 615 return buf_next; 616 } 617 618 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 619 int *buf_lst) 620 { 621 *buf_lst = 0; 622 return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 623 } 624 625 static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext, 626 struct smc_link *link, struct smc_link *link_new) 627 { 628 struct smc_link_group *lgr = link->lgr; 629 struct smc_buf_desc *buf_pos; 630 int prim_lnk_idx, lnk_idx, i; 631 struct smc_buf_desc *rmb; 632 int len = sizeof(*ext); 633 int buf_lst; 634 635 ext->v2_direct = !lgr->uses_gateway; 636 memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE); 637 638 prim_lnk_idx = link->link_idx; 639 lnk_idx = link_new->link_idx; 640 down_write(&lgr->rmbs_lock); 641 ext->num_rkeys = lgr->conns_num; 642 if (!ext->num_rkeys) 643 goto out; 644 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 645 for (i = 0; i < ext->num_rkeys; i++) { 646 while (buf_pos && !(buf_pos)->used) 647 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos); 648 if (!buf_pos) 649 break; 650 rmb = buf_pos; 651 ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey); 652 ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey); 653 ext->rt[i].rmb_vaddr_new = rmb->is_vm ? 654 cpu_to_be64((uintptr_t)rmb->cpu_addr) : 655 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 656 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos); 657 } 658 len += i * sizeof(ext->rt[0]); 659 out: 660 up_write(&lgr->rmbs_lock); 661 return len; 662 } 663 664 /* send ADD LINK request or response */ 665 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 666 struct smc_link *link_new, 667 enum smc_llc_reqresp reqresp) 668 { 669 struct smc_llc_msg_add_link_v2_ext *ext = NULL; 670 struct smc_llc_msg_add_link *addllc; 671 struct smc_wr_tx_pend_priv *pend; 672 int len = sizeof(*addllc); 673 int rc; 674 675 if (!smc_wr_tx_link_hold(link)) 676 return -ENOLINK; 677 if (link->lgr->smc_version == SMC_V2) { 678 struct smc_wr_v2_buf *wr_buf; 679 680 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend); 681 if (rc) 682 goto put_out; 683 addllc = (struct smc_llc_msg_add_link *)wr_buf; 684 ext = (struct smc_llc_msg_add_link_v2_ext *) 685 &wr_buf->raw[sizeof(*addllc)]; 686 memset(ext, 0, SMC_WR_TX_SIZE); 687 } else { 688 struct smc_wr_buf *wr_buf; 689 690 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 691 if (rc) 692 goto put_out; 693 addllc = (struct smc_llc_msg_add_link *)wr_buf; 694 } 695 696 memset(addllc, 0, sizeof(*addllc)); 697 addllc->hd.common.llc_type = SMC_LLC_ADD_LINK; 698 if (reqresp == SMC_LLC_RESP) 699 addllc->hd.flags |= SMC_LLC_FLAG_RESP; 700 memcpy(addllc->sender_mac, mac, ETH_ALEN); 701 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 702 if (link_new) { 703 addllc->link_num = link_new->link_id; 704 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 705 hton24(addllc->initial_psn, link_new->psn_initial); 706 if (reqresp == SMC_LLC_REQ) 707 addllc->qp_mtu = link_new->path_mtu; 708 else 709 addllc->qp_mtu = min(link_new->path_mtu, 710 link_new->peer_mtu); 711 } 712 if (ext && link_new) 713 len += smc_llc_fill_ext_v2(ext, link, link_new); 714 smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len); 715 /* send llc message */ 716 if (link->lgr->smc_version == SMC_V2) 717 rc = smc_wr_tx_v2_send(link, pend, len); 718 else 719 rc = smc_wr_tx_send(link, pend); 720 put_out: 721 smc_wr_tx_link_put(link); 722 return rc; 723 } 724 725 /* send DELETE LINK request or response */ 726 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 727 enum smc_llc_reqresp reqresp, bool orderly, 728 u32 reason) 729 { 730 struct smc_llc_msg_del_link *delllc; 731 struct smc_wr_tx_pend_priv *pend; 732 struct smc_wr_buf *wr_buf; 733 int rc; 734 735 if (!smc_wr_tx_link_hold(link)) 736 return -ENOLINK; 737 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 738 if (rc) 739 goto put_out; 740 delllc = (struct smc_llc_msg_del_link *)wr_buf; 741 742 memset(delllc, 0, sizeof(*delllc)); 743 delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK; 744 smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc)); 745 if (reqresp == SMC_LLC_RESP) 746 delllc->hd.flags |= SMC_LLC_FLAG_RESP; 747 if (orderly) 748 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 749 if (link_del_id) 750 delllc->link_num = link_del_id; 751 else 752 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 753 delllc->reason = htonl(reason); 754 /* send llc message */ 755 rc = smc_wr_tx_send(link, pend); 756 put_out: 757 smc_wr_tx_link_put(link); 758 return rc; 759 } 760 761 /* send LLC test link request */ 762 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 763 { 764 struct smc_llc_msg_test_link *testllc; 765 struct smc_wr_tx_pend_priv *pend; 766 struct smc_wr_buf *wr_buf; 767 int rc; 768 769 if (!smc_wr_tx_link_hold(link)) 770 return -ENOLINK; 771 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 772 if (rc) 773 goto put_out; 774 testllc = (struct smc_llc_msg_test_link *)wr_buf; 775 memset(testllc, 0, sizeof(*testllc)); 776 testllc->hd.common.llc_type = SMC_LLC_TEST_LINK; 777 smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc)); 778 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 779 /* send llc message */ 780 rc = smc_wr_tx_send(link, pend); 781 put_out: 782 smc_wr_tx_link_put(link); 783 return rc; 784 } 785 786 /* schedule an llc send on link, may wait for buffers */ 787 static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 788 { 789 struct smc_wr_tx_pend_priv *pend; 790 struct smc_wr_buf *wr_buf; 791 int rc; 792 793 if (!smc_wr_tx_link_hold(link)) 794 return -ENOLINK; 795 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 796 if (rc) 797 goto put_out; 798 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 799 rc = smc_wr_tx_send(link, pend); 800 put_out: 801 smc_wr_tx_link_put(link); 802 return rc; 803 } 804 805 /* schedule an llc send on link, may wait for buffers, 806 * and wait for send completion notification. 807 * @return 0 on success 808 */ 809 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 810 { 811 struct smc_wr_tx_pend_priv *pend; 812 struct smc_wr_buf *wr_buf; 813 int rc; 814 815 if (!smc_wr_tx_link_hold(link)) 816 return -ENOLINK; 817 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 818 if (rc) 819 goto put_out; 820 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 821 rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 822 put_out: 823 smc_wr_tx_link_put(link); 824 return rc; 825 } 826 827 /********************************* receive ***********************************/ 828 829 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 830 enum smc_lgr_type lgr_new_t) 831 { 832 int i; 833 834 if (lgr->type == SMC_LGR_SYMMETRIC || 835 (lgr->type != SMC_LGR_SINGLE && 836 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 837 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 838 return -EMLINK; 839 840 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 841 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 842 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 843 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 844 return i; 845 } else { 846 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 847 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 848 return i; 849 } 850 return -EMLINK; 851 } 852 853 /* send one add_link_continue msg */ 854 static int smc_llc_add_link_cont(struct smc_link *link, 855 struct smc_link *link_new, u8 *num_rkeys_todo, 856 int *buf_lst, struct smc_buf_desc **buf_pos) 857 { 858 struct smc_llc_msg_add_link_cont *addc_llc; 859 struct smc_link_group *lgr = link->lgr; 860 int prim_lnk_idx, lnk_idx, i, rc; 861 struct smc_wr_tx_pend_priv *pend; 862 struct smc_wr_buf *wr_buf; 863 struct smc_buf_desc *rmb; 864 u8 n; 865 866 if (!smc_wr_tx_link_hold(link)) 867 return -ENOLINK; 868 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 869 if (rc) 870 goto put_out; 871 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 872 memset(addc_llc, 0, sizeof(*addc_llc)); 873 874 prim_lnk_idx = link->link_idx; 875 lnk_idx = link_new->link_idx; 876 addc_llc->link_num = link_new->link_id; 877 addc_llc->num_rkeys = *num_rkeys_todo; 878 n = *num_rkeys_todo; 879 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 880 while (*buf_pos && !(*buf_pos)->used) 881 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 882 if (!*buf_pos) { 883 addc_llc->num_rkeys = addc_llc->num_rkeys - 884 *num_rkeys_todo; 885 *num_rkeys_todo = 0; 886 break; 887 } 888 rmb = *buf_pos; 889 890 addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey); 891 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey); 892 addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ? 893 cpu_to_be64((uintptr_t)rmb->cpu_addr) : 894 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 895 896 (*num_rkeys_todo)--; 897 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 898 } 899 addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT; 900 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 901 if (lgr->role == SMC_CLNT) 902 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 903 rc = smc_wr_tx_send(link, pend); 904 put_out: 905 smc_wr_tx_link_put(link); 906 return rc; 907 } 908 909 static int smc_llc_cli_rkey_exchange(struct smc_link *link, 910 struct smc_link *link_new) 911 { 912 struct smc_llc_msg_add_link_cont *addc_llc; 913 struct smc_link_group *lgr = link->lgr; 914 u8 max, num_rkeys_send, num_rkeys_recv; 915 struct smc_llc_qentry *qentry; 916 struct smc_buf_desc *buf_pos; 917 int buf_lst; 918 int rc = 0; 919 int i; 920 921 down_write(&lgr->rmbs_lock); 922 num_rkeys_send = lgr->conns_num; 923 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 924 do { 925 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 926 SMC_LLC_ADD_LINK_CONT); 927 if (!qentry) { 928 rc = -ETIMEDOUT; 929 break; 930 } 931 addc_llc = &qentry->msg.add_link_cont; 932 num_rkeys_recv = addc_llc->num_rkeys; 933 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 934 for (i = 0; i < max; i++) { 935 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 936 addc_llc->rt[i].rmb_key, 937 addc_llc->rt[i].rmb_vaddr_new, 938 addc_llc->rt[i].rmb_key_new); 939 num_rkeys_recv--; 940 } 941 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 942 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 943 &buf_lst, &buf_pos); 944 if (rc) 945 break; 946 } while (num_rkeys_send || num_rkeys_recv); 947 948 up_write(&lgr->rmbs_lock); 949 return rc; 950 } 951 952 /* prepare and send an add link reject response */ 953 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 954 { 955 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 956 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 957 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 958 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr, 959 sizeof(qentry->msg)); 960 return smc_llc_send_message(qentry->link, &qentry->msg); 961 } 962 963 static int smc_llc_cli_conf_link(struct smc_link *link, 964 struct smc_init_info *ini, 965 struct smc_link *link_new, 966 enum smc_lgr_type lgr_new_t) 967 { 968 struct smc_link_group *lgr = link->lgr; 969 struct smc_llc_qentry *qentry = NULL; 970 int rc = 0; 971 972 /* receive CONFIRM LINK request over RoCE fabric */ 973 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 974 if (!qentry) { 975 rc = smc_llc_send_delete_link(link, link_new->link_id, 976 SMC_LLC_REQ, false, 977 SMC_LLC_DEL_LOST_PATH); 978 return -ENOLINK; 979 } 980 if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) { 981 /* received DELETE_LINK instead */ 982 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 983 smc_llc_send_message(link, &qentry->msg); 984 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 985 return -ENOLINK; 986 } 987 smc_llc_save_peer_uid(qentry); 988 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 989 990 rc = smc_ib_modify_qp_rts(link_new); 991 if (rc) { 992 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 993 false, SMC_LLC_DEL_LOST_PATH); 994 return -ENOLINK; 995 } 996 smc_wr_remember_qp_attr(link_new); 997 998 rc = smcr_buf_reg_lgr(link_new); 999 if (rc) { 1000 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 1001 false, SMC_LLC_DEL_LOST_PATH); 1002 return -ENOLINK; 1003 } 1004 1005 /* send CONFIRM LINK response over RoCE fabric */ 1006 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 1007 if (rc) { 1008 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 1009 false, SMC_LLC_DEL_LOST_PATH); 1010 return -ENOLINK; 1011 } 1012 smc_llc_link_active(link_new); 1013 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1014 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1015 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1016 else 1017 smcr_lgr_set_type(lgr, lgr_new_t); 1018 return 0; 1019 } 1020 1021 static void smc_llc_save_add_link_rkeys(struct smc_link *link, 1022 struct smc_link *link_new, 1023 struct smc_llc_qentry *qentry) 1024 { 1025 const u32 rt_off = offsetof(struct smc_llc_msg_add_link_v2_ext, rt); 1026 struct smc_llc_msg_add_link_v2_ext *ext; 1027 struct smc_link_group *lgr = link->lgr; 1028 int max, i; 1029 1030 /* the rkey count itself is only there if enough bytes arrived */ 1031 if (qentry->body_len < rt_off) 1032 return; 1033 ext = (struct smc_llc_msg_add_link_v2_ext *)qentry->body; 1034 max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); 1035 max = min_t(u32, max, (qentry->body_len - rt_off) / 1036 sizeof(ext->rt[0])); 1037 down_write(&lgr->rmbs_lock); 1038 for (i = 0; i < max; i++) { 1039 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1040 ext->rt[i].rmb_key, 1041 ext->rt[i].rmb_vaddr_new, 1042 ext->rt[i].rmb_key_new); 1043 } 1044 up_write(&lgr->rmbs_lock); 1045 } 1046 1047 static void smc_llc_save_add_link_info(struct smc_link *link, 1048 struct smc_llc_msg_add_link *add_llc) 1049 { 1050 link->peer_qpn = ntoh24(add_llc->sender_qp_num); 1051 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 1052 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 1053 link->peer_psn = ntoh24(add_llc->initial_psn); 1054 link->peer_mtu = add_llc->qp_mtu; 1055 } 1056 1057 /* as an SMC client, process an add link request */ 1058 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 1059 { 1060 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 1061 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1062 struct smc_link_group *lgr = smc_get_lgr(link); 1063 struct smc_init_info *ini = NULL; 1064 struct smc_link *lnk_new = NULL; 1065 int lnk_idx, rc = 0; 1066 1067 if (!llc->qp_mtu) 1068 goto out_reject; 1069 1070 ini = kzalloc_obj(*ini); 1071 if (!ini) { 1072 rc = -ENOMEM; 1073 goto out_reject; 1074 } 1075 1076 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) { 1077 rc = 0; 1078 goto out_reject; 1079 } 1080 1081 ini->vlan_id = lgr->vlan_id; 1082 if (lgr->smc_version == SMC_V2) { 1083 ini->check_smcrv2 = true; 1084 ini->smcrv2.saddr = lgr->saddr; 1085 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid); 1086 } 1087 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1088 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1089 (lgr->smc_version == SMC_V2 || 1090 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) { 1091 if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2) 1092 goto out_reject; 1093 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1094 } 1095 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) { 1096 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1097 ini->smcrv2.ib_dev_v2 = link->smcibdev; 1098 ini->smcrv2.ib_port_v2 = link->ibport; 1099 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) { 1100 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1101 ini->ib_dev = link->smcibdev; 1102 ini->ib_port = link->ibport; 1103 } 1104 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1105 if (lnk_idx < 0) 1106 goto out_reject; 1107 lnk_new = &lgr->lnk[lnk_idx]; 1108 rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini); 1109 if (rc) 1110 goto out_reject; 1111 smc_llc_save_add_link_info(lnk_new, llc); 1112 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 1113 smc_llc_link_set_uid(lnk_new); 1114 1115 rc = smc_ib_ready_link(lnk_new); 1116 if (rc) 1117 goto out_clear_lnk; 1118 1119 rc = smcr_buf_map_lgr(lnk_new); 1120 if (rc) 1121 goto out_clear_lnk; 1122 1123 rc = smc_llc_send_add_link(link, 1124 lnk_new->smcibdev->mac[lnk_new->ibport - 1], 1125 lnk_new->gid, lnk_new, SMC_LLC_RESP); 1126 if (rc) 1127 goto out_clear_lnk; 1128 if (lgr->smc_version == SMC_V2) { 1129 smc_llc_save_add_link_rkeys(link, lnk_new, qentry); 1130 } else { 1131 rc = smc_llc_cli_rkey_exchange(link, lnk_new); 1132 if (rc) { 1133 rc = 0; 1134 goto out_clear_lnk; 1135 } 1136 } 1137 rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t); 1138 if (!rc) 1139 goto out; 1140 out_clear_lnk: 1141 lnk_new->state = SMC_LNK_INACTIVE; 1142 smcr_link_clear(lnk_new, false); 1143 out_reject: 1144 smc_llc_cli_add_link_reject(qentry); 1145 out: 1146 kfree(ini); 1147 kfree(qentry); 1148 return rc; 1149 } 1150 1151 static void smc_llc_send_request_add_link(struct smc_link *link) 1152 { 1153 struct smc_llc_msg_req_add_link_v2 *llc; 1154 struct smc_wr_tx_pend_priv *pend; 1155 struct smc_wr_v2_buf *wr_buf; 1156 struct smc_gidlist gidlist; 1157 int rc, len, i; 1158 1159 if (!smc_wr_tx_link_hold(link)) 1160 return; 1161 if (link->lgr->type == SMC_LGR_SYMMETRIC || 1162 link->lgr->type == SMC_LGR_ASYMMETRIC_PEER) 1163 goto put_out; 1164 1165 smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid); 1166 if (gidlist.len <= 1) 1167 goto put_out; 1168 1169 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend); 1170 if (rc) 1171 goto put_out; 1172 llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf; 1173 memset(llc, 0, SMC_WR_TX_SIZE); 1174 1175 llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK; 1176 for (i = 0; i < gidlist.len; i++) 1177 memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0])); 1178 llc->gid_cnt = gidlist.len; 1179 len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0])); 1180 smc_llc_init_msg_hdr(&llc->hd, link->lgr, len); 1181 rc = smc_wr_tx_v2_send(link, pend, len); 1182 if (!rc) 1183 /* set REQ_ADD_LINK flow and wait for response from peer */ 1184 link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK; 1185 put_out: 1186 smc_wr_tx_link_put(link); 1187 } 1188 1189 /* as an SMC client, invite server to start the add_link processing */ 1190 static void smc_llc_cli_add_link_invite(struct smc_link *link, 1191 struct smc_llc_qentry *qentry) 1192 { 1193 struct smc_link_group *lgr = smc_get_lgr(link); 1194 struct smc_init_info *ini = NULL; 1195 1196 if (lgr->smc_version == SMC_V2) { 1197 smc_llc_send_request_add_link(link); 1198 goto out; 1199 } 1200 1201 if (lgr->type == SMC_LGR_SYMMETRIC || 1202 lgr->type == SMC_LGR_ASYMMETRIC_PEER) 1203 goto out; 1204 1205 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) 1206 goto out; 1207 1208 ini = kzalloc_obj(*ini); 1209 if (!ini) 1210 goto out; 1211 1212 ini->vlan_id = lgr->vlan_id; 1213 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1214 if (!ini->ib_dev) 1215 goto out; 1216 1217 smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1], 1218 ini->ib_gid, NULL, SMC_LLC_REQ); 1219 out: 1220 kfree(ini); 1221 kfree(qentry); 1222 } 1223 1224 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc) 1225 { 1226 int i; 1227 1228 for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++) 1229 if (llc->raw.data[i]) 1230 return false; 1231 return true; 1232 } 1233 1234 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc) 1235 { 1236 if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK && 1237 smc_llc_is_empty_llc_message(llc)) 1238 return true; 1239 return false; 1240 } 1241 1242 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 1243 { 1244 struct smc_llc_qentry *qentry; 1245 1246 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1247 1248 down_write(&lgr->llc_conf_mutex); 1249 if (smc_llc_is_local_add_link(&qentry->msg)) 1250 smc_llc_cli_add_link_invite(qentry->link, qentry); 1251 else 1252 smc_llc_cli_add_link(qentry->link, qentry); 1253 up_write(&lgr->llc_conf_mutex); 1254 } 1255 1256 static int smc_llc_active_link_count(struct smc_link_group *lgr) 1257 { 1258 int i, link_count = 0; 1259 1260 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1261 if (!smc_link_active(&lgr->lnk[i])) 1262 continue; 1263 link_count++; 1264 } 1265 return link_count; 1266 } 1267 1268 /* find the asymmetric link when 3 links are established */ 1269 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 1270 { 1271 int asym_idx = -ENOENT; 1272 int i, j, k; 1273 bool found; 1274 1275 /* determine asymmetric link */ 1276 found = false; 1277 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1278 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 1279 if (!smc_link_usable(&lgr->lnk[i]) || 1280 !smc_link_usable(&lgr->lnk[j])) 1281 continue; 1282 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 1283 SMC_GID_SIZE)) { 1284 found = true; /* asym_lnk is i or j */ 1285 break; 1286 } 1287 } 1288 if (found) 1289 break; 1290 } 1291 if (!found) 1292 goto out; /* no asymmetric link */ 1293 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 1294 if (!smc_link_usable(&lgr->lnk[k])) 1295 continue; 1296 if (k != i && 1297 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 1298 SMC_GID_SIZE)) { 1299 asym_idx = i; 1300 break; 1301 } 1302 if (k != j && 1303 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 1304 SMC_GID_SIZE)) { 1305 asym_idx = j; 1306 break; 1307 } 1308 } 1309 out: 1310 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 1311 } 1312 1313 static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 1314 { 1315 struct smc_link *lnk_new = NULL, *lnk_asym; 1316 struct smc_llc_qentry *qentry; 1317 int rc; 1318 1319 lnk_asym = smc_llc_find_asym_link(lgr); 1320 if (!lnk_asym) 1321 return; /* no asymmetric link */ 1322 if (!smc_link_downing(&lnk_asym->state)) 1323 return; 1324 lnk_new = smc_switch_conns(lgr, lnk_asym, false); 1325 smc_wr_tx_wait_no_pending_sends(lnk_asym); 1326 if (!lnk_new) 1327 goto out_free; 1328 /* change flow type from ADD_LINK into DEL_LINK */ 1329 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 1330 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 1331 true, SMC_LLC_DEL_NO_ASYM_NEEDED); 1332 if (rc) { 1333 smcr_link_down_cond(lnk_new); 1334 goto out_free; 1335 } 1336 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 1337 SMC_LLC_DELETE_LINK); 1338 if (!qentry) { 1339 smcr_link_down_cond(lnk_new); 1340 goto out_free; 1341 } 1342 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1343 out_free: 1344 smcr_link_clear(lnk_asym, true); 1345 } 1346 1347 static int smc_llc_srv_rkey_exchange(struct smc_link *link, 1348 struct smc_link *link_new) 1349 { 1350 struct smc_llc_msg_add_link_cont *addc_llc; 1351 struct smc_link_group *lgr = link->lgr; 1352 u8 max, num_rkeys_send, num_rkeys_recv; 1353 struct smc_llc_qentry *qentry = NULL; 1354 struct smc_buf_desc *buf_pos; 1355 int buf_lst; 1356 int rc = 0; 1357 int i; 1358 1359 down_write(&lgr->rmbs_lock); 1360 num_rkeys_send = lgr->conns_num; 1361 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 1362 do { 1363 smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 1364 &buf_lst, &buf_pos); 1365 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 1366 SMC_LLC_ADD_LINK_CONT); 1367 if (!qentry) { 1368 rc = -ETIMEDOUT; 1369 goto out; 1370 } 1371 addc_llc = &qentry->msg.add_link_cont; 1372 num_rkeys_recv = addc_llc->num_rkeys; 1373 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 1374 for (i = 0; i < max; i++) { 1375 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1376 addc_llc->rt[i].rmb_key, 1377 addc_llc->rt[i].rmb_vaddr_new, 1378 addc_llc->rt[i].rmb_key_new); 1379 num_rkeys_recv--; 1380 } 1381 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1382 } while (num_rkeys_send || num_rkeys_recv); 1383 out: 1384 up_write(&lgr->rmbs_lock); 1385 return rc; 1386 } 1387 1388 static int smc_llc_srv_conf_link(struct smc_link *link, 1389 struct smc_link *link_new, 1390 enum smc_lgr_type lgr_new_t) 1391 { 1392 struct smc_link_group *lgr = link->lgr; 1393 struct smc_llc_qentry *qentry = NULL; 1394 int rc; 1395 1396 /* send CONFIRM LINK request over the RoCE fabric */ 1397 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 1398 if (rc) 1399 return -ENOLINK; 1400 /* receive CONFIRM LINK response over the RoCE fabric */ 1401 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0); 1402 if (!qentry || 1403 qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) { 1404 /* send DELETE LINK */ 1405 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 1406 false, SMC_LLC_DEL_LOST_PATH); 1407 if (qentry) 1408 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1409 return -ENOLINK; 1410 } 1411 smc_llc_save_peer_uid(qentry); 1412 smc_llc_link_active(link_new); 1413 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1414 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1415 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1416 else 1417 smcr_lgr_set_type(lgr, lgr_new_t); 1418 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1419 return 0; 1420 } 1421 1422 static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry) 1423 { 1424 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 1425 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr, 1426 sizeof(qentry->msg)); 1427 memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data)); 1428 smc_llc_send_message(qentry->link, &qentry->msg); 1429 } 1430 1431 int smc_llc_srv_add_link(struct smc_link *link, 1432 struct smc_llc_qentry *req_qentry) 1433 { 1434 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1435 struct smc_link_group *lgr = link->lgr; 1436 struct smc_llc_msg_add_link *add_llc; 1437 struct smc_llc_qentry *qentry = NULL; 1438 bool send_req_add_link_resp = false; 1439 struct smc_link *link_new = NULL; 1440 struct smc_init_info *ini = NULL; 1441 int lnk_idx, rc = 0; 1442 1443 if (req_qentry && 1444 req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK) 1445 send_req_add_link_resp = true; 1446 1447 ini = kzalloc_obj(*ini); 1448 if (!ini) { 1449 rc = -ENOMEM; 1450 goto out; 1451 } 1452 1453 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) { 1454 rc = 0; 1455 goto out; 1456 } 1457 1458 /* ignore client add link recommendation, start new flow */ 1459 ini->vlan_id = lgr->vlan_id; 1460 if (lgr->smc_version == SMC_V2) { 1461 ini->check_smcrv2 = true; 1462 ini->smcrv2.saddr = lgr->saddr; 1463 if (send_req_add_link_resp) { 1464 struct smc_llc_msg_req_add_link_v2 *req_add = 1465 &req_qentry->msg.req_add_link; 1466 1467 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]); 1468 } 1469 } 1470 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1471 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) { 1472 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1473 ini->smcrv2.ib_dev_v2 = link->smcibdev; 1474 ini->smcrv2.ib_port_v2 = link->ibport; 1475 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) { 1476 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1477 ini->ib_dev = link->smcibdev; 1478 ini->ib_port = link->ibport; 1479 } 1480 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1481 if (lnk_idx < 0) { 1482 rc = 0; 1483 goto out; 1484 } 1485 1486 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini); 1487 if (rc) 1488 goto out; 1489 link_new = &lgr->lnk[lnk_idx]; 1490 1491 rc = smcr_buf_map_lgr(link_new); 1492 if (rc) 1493 goto out_err; 1494 1495 rc = smc_llc_send_add_link(link, 1496 link_new->smcibdev->mac[link_new->ibport-1], 1497 link_new->gid, link_new, SMC_LLC_REQ); 1498 if (rc) 1499 goto out_err; 1500 send_req_add_link_resp = false; 1501 /* receive ADD LINK response over the RoCE fabric */ 1502 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 1503 if (!qentry) { 1504 rc = -ETIMEDOUT; 1505 goto out_err; 1506 } 1507 add_llc = &qentry->msg.add_link; 1508 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 1509 smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1510 rc = -ENOLINK; 1511 goto out_err; 1512 } 1513 if (lgr->type == SMC_LGR_SINGLE && 1514 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1515 (lgr->smc_version == SMC_V2 || 1516 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) { 1517 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1518 } 1519 smc_llc_save_add_link_info(link_new, add_llc); 1520 /* add_llc still points into qentry, so only detach it here */ 1521 smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1522 1523 rc = smc_ib_ready_link(link_new); 1524 if (rc) 1525 goto out_err; 1526 rc = smcr_buf_reg_lgr(link_new); 1527 if (rc) 1528 goto out_err; 1529 if (lgr->smc_version == SMC_V2) { 1530 smc_llc_save_add_link_rkeys(link, link_new, qentry); 1531 } else { 1532 rc = smc_llc_srv_rkey_exchange(link, link_new); 1533 if (rc) 1534 goto out_err; 1535 } 1536 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 1537 if (rc) 1538 goto out_err; 1539 goto out; 1540 out_err: 1541 if (link_new) { 1542 link_new->state = SMC_LNK_INACTIVE; 1543 smcr_link_clear(link_new, false); 1544 } 1545 out: 1546 kfree(qentry); 1547 kfree(ini); 1548 if (send_req_add_link_resp) 1549 smc_llc_send_req_add_link_response(req_qentry); 1550 return rc; 1551 } 1552 1553 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 1554 { 1555 struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 1556 struct smc_llc_qentry *qentry; 1557 int rc; 1558 1559 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1560 1561 down_write(&lgr->llc_conf_mutex); 1562 rc = smc_llc_srv_add_link(link, qentry); 1563 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 1564 /* delete any asymmetric link */ 1565 smc_llc_delete_asym_link(lgr); 1566 } 1567 up_write(&lgr->llc_conf_mutex); 1568 kfree(qentry); 1569 } 1570 1571 /* enqueue a local add_link req to trigger a new add_link flow */ 1572 void smc_llc_add_link_local(struct smc_link *link) 1573 { 1574 struct smc_llc_msg_add_link add_llc = {}; 1575 1576 add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK; 1577 smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc)); 1578 /* no dev and port needed */ 1579 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc, 1580 sizeof(union smc_llc_msg)); 1581 } 1582 1583 /* worker to process an add link message */ 1584 static void smc_llc_add_link_work(struct work_struct *work) 1585 { 1586 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1587 llc_add_link_work); 1588 1589 if (list_empty(&lgr->list)) { 1590 /* link group is terminating */ 1591 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1592 goto out; 1593 } 1594 1595 if (lgr->role == SMC_CLNT) 1596 smc_llc_process_cli_add_link(lgr); 1597 else 1598 smc_llc_process_srv_add_link(lgr); 1599 out: 1600 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK) 1601 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1602 } 1603 1604 /* enqueue a local del_link msg to trigger a new del_link flow, 1605 * called only for role SMC_SERV 1606 */ 1607 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 1608 { 1609 struct smc_llc_msg_del_link del_llc = {}; 1610 1611 del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK; 1612 smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc)); 1613 del_llc.link_num = del_link_id; 1614 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 1615 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1616 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc, 1617 sizeof(union smc_llc_msg)); 1618 } 1619 1620 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 1621 { 1622 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 1623 struct smc_llc_msg_del_link *del_llc; 1624 struct smc_llc_qentry *qentry; 1625 int active_links; 1626 int lnk_idx; 1627 1628 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1629 lnk = qentry->link; 1630 del_llc = &qentry->msg.delete_link; 1631 1632 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1633 smc_lgr_terminate_sched(lgr); 1634 goto out; 1635 } 1636 down_write(&lgr->llc_conf_mutex); 1637 /* delete single link */ 1638 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 1639 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 1640 continue; 1641 lnk_del = &lgr->lnk[lnk_idx]; 1642 break; 1643 } 1644 del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 1645 if (!lnk_del) { 1646 /* link was not found */ 1647 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 1648 smc_llc_send_message(lnk, &qentry->msg); 1649 goto out_unlock; 1650 } 1651 lnk_asym = smc_llc_find_asym_link(lgr); 1652 1653 del_llc->reason = 0; 1654 smc_llc_send_message(lnk, &qentry->msg); /* response */ 1655 1656 if (smc_link_downing(&lnk_del->state)) 1657 smc_switch_conns(lgr, lnk_del, false); 1658 smcr_link_clear(lnk_del, true); 1659 1660 active_links = smc_llc_active_link_count(lgr); 1661 if (lnk_del == lnk_asym) { 1662 /* expected deletion of asym link, don't change lgr state */ 1663 } else if (active_links == 1) { 1664 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1665 } else if (!active_links) { 1666 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1667 smc_lgr_terminate_sched(lgr); 1668 } 1669 out_unlock: 1670 up_write(&lgr->llc_conf_mutex); 1671 out: 1672 kfree(qentry); 1673 } 1674 1675 /* try to send a DELETE LINK ALL request on any active link, 1676 * waiting for send completion 1677 */ 1678 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1679 { 1680 struct smc_llc_msg_del_link delllc = {}; 1681 int i; 1682 1683 delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK; 1684 smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc)); 1685 if (ord) 1686 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1687 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1688 delllc.reason = htonl(rsn); 1689 1690 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1691 if (!smc_link_sendable(&lgr->lnk[i])) 1692 continue; 1693 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1694 break; 1695 } 1696 } 1697 1698 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 1699 { 1700 struct smc_llc_msg_del_link *del_llc; 1701 struct smc_link *lnk, *lnk_del; 1702 struct smc_llc_qentry *qentry; 1703 int active_links; 1704 int i; 1705 1706 down_write(&lgr->llc_conf_mutex); 1707 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1708 lnk = qentry->link; 1709 del_llc = &qentry->msg.delete_link; 1710 1711 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1712 /* delete entire lgr */ 1713 smc_llc_send_link_delete_all(lgr, true, ntohl( 1714 qentry->msg.delete_link.reason)); 1715 smc_lgr_terminate_sched(lgr); 1716 goto out; 1717 } 1718 /* delete single link */ 1719 lnk_del = NULL; 1720 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1721 if (lgr->lnk[i].link_id == del_llc->link_num) { 1722 lnk_del = &lgr->lnk[i]; 1723 break; 1724 } 1725 } 1726 if (!lnk_del) 1727 goto out; /* asymmetric link already deleted */ 1728 1729 if (smc_link_downing(&lnk_del->state)) { 1730 if (smc_switch_conns(lgr, lnk_del, false)) 1731 smc_wr_tx_wait_no_pending_sends(lnk_del); 1732 } 1733 if (!list_empty(&lgr->list)) { 1734 /* qentry is either a request from peer (send it back to 1735 * initiate the DELETE_LINK processing), or a locally 1736 * enqueued DELETE_LINK request (forward it) 1737 */ 1738 if (!smc_llc_send_message(lnk, &qentry->msg)) { 1739 struct smc_llc_qentry *qentry2; 1740 1741 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 1742 SMC_LLC_DELETE_LINK); 1743 if (qentry2) 1744 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1745 } 1746 } 1747 smcr_link_clear(lnk_del, true); 1748 1749 active_links = smc_llc_active_link_count(lgr); 1750 if (active_links == 1) { 1751 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1752 } else if (!active_links) { 1753 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1754 smc_lgr_terminate_sched(lgr); 1755 } 1756 1757 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 1758 /* trigger setup of asymm alt link */ 1759 smc_llc_add_link_local(lnk); 1760 } 1761 out: 1762 up_write(&lgr->llc_conf_mutex); 1763 kfree(qentry); 1764 } 1765 1766 static void smc_llc_delete_link_work(struct work_struct *work) 1767 { 1768 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1769 llc_del_link_work); 1770 1771 if (list_empty(&lgr->list)) { 1772 /* link group is terminating */ 1773 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1774 goto out; 1775 } 1776 1777 if (lgr->role == SMC_CLNT) 1778 smc_llc_process_cli_delete_link(lgr); 1779 else 1780 smc_llc_process_srv_delete_link(lgr); 1781 out: 1782 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1783 } 1784 1785 /* process a confirm_rkey request from peer, remote flow */ 1786 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 1787 { 1788 struct smc_llc_msg_confirm_rkey *llc; 1789 struct smc_llc_qentry *qentry; 1790 struct smc_link *link; 1791 int num_entries; 1792 int rk_idx; 1793 int i; 1794 1795 qentry = lgr->llc_flow_rmt.qentry; 1796 llc = &qentry->msg.confirm_rkey; 1797 link = qentry->link; 1798 1799 num_entries = llc->rtoken[0].num_rkeys; 1800 if (num_entries > SMC_LLC_RKEYS_PER_MSG) 1801 goto out_err; 1802 /* first rkey entry is for receiving link */ 1803 rk_idx = smc_rtoken_add(link, 1804 llc->rtoken[0].rmb_vaddr, 1805 llc->rtoken[0].rmb_key); 1806 if (rk_idx < 0) 1807 goto out_err; 1808 1809 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 1810 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 1811 llc->rtoken[i].rmb_vaddr, 1812 llc->rtoken[i].rmb_key); 1813 /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 1814 goto out; 1815 out_err: 1816 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1817 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 1818 out: 1819 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1820 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc)); 1821 smc_llc_send_message(link, &qentry->msg); 1822 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1823 } 1824 1825 /* process a delete_rkey request from peer, remote flow */ 1826 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 1827 { 1828 struct smc_llc_msg_delete_rkey *llc; 1829 struct smc_llc_qentry *qentry; 1830 struct smc_link *link; 1831 u8 err_mask = 0; 1832 int i, max; 1833 1834 qentry = lgr->llc_flow_rmt.qentry; 1835 llc = &qentry->msg.delete_rkey; 1836 link = qentry->link; 1837 1838 if (lgr->smc_version == SMC_V2) { 1839 struct smc_llc_msg_delete_rkey_v2_inline *llcv2; 1840 1841 /* The leading SMC_LLC_DEL_RKEY_V2_INLINE rkeys are declared in 1842 * the message itself, any further ones were received into 1843 * qentry->body. 1844 */ 1845 llcv2 = &qentry->msg.delete_rkey_v2; 1846 llcv2->num_inval_rkeys = 0; 1847 1848 max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); 1849 max = min_t(u32, max, SMC_LLC_DEL_RKEY_V2_INLINE + 1850 qentry->body_len / sizeof(__be32)); 1851 for (i = 0; i < max; i++) { 1852 __be32 rkey; 1853 1854 if (i < SMC_LLC_DEL_RKEY_V2_INLINE) 1855 rkey = llcv2->rkey[i]; 1856 else 1857 memcpy(&rkey, qentry->body + 1858 (i - SMC_LLC_DEL_RKEY_V2_INLINE) * 1859 sizeof(rkey), sizeof(rkey)); 1860 if (smc_rtoken_delete(link, rkey)) 1861 llcv2->num_inval_rkeys++; 1862 } 1863 memset(&llc->rkey[0], 0, sizeof(llc->rkey)); 1864 memset(&llc->reserved2, 0, sizeof(llc->reserved2)); 1865 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc)); 1866 if (llcv2->num_inval_rkeys) { 1867 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1868 llc->err_mask = llcv2->num_inval_rkeys; 1869 } 1870 goto finish; 1871 } 1872 1873 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 1874 for (i = 0; i < max; i++) { 1875 if (smc_rtoken_delete(link, llc->rkey[i])) 1876 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 1877 } 1878 if (err_mask) { 1879 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1880 llc->err_mask = err_mask; 1881 } 1882 finish: 1883 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1884 smc_llc_send_message(link, &qentry->msg); 1885 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1886 } 1887 1888 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 1889 { 1890 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: " 1891 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, 1892 lgr->net->net_cookie, type); 1893 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 1894 smc_lgr_terminate_sched(lgr); 1895 } 1896 1897 /* flush the llc event queue */ 1898 static void smc_llc_event_flush(struct smc_link_group *lgr) 1899 { 1900 struct smc_llc_qentry *qentry, *q; 1901 1902 spin_lock_bh(&lgr->llc_event_q_lock); 1903 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 1904 list_del_init(&qentry->list); 1905 kfree(qentry); 1906 } 1907 spin_unlock_bh(&lgr->llc_event_q_lock); 1908 } 1909 1910 static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 1911 { 1912 union smc_llc_msg *llc = &qentry->msg; 1913 struct smc_link *link = qentry->link; 1914 struct smc_link_group *lgr = link->lgr; 1915 1916 if (!smc_link_usable(link)) 1917 goto out; 1918 1919 switch (llc->raw.hdr.common.llc_type) { 1920 case SMC_LLC_TEST_LINK: 1921 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 1922 smc_llc_send_message(link, llc); 1923 break; 1924 case SMC_LLC_ADD_LINK: 1925 if (list_empty(&lgr->list)) 1926 goto out; /* lgr is terminating */ 1927 if (lgr->role == SMC_CLNT) { 1928 if (smc_llc_is_local_add_link(llc)) { 1929 if (lgr->llc_flow_lcl.type == 1930 SMC_LLC_FLOW_ADD_LINK) 1931 break; /* add_link in progress */ 1932 if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1933 qentry)) { 1934 schedule_work(&lgr->llc_add_link_work); 1935 } 1936 return; 1937 } 1938 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1939 !lgr->llc_flow_lcl.qentry) { 1940 /* a flow is waiting for this message */ 1941 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1942 qentry); 1943 wake_up(&lgr->llc_msg_waiter); 1944 return; 1945 } 1946 if (lgr->llc_flow_lcl.type == 1947 SMC_LLC_FLOW_REQ_ADD_LINK) { 1948 /* server started add_link processing */ 1949 /* free any qentry stashed in REQ_ADD_LINK state */ 1950 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1951 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; 1952 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1953 qentry); 1954 schedule_work(&lgr->llc_add_link_work); 1955 return; 1956 } 1957 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1958 schedule_work(&lgr->llc_add_link_work); 1959 } 1960 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1961 /* as smc server, handle client suggestion */ 1962 schedule_work(&lgr->llc_add_link_work); 1963 } 1964 return; 1965 case SMC_LLC_CONFIRM_LINK: 1966 case SMC_LLC_ADD_LINK_CONT: 1967 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE && 1968 !lgr->llc_flow_lcl.qentry) { 1969 /* a flow is waiting for this message */ 1970 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1971 wake_up(&lgr->llc_msg_waiter); 1972 return; 1973 } 1974 break; 1975 case SMC_LLC_DELETE_LINK: 1976 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1977 !lgr->llc_flow_lcl.qentry) { 1978 /* DEL LINK REQ during ADD LINK SEQ */ 1979 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1980 wake_up(&lgr->llc_msg_waiter); 1981 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1982 schedule_work(&lgr->llc_del_link_work); 1983 } 1984 return; 1985 case SMC_LLC_CONFIRM_RKEY: 1986 /* new request from remote, assign to remote flow */ 1987 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1988 /* process here, does not wait for more llc msgs */ 1989 smc_llc_rmt_conf_rkey(lgr); 1990 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1991 } 1992 return; 1993 case SMC_LLC_CONFIRM_RKEY_CONT: 1994 /* not used because max links is 3, and 3 rkeys fit into 1995 * one CONFIRM_RKEY message 1996 */ 1997 break; 1998 case SMC_LLC_DELETE_RKEY: 1999 /* new request from remote, assign to remote flow */ 2000 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 2001 /* process here, does not wait for more llc msgs */ 2002 smc_llc_rmt_delete_rkey(lgr); 2003 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 2004 } 2005 return; 2006 case SMC_LLC_REQ_ADD_LINK: 2007 /* handle response here, smc_llc_flow_stop() cannot be called 2008 * in tasklet context 2009 */ 2010 if (lgr->role == SMC_CLNT && 2011 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK && 2012 (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) { 2013 smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl); 2014 } else if (lgr->role == SMC_SERV) { 2015 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 2016 /* as smc server, handle client suggestion */ 2017 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; 2018 schedule_work(&lgr->llc_add_link_work); 2019 } 2020 return; 2021 } 2022 break; 2023 default: 2024 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 2025 break; 2026 } 2027 out: 2028 kfree(qentry); 2029 } 2030 2031 /* worker to process llc messages on the event queue */ 2032 static void smc_llc_event_work(struct work_struct *work) 2033 { 2034 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 2035 llc_event_work); 2036 struct smc_llc_qentry *qentry; 2037 2038 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 2039 qentry = lgr->delayed_event; 2040 lgr->delayed_event = NULL; 2041 if (smc_link_usable(qentry->link)) 2042 smc_llc_event_handler(qentry); 2043 else 2044 kfree(qentry); 2045 } 2046 2047 again: 2048 spin_lock_bh(&lgr->llc_event_q_lock); 2049 if (!list_empty(&lgr->llc_event_q)) { 2050 qentry = list_first_entry(&lgr->llc_event_q, 2051 struct smc_llc_qentry, list); 2052 list_del_init(&qentry->list); 2053 spin_unlock_bh(&lgr->llc_event_q_lock); 2054 smc_llc_event_handler(qentry); 2055 goto again; 2056 } 2057 spin_unlock_bh(&lgr->llc_event_q_lock); 2058 } 2059 2060 /* process llc responses in tasklet context */ 2061 static void smc_llc_rx_response(struct smc_link *link, 2062 struct smc_llc_qentry *qentry) 2063 { 2064 enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type; 2065 struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl; 2066 u8 llc_type = qentry->msg.raw.hdr.common.llc_type; 2067 2068 switch (llc_type) { 2069 case SMC_LLC_TEST_LINK: 2070 if (smc_link_active(link)) 2071 complete(&link->llc_testlink_resp); 2072 break; 2073 case SMC_LLC_ADD_LINK: 2074 case SMC_LLC_ADD_LINK_CONT: 2075 case SMC_LLC_CONFIRM_LINK: 2076 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry) 2077 break; /* drop out-of-flow response */ 2078 goto assign; 2079 case SMC_LLC_DELETE_LINK: 2080 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry) 2081 break; /* drop out-of-flow response */ 2082 goto assign; 2083 case SMC_LLC_CONFIRM_RKEY: 2084 case SMC_LLC_DELETE_RKEY: 2085 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry) 2086 break; /* drop out-of-flow response */ 2087 goto assign; 2088 case SMC_LLC_CONFIRM_RKEY_CONT: 2089 /* not used because max links is 3 */ 2090 break; 2091 default: 2092 smc_llc_protocol_violation(link->lgr, 2093 qentry->msg.raw.hdr.common.type); 2094 break; 2095 } 2096 kfree(qentry); 2097 return; 2098 assign: 2099 /* assign responses to the local flow, we requested them */ 2100 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 2101 wake_up(&link->lgr->llc_msg_waiter); 2102 } 2103 2104 /* the longest tail either reader of qentry->body can use */ 2105 static u32 smc_llc_max_body_len(union smc_llc_msg *llc) 2106 { 2107 switch (llc->raw.hdr.common.llc_type) { 2108 case SMC_LLC_ADD_LINK: 2109 return offsetof(struct smc_llc_msg_add_link_v2_ext, rt) + 2110 SMC_LLC_RKEYS_PER_MSG_V2 * 2111 sizeof(struct smc_llc_msg_add_link_cont_rt); 2112 case SMC_LLC_DELETE_RKEY: 2113 return (SMC_LLC_RKEYS_PER_MSG_V2 - 2114 SMC_LLC_DEL_RKEY_V2_INLINE) * sizeof(__be32); 2115 default: 2116 return 0; 2117 } 2118 } 2119 2120 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, 2121 u32 byte_len) 2122 { 2123 struct smc_link_group *lgr = link->lgr; 2124 struct smc_llc_qentry *qentry; 2125 unsigned long flags; 2126 u16 body_len = 0; 2127 2128 /* V2 messages can be longer than the inline union smc_llc_msg. Carry 2129 * the remainder in the qentry itself, so that its lifetime and its 2130 * length match the message the peer actually sent. 2131 */ 2132 if (lgr->smc_version == SMC_V2 && byte_len > SMC_WR_TX_SIZE) 2133 body_len = min_t(u32, byte_len, SMC_WR_BUF_V2_SIZE) - 2134 SMC_WR_TX_SIZE; 2135 body_len = min_t(u32, body_len, smc_llc_max_body_len(llc)); 2136 2137 qentry = kmalloc_flex(*qentry, body, body_len, GFP_ATOMIC); 2138 if (!qentry) 2139 return; 2140 qentry->body_len = body_len; 2141 qentry->link = link; 2142 INIT_LIST_HEAD(&qentry->list); 2143 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 2144 if (body_len) { 2145 u8 *src = smc_link_shared_v2_rxbuf(link) ? 2146 (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc; 2147 2148 memcpy(qentry->body, src + SMC_WR_TX_SIZE, body_len); 2149 } 2150 2151 /* process responses immediately */ 2152 if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) && 2153 llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) { 2154 smc_llc_rx_response(link, qentry); 2155 return; 2156 } 2157 2158 /* add requests to event queue */ 2159 spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 2160 list_add_tail(&qentry->list, &lgr->llc_event_q); 2161 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 2162 queue_work(system_highpri_wq, &lgr->llc_event_work); 2163 } 2164 2165 /* copy received msg and add it to the event queue */ 2166 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 2167 { 2168 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 2169 union smc_llc_msg *llc = buf; 2170 2171 if (wc->byte_len < sizeof(*llc)) 2172 return; /* short message */ 2173 if (!llc->raw.hdr.common.llc_version) { 2174 if (llc->raw.hdr.length != sizeof(*llc)) 2175 return; /* invalid message */ 2176 } else { 2177 if (llc->raw.hdr.length_v2 < sizeof(*llc)) 2178 return; /* invalid message */ 2179 } 2180 2181 smc_llc_enqueue(link, llc, wc->byte_len); 2182 } 2183 2184 /***************************** worker, utils *********************************/ 2185 2186 static void smc_llc_testlink_work(struct work_struct *work) 2187 { 2188 struct smc_link *link = container_of(to_delayed_work(work), 2189 struct smc_link, llc_testlink_wrk); 2190 unsigned long next_interval; 2191 unsigned long expire_time; 2192 u8 user_data[16] = { 0 }; 2193 int rc; 2194 2195 if (!smc_link_active(link)) 2196 return; /* don't reschedule worker */ 2197 expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 2198 if (time_is_after_jiffies(expire_time)) { 2199 next_interval = expire_time - jiffies; 2200 goto out; 2201 } 2202 reinit_completion(&link->llc_testlink_resp); 2203 smc_llc_send_test_link(link, user_data); 2204 /* receive TEST LINK response over RoCE fabric */ 2205 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 2206 SMC_LLC_WAIT_TIME); 2207 if (!smc_link_active(link)) 2208 return; /* link state changed */ 2209 if (rc <= 0) { 2210 smcr_link_down_cond_sched(link); 2211 return; 2212 } 2213 next_interval = link->llc_testlink_time; 2214 out: 2215 schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 2216 } 2217 2218 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 2219 { 2220 struct net *net = sock_net(smc->clcsock->sk); 2221 2222 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 2223 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 2224 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 2225 INIT_LIST_HEAD(&lgr->llc_event_q); 2226 spin_lock_init(&lgr->llc_event_q_lock); 2227 spin_lock_init(&lgr->llc_flow_lock); 2228 init_waitqueue_head(&lgr->llc_flow_waiter); 2229 init_waitqueue_head(&lgr->llc_msg_waiter); 2230 init_rwsem(&lgr->llc_conf_mutex); 2231 lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time); 2232 lgr->max_send_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_send_wr)); 2233 lgr->max_recv_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_recv_wr)); 2234 } 2235 2236 /* called after lgr was removed from lgr_list */ 2237 void smc_llc_lgr_clear(struct smc_link_group *lgr) 2238 { 2239 smc_llc_event_flush(lgr); 2240 wake_up_all(&lgr->llc_flow_waiter); 2241 wake_up_all(&lgr->llc_msg_waiter); 2242 cancel_work_sync(&lgr->llc_event_work); 2243 cancel_work_sync(&lgr->llc_add_link_work); 2244 cancel_work_sync(&lgr->llc_del_link_work); 2245 if (lgr->delayed_event) { 2246 kfree(lgr->delayed_event); 2247 lgr->delayed_event = NULL; 2248 } 2249 } 2250 2251 int smc_llc_link_init(struct smc_link *link) 2252 { 2253 init_completion(&link->llc_testlink_resp); 2254 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 2255 return 0; 2256 } 2257 2258 void smc_llc_link_active(struct smc_link *link) 2259 { 2260 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, " 2261 "peerid %*phN, ibdev %s, ibport %d\n", 2262 SMC_LGR_ID_SIZE, &link->lgr->id, 2263 link->lgr->net->net_cookie, 2264 SMC_LGR_ID_SIZE, &link->link_uid, 2265 SMC_LGR_ID_SIZE, &link->peer_link_uid, 2266 link->smcibdev->ibdev->name, link->ibport); 2267 link->state = SMC_LNK_ACTIVE; 2268 if (link->lgr->llc_testlink_time) { 2269 link->llc_testlink_time = link->lgr->llc_testlink_time; 2270 schedule_delayed_work(&link->llc_testlink_wrk, 2271 link->llc_testlink_time); 2272 } 2273 } 2274 2275 /* called in worker context */ 2276 void smc_llc_link_clear(struct smc_link *link, bool log) 2277 { 2278 if (log) 2279 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN" 2280 ", peerid %*phN, ibdev %s, ibport %d\n", 2281 SMC_LGR_ID_SIZE, &link->lgr->id, 2282 link->lgr->net->net_cookie, 2283 SMC_LGR_ID_SIZE, &link->link_uid, 2284 SMC_LGR_ID_SIZE, &link->peer_link_uid, 2285 link->smcibdev->ibdev->name, link->ibport); 2286 complete(&link->llc_testlink_resp); 2287 cancel_delayed_work_sync(&link->llc_testlink_wrk); 2288 } 2289 2290 /* register a new rtoken at the remote peer (for all links) */ 2291 int smc_llc_do_confirm_rkey(struct smc_link *send_link, 2292 struct smc_buf_desc *rmb_desc) 2293 { 2294 struct smc_link_group *lgr = send_link->lgr; 2295 struct smc_llc_qentry *qentry = NULL; 2296 int rc = 0; 2297 2298 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 2299 if (rc) 2300 goto out; 2301 /* receive CONFIRM RKEY response from server over RoCE fabric */ 2302 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 2303 SMC_LLC_CONFIRM_RKEY); 2304 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 2305 rc = -EFAULT; 2306 out: 2307 if (qentry) 2308 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 2309 return rc; 2310 } 2311 2312 /* unregister an rtoken at the remote peer */ 2313 int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 2314 struct smc_buf_desc *rmb_desc) 2315 { 2316 struct smc_llc_qentry *qentry = NULL; 2317 struct smc_link *send_link; 2318 int rc = 0; 2319 2320 send_link = smc_llc_usable_link(lgr); 2321 if (!send_link) 2322 return -ENOLINK; 2323 2324 /* protected by llc_flow control */ 2325 rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 2326 if (rc) 2327 goto out; 2328 /* receive DELETE RKEY response from server over RoCE fabric */ 2329 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 2330 SMC_LLC_DELETE_RKEY); 2331 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 2332 rc = -EFAULT; 2333 out: 2334 if (qentry) 2335 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 2336 return rc; 2337 } 2338 2339 void smc_llc_link_set_uid(struct smc_link *link) 2340 { 2341 __be32 link_uid; 2342 2343 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 2344 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 2345 } 2346 2347 /* save peers link user id, used for debug purposes */ 2348 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 2349 { 2350 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 2351 SMC_LGR_ID_SIZE); 2352 } 2353 2354 /* evaluate confirm link request or response */ 2355 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 2356 enum smc_llc_reqresp type) 2357 { 2358 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 2359 qentry->link->link_id = qentry->msg.confirm_link.link_num; 2360 smc_llc_link_set_uid(qentry->link); 2361 } 2362 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 2363 return -ENOTSUPP; 2364 return 0; 2365 } 2366 2367 /***************************** init, exit, misc ******************************/ 2368 2369 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 2370 { 2371 .handler = smc_llc_rx_handler, 2372 .type = SMC_LLC_CONFIRM_LINK 2373 }, 2374 { 2375 .handler = smc_llc_rx_handler, 2376 .type = SMC_LLC_TEST_LINK 2377 }, 2378 { 2379 .handler = smc_llc_rx_handler, 2380 .type = SMC_LLC_ADD_LINK 2381 }, 2382 { 2383 .handler = smc_llc_rx_handler, 2384 .type = SMC_LLC_ADD_LINK_CONT 2385 }, 2386 { 2387 .handler = smc_llc_rx_handler, 2388 .type = SMC_LLC_DELETE_LINK 2389 }, 2390 { 2391 .handler = smc_llc_rx_handler, 2392 .type = SMC_LLC_CONFIRM_RKEY 2393 }, 2394 { 2395 .handler = smc_llc_rx_handler, 2396 .type = SMC_LLC_CONFIRM_RKEY_CONT 2397 }, 2398 { 2399 .handler = smc_llc_rx_handler, 2400 .type = SMC_LLC_DELETE_RKEY 2401 }, 2402 /* V2 types */ 2403 { 2404 .handler = smc_llc_rx_handler, 2405 .type = SMC_LLC_CONFIRM_LINK_V2 2406 }, 2407 { 2408 .handler = smc_llc_rx_handler, 2409 .type = SMC_LLC_TEST_LINK_V2 2410 }, 2411 { 2412 .handler = smc_llc_rx_handler, 2413 .type = SMC_LLC_ADD_LINK_V2 2414 }, 2415 { 2416 .handler = smc_llc_rx_handler, 2417 .type = SMC_LLC_DELETE_LINK_V2 2418 }, 2419 { 2420 .handler = smc_llc_rx_handler, 2421 .type = SMC_LLC_REQ_ADD_LINK_V2 2422 }, 2423 { 2424 .handler = smc_llc_rx_handler, 2425 .type = SMC_LLC_CONFIRM_RKEY_V2 2426 }, 2427 { 2428 .handler = smc_llc_rx_handler, 2429 .type = SMC_LLC_DELETE_RKEY_V2 2430 }, 2431 { 2432 .handler = NULL, 2433 } 2434 }; 2435 2436 int __init smc_llc_init(void) 2437 { 2438 struct smc_wr_rx_handler *handler; 2439 int rc = 0; 2440 2441 for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 2442 INIT_HLIST_NODE(&handler->list); 2443 rc = smc_wr_rx_register_handler(handler); 2444 if (rc) 2445 break; 2446 } 2447 return rc; 2448 } 2449