1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Basic Transport Functions exploiting Infiniband API 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 10 */ 11 12 #include <linux/socket.h> 13 #include <linux/if_vlan.h> 14 #include <linux/random.h> 15 #include <linux/workqueue.h> 16 #include <net/tcp.h> 17 #include <net/sock.h> 18 #include <rdma/ib_verbs.h> 19 #include <rdma/ib_cache.h> 20 21 #include "smc.h" 22 #include "smc_clc.h" 23 #include "smc_core.h" 24 #include "smc_ib.h" 25 #include "smc_wr.h" 26 #include "smc_llc.h" 27 #include "smc_cdc.h" 28 #include "smc_close.h" 29 #include "smc_ism.h" 30 31 #define SMC_LGR_NUM_INCR 256 32 #define SMC_LGR_FREE_DELAY_SERV (600 * HZ) 33 #define SMC_LGR_FREE_DELAY_CLNT (SMC_LGR_FREE_DELAY_SERV + 10 * HZ) 34 #define SMC_LGR_FREE_DELAY_FAST (8 * HZ) 35 36 static struct smc_lgr_list smc_lgr_list = { /* established link groups */ 37 .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock), 38 .list = LIST_HEAD_INIT(smc_lgr_list.list), 39 .num = 0, 40 }; 41 42 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb, 43 struct smc_buf_desc *buf_desc); 44 45 static void smc_lgr_schedule_free_work(struct smc_link_group *lgr) 46 { 47 /* client link group creation always follows the server link group 48 * creation. For client use a somewhat higher removal delay time, 49 * otherwise there is a risk of out-of-sync link groups. 50 */ 51 mod_delayed_work(system_wq, &lgr->free_work, 52 (!lgr->is_smcd && lgr->role == SMC_CLNT) ? 53 SMC_LGR_FREE_DELAY_CLNT : SMC_LGR_FREE_DELAY_SERV); 54 } 55 56 void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr) 57 { 58 mod_delayed_work(system_wq, &lgr->free_work, SMC_LGR_FREE_DELAY_FAST); 59 } 60 61 /* Register connection's alert token in our lookup structure. 62 * To use rbtrees we have to implement our own insert core. 63 * Requires @conns_lock 64 * @smc connection to register 65 * Returns 0 on success, != otherwise. 66 */ 67 static void smc_lgr_add_alert_token(struct smc_connection *conn) 68 { 69 struct rb_node **link, *parent = NULL; 70 u32 token = conn->alert_token_local; 71 72 link = &conn->lgr->conns_all.rb_node; 73 while (*link) { 74 struct smc_connection *cur = rb_entry(*link, 75 struct smc_connection, alert_node); 76 77 parent = *link; 78 if (cur->alert_token_local > token) 79 link = &parent->rb_left; 80 else 81 link = &parent->rb_right; 82 } 83 /* Put the new node there */ 84 rb_link_node(&conn->alert_node, parent, link); 85 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); 86 } 87 88 /* Register connection in link group by assigning an alert token 89 * registered in a search tree. 90 * Requires @conns_lock 91 * Note that '0' is a reserved value and not assigned. 92 */ 93 static void smc_lgr_register_conn(struct smc_connection *conn) 94 { 95 struct smc_sock *smc = container_of(conn, struct smc_sock, conn); 96 static atomic_t nexttoken = ATOMIC_INIT(0); 97 98 /* find a new alert_token_local value not yet used by some connection 99 * in this link group 100 */ 101 sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */ 102 while (!conn->alert_token_local) { 103 conn->alert_token_local = atomic_inc_return(&nexttoken); 104 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) 105 conn->alert_token_local = 0; 106 } 107 smc_lgr_add_alert_token(conn); 108 conn->lgr->conns_num++; 109 } 110 111 /* Unregister connection and reset the alert token of the given connection< 112 */ 113 static void __smc_lgr_unregister_conn(struct smc_connection *conn) 114 { 115 struct smc_sock *smc = container_of(conn, struct smc_sock, conn); 116 struct smc_link_group *lgr = conn->lgr; 117 118 rb_erase(&conn->alert_node, &lgr->conns_all); 119 lgr->conns_num--; 120 conn->alert_token_local = 0; 121 sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */ 122 } 123 124 /* Unregister connection from lgr 125 */ 126 static void smc_lgr_unregister_conn(struct smc_connection *conn) 127 { 128 struct smc_link_group *lgr = conn->lgr; 129 130 if (!lgr) 131 return; 132 write_lock_bh(&lgr->conns_lock); 133 if (conn->alert_token_local) { 134 __smc_lgr_unregister_conn(conn); 135 } 136 write_unlock_bh(&lgr->conns_lock); 137 } 138 139 /* Send delete link, either as client to request the initiation 140 * of the DELETE LINK sequence from server; or as server to 141 * initiate the delete processing. See smc_llc_rx_delete_link(). 142 */ 143 static int smc_link_send_delete(struct smc_link *lnk) 144 { 145 if (lnk->state == SMC_LNK_ACTIVE && 146 !smc_llc_send_delete_link(lnk, SMC_LLC_REQ, true)) { 147 smc_llc_link_deleting(lnk); 148 return 0; 149 } 150 return -ENOTCONN; 151 } 152 153 static void smc_lgr_free(struct smc_link_group *lgr); 154 155 static void smc_lgr_free_work(struct work_struct *work) 156 { 157 struct smc_link_group *lgr = container_of(to_delayed_work(work), 158 struct smc_link_group, 159 free_work); 160 bool conns; 161 162 spin_lock_bh(&smc_lgr_list.lock); 163 read_lock_bh(&lgr->conns_lock); 164 conns = RB_EMPTY_ROOT(&lgr->conns_all); 165 read_unlock_bh(&lgr->conns_lock); 166 if (!conns) { /* number of lgr connections is no longer zero */ 167 spin_unlock_bh(&smc_lgr_list.lock); 168 return; 169 } 170 if (!list_empty(&lgr->list)) 171 list_del_init(&lgr->list); /* remove from smc_lgr_list */ 172 spin_unlock_bh(&smc_lgr_list.lock); 173 174 if (!lgr->is_smcd && !lgr->terminating) { 175 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 176 177 /* try to send del link msg, on error free lgr immediately */ 178 if (lnk->state == SMC_LNK_ACTIVE && 179 !smc_link_send_delete(lnk)) { 180 /* reschedule in case we never receive a response */ 181 smc_lgr_schedule_free_work(lgr); 182 return; 183 } 184 } 185 186 if (!delayed_work_pending(&lgr->free_work)) { 187 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 188 189 if (!lgr->is_smcd && lnk->state != SMC_LNK_INACTIVE) 190 smc_llc_link_inactive(lnk); 191 if (lgr->is_smcd) 192 smc_ism_signal_shutdown(lgr); 193 smc_lgr_free(lgr); 194 } 195 } 196 197 /* create a new SMC link group */ 198 static int smc_lgr_create(struct smc_sock *smc, bool is_smcd, 199 struct smc_ib_device *smcibdev, u8 ibport, 200 char *peer_systemid, unsigned short vlan_id, 201 struct smcd_dev *smcismdev, u64 peer_gid) 202 { 203 struct smc_link_group *lgr; 204 struct smc_link *lnk; 205 u8 rndvec[3]; 206 int rc = 0; 207 int i; 208 209 if (is_smcd && vlan_id) { 210 rc = smc_ism_get_vlan(smcismdev, vlan_id); 211 if (rc) 212 goto out; 213 } 214 215 lgr = kzalloc(sizeof(*lgr), GFP_KERNEL); 216 if (!lgr) { 217 rc = -ENOMEM; 218 goto out; 219 } 220 lgr->is_smcd = is_smcd; 221 lgr->sync_err = 0; 222 lgr->vlan_id = vlan_id; 223 rwlock_init(&lgr->sndbufs_lock); 224 rwlock_init(&lgr->rmbs_lock); 225 rwlock_init(&lgr->conns_lock); 226 for (i = 0; i < SMC_RMBE_SIZES; i++) { 227 INIT_LIST_HEAD(&lgr->sndbufs[i]); 228 INIT_LIST_HEAD(&lgr->rmbs[i]); 229 } 230 smc_lgr_list.num += SMC_LGR_NUM_INCR; 231 memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE); 232 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); 233 lgr->conns_all = RB_ROOT; 234 if (is_smcd) { 235 /* SMC-D specific settings */ 236 lgr->peer_gid = peer_gid; 237 lgr->smcd = smcismdev; 238 } else { 239 /* SMC-R specific settings */ 240 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; 241 memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN); 242 243 lnk = &lgr->lnk[SMC_SINGLE_LINK]; 244 /* initialize link */ 245 lnk->state = SMC_LNK_ACTIVATING; 246 lnk->link_id = SMC_SINGLE_LINK; 247 lnk->smcibdev = smcibdev; 248 lnk->ibport = ibport; 249 lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu; 250 if (!smcibdev->initialized) 251 smc_ib_setup_per_ibdev(smcibdev); 252 get_random_bytes(rndvec, sizeof(rndvec)); 253 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + 254 (rndvec[2] << 16); 255 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport, 256 vlan_id, lnk->gid, &lnk->sgid_index); 257 if (rc) 258 goto free_lgr; 259 rc = smc_llc_link_init(lnk); 260 if (rc) 261 goto free_lgr; 262 rc = smc_wr_alloc_link_mem(lnk); 263 if (rc) 264 goto clear_llc_lnk; 265 rc = smc_ib_create_protection_domain(lnk); 266 if (rc) 267 goto free_link_mem; 268 rc = smc_ib_create_queue_pair(lnk); 269 if (rc) 270 goto dealloc_pd; 271 rc = smc_wr_create_link(lnk); 272 if (rc) 273 goto destroy_qp; 274 } 275 smc->conn.lgr = lgr; 276 spin_lock_bh(&smc_lgr_list.lock); 277 list_add(&lgr->list, &smc_lgr_list.list); 278 spin_unlock_bh(&smc_lgr_list.lock); 279 return 0; 280 281 destroy_qp: 282 smc_ib_destroy_queue_pair(lnk); 283 dealloc_pd: 284 smc_ib_dealloc_protection_domain(lnk); 285 free_link_mem: 286 smc_wr_free_link_mem(lnk); 287 clear_llc_lnk: 288 smc_llc_link_clear(lnk); 289 free_lgr: 290 kfree(lgr); 291 out: 292 return rc; 293 } 294 295 static void smc_buf_unuse(struct smc_connection *conn, 296 struct smc_link_group *lgr) 297 { 298 if (conn->sndbuf_desc) 299 conn->sndbuf_desc->used = 0; 300 if (conn->rmb_desc) { 301 if (!conn->rmb_desc->regerr) { 302 if (!lgr->is_smcd) { 303 /* unregister rmb with peer */ 304 smc_llc_do_delete_rkey( 305 &lgr->lnk[SMC_SINGLE_LINK], 306 conn->rmb_desc); 307 } 308 conn->rmb_desc->used = 0; 309 } else { 310 /* buf registration failed, reuse not possible */ 311 write_lock_bh(&lgr->rmbs_lock); 312 list_del(&conn->rmb_desc->list); 313 write_unlock_bh(&lgr->rmbs_lock); 314 315 smc_buf_free(lgr, true, conn->rmb_desc); 316 } 317 } 318 } 319 320 /* remove a finished connection from its link group */ 321 void smc_conn_free(struct smc_connection *conn) 322 { 323 struct smc_link_group *lgr = conn->lgr; 324 325 if (!lgr) 326 return; 327 if (lgr->is_smcd) { 328 smc_ism_unset_conn(conn); 329 tasklet_kill(&conn->rx_tsklet); 330 } else { 331 smc_cdc_tx_dismiss_slots(conn); 332 } 333 smc_lgr_unregister_conn(conn); 334 smc_buf_unuse(conn, lgr); /* allow buffer reuse */ 335 conn->lgr = NULL; 336 337 if (!lgr->conns_num) 338 smc_lgr_schedule_free_work(lgr); 339 } 340 341 static void smc_link_clear(struct smc_link *lnk) 342 { 343 lnk->peer_qpn = 0; 344 smc_llc_link_clear(lnk); 345 smc_ib_modify_qp_reset(lnk); 346 smc_wr_free_link(lnk); 347 smc_ib_destroy_queue_pair(lnk); 348 smc_ib_dealloc_protection_domain(lnk); 349 smc_wr_free_link_mem(lnk); 350 } 351 352 static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb, 353 struct smc_buf_desc *buf_desc) 354 { 355 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 356 357 if (is_rmb) { 358 if (buf_desc->mr_rx[SMC_SINGLE_LINK]) 359 smc_ib_put_memory_region( 360 buf_desc->mr_rx[SMC_SINGLE_LINK]); 361 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, 362 DMA_FROM_DEVICE); 363 } else { 364 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, 365 DMA_TO_DEVICE); 366 } 367 sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]); 368 if (buf_desc->pages) 369 __free_pages(buf_desc->pages, buf_desc->order); 370 kfree(buf_desc); 371 } 372 373 static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb, 374 struct smc_buf_desc *buf_desc) 375 { 376 if (is_dmb) { 377 /* restore original buf len */ 378 buf_desc->len += sizeof(struct smcd_cdc_msg); 379 smc_ism_unregister_dmb(lgr->smcd, buf_desc); 380 } else { 381 kfree(buf_desc->cpu_addr); 382 } 383 kfree(buf_desc); 384 } 385 386 static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb, 387 struct smc_buf_desc *buf_desc) 388 { 389 if (lgr->is_smcd) 390 smcd_buf_free(lgr, is_rmb, buf_desc); 391 else 392 smcr_buf_free(lgr, is_rmb, buf_desc); 393 } 394 395 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb) 396 { 397 struct smc_buf_desc *buf_desc, *bf_desc; 398 struct list_head *buf_list; 399 int i; 400 401 for (i = 0; i < SMC_RMBE_SIZES; i++) { 402 if (is_rmb) 403 buf_list = &lgr->rmbs[i]; 404 else 405 buf_list = &lgr->sndbufs[i]; 406 list_for_each_entry_safe(buf_desc, bf_desc, buf_list, 407 list) { 408 list_del(&buf_desc->list); 409 smc_buf_free(lgr, is_rmb, buf_desc); 410 } 411 } 412 } 413 414 static void smc_lgr_free_bufs(struct smc_link_group *lgr) 415 { 416 /* free send buffers */ 417 __smc_lgr_free_bufs(lgr, false); 418 /* free rmbs */ 419 __smc_lgr_free_bufs(lgr, true); 420 } 421 422 /* remove a link group */ 423 static void smc_lgr_free(struct smc_link_group *lgr) 424 { 425 smc_lgr_free_bufs(lgr); 426 if (lgr->is_smcd) 427 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id); 428 else 429 smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]); 430 kfree(lgr); 431 } 432 433 void smc_lgr_forget(struct smc_link_group *lgr) 434 { 435 spin_lock_bh(&smc_lgr_list.lock); 436 /* do not use this link group for new connections */ 437 if (!list_empty(&lgr->list)) 438 list_del_init(&lgr->list); 439 spin_unlock_bh(&smc_lgr_list.lock); 440 } 441 442 /* terminate linkgroup abnormally */ 443 static void __smc_lgr_terminate(struct smc_link_group *lgr) 444 { 445 struct smc_connection *conn; 446 struct smc_sock *smc; 447 struct rb_node *node; 448 449 if (lgr->terminating) 450 return; /* lgr already terminating */ 451 lgr->terminating = 1; 452 if (!list_empty(&lgr->list)) /* forget lgr */ 453 list_del_init(&lgr->list); 454 if (!lgr->is_smcd) 455 smc_llc_link_inactive(&lgr->lnk[SMC_SINGLE_LINK]); 456 457 write_lock_bh(&lgr->conns_lock); 458 node = rb_first(&lgr->conns_all); 459 while (node) { 460 conn = rb_entry(node, struct smc_connection, alert_node); 461 smc = container_of(conn, struct smc_sock, conn); 462 sock_hold(&smc->sk); /* sock_put in close work */ 463 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; 464 __smc_lgr_unregister_conn(conn); 465 conn->lgr = NULL; 466 write_unlock_bh(&lgr->conns_lock); 467 if (!schedule_work(&conn->close_work)) 468 sock_put(&smc->sk); 469 write_lock_bh(&lgr->conns_lock); 470 node = rb_first(&lgr->conns_all); 471 } 472 write_unlock_bh(&lgr->conns_lock); 473 if (!lgr->is_smcd) 474 wake_up(&lgr->lnk[SMC_SINGLE_LINK].wr_reg_wait); 475 smc_lgr_schedule_free_work(lgr); 476 } 477 478 void smc_lgr_terminate(struct smc_link_group *lgr) 479 { 480 spin_lock_bh(&smc_lgr_list.lock); 481 __smc_lgr_terminate(lgr); 482 spin_unlock_bh(&smc_lgr_list.lock); 483 } 484 485 /* Called when IB port is terminated */ 486 void smc_port_terminate(struct smc_ib_device *smcibdev, u8 ibport) 487 { 488 struct smc_link_group *lgr, *l; 489 490 spin_lock_bh(&smc_lgr_list.lock); 491 list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) { 492 if (!lgr->is_smcd && 493 lgr->lnk[SMC_SINGLE_LINK].smcibdev == smcibdev && 494 lgr->lnk[SMC_SINGLE_LINK].ibport == ibport) 495 __smc_lgr_terminate(lgr); 496 } 497 spin_unlock_bh(&smc_lgr_list.lock); 498 } 499 500 /* Called when SMC-D device is terminated or peer is lost */ 501 void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid, unsigned short vlan) 502 { 503 struct smc_link_group *lgr, *l; 504 LIST_HEAD(lgr_free_list); 505 506 /* run common cleanup function and build free list */ 507 spin_lock_bh(&smc_lgr_list.lock); 508 list_for_each_entry_safe(lgr, l, &smc_lgr_list.list, list) { 509 if (lgr->is_smcd && lgr->smcd == dev && 510 (!peer_gid || lgr->peer_gid == peer_gid) && 511 (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) { 512 __smc_lgr_terminate(lgr); 513 list_move(&lgr->list, &lgr_free_list); 514 } 515 } 516 spin_unlock_bh(&smc_lgr_list.lock); 517 518 /* cancel the regular free workers and actually free lgrs */ 519 list_for_each_entry_safe(lgr, l, &lgr_free_list, list) { 520 list_del_init(&lgr->list); 521 cancel_delayed_work_sync(&lgr->free_work); 522 if (!peer_gid && vlan == VLAN_VID_MASK) /* dev terminated? */ 523 smc_ism_signal_shutdown(lgr); 524 smc_lgr_free(lgr); 525 } 526 } 527 528 /* Determine vlan of internal TCP socket. 529 * @vlan_id: address to store the determined vlan id into 530 */ 531 int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id) 532 { 533 struct dst_entry *dst = sk_dst_get(clcsock->sk); 534 struct net_device *ndev; 535 int i, nest_lvl, rc = 0; 536 537 *vlan_id = 0; 538 if (!dst) { 539 rc = -ENOTCONN; 540 goto out; 541 } 542 if (!dst->dev) { 543 rc = -ENODEV; 544 goto out_rel; 545 } 546 547 ndev = dst->dev; 548 if (is_vlan_dev(ndev)) { 549 *vlan_id = vlan_dev_vlan_id(ndev); 550 goto out_rel; 551 } 552 553 rtnl_lock(); 554 nest_lvl = dev_get_nest_level(ndev); 555 for (i = 0; i < nest_lvl; i++) { 556 struct list_head *lower = &ndev->adj_list.lower; 557 558 if (list_empty(lower)) 559 break; 560 lower = lower->next; 561 ndev = (struct net_device *)netdev_lower_get_next(ndev, &lower); 562 if (is_vlan_dev(ndev)) { 563 *vlan_id = vlan_dev_vlan_id(ndev); 564 break; 565 } 566 } 567 rtnl_unlock(); 568 569 out_rel: 570 dst_release(dst); 571 out: 572 return rc; 573 } 574 575 static bool smcr_lgr_match(struct smc_link_group *lgr, 576 struct smc_clc_msg_local *lcl, 577 enum smc_lgr_role role, u32 clcqpn) 578 { 579 return !memcmp(lgr->peer_systemid, lcl->id_for_peer, 580 SMC_SYSTEMID_LEN) && 581 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid, 582 SMC_GID_SIZE) && 583 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac, 584 sizeof(lcl->mac)) && 585 lgr->role == role && 586 (lgr->role == SMC_SERV || 587 lgr->lnk[SMC_SINGLE_LINK].peer_qpn == clcqpn); 588 } 589 590 static bool smcd_lgr_match(struct smc_link_group *lgr, 591 struct smcd_dev *smcismdev, u64 peer_gid) 592 { 593 return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev; 594 } 595 596 /* create a new SMC connection (and a new link group if necessary) */ 597 int smc_conn_create(struct smc_sock *smc, bool is_smcd, int srv_first_contact, 598 struct smc_ib_device *smcibdev, u8 ibport, u32 clcqpn, 599 struct smc_clc_msg_local *lcl, struct smcd_dev *smcd, 600 u64 peer_gid) 601 { 602 struct smc_connection *conn = &smc->conn; 603 int local_contact = SMC_FIRST_CONTACT; 604 struct smc_link_group *lgr; 605 unsigned short vlan_id; 606 enum smc_lgr_role role; 607 int rc = 0; 608 609 role = smc->listen_smc ? SMC_SERV : SMC_CLNT; 610 rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id); 611 if (rc) 612 return rc; 613 614 if ((role == SMC_CLNT) && srv_first_contact) 615 /* create new link group as well */ 616 goto create; 617 618 /* determine if an existing link group can be reused */ 619 spin_lock_bh(&smc_lgr_list.lock); 620 list_for_each_entry(lgr, &smc_lgr_list.list, list) { 621 write_lock_bh(&lgr->conns_lock); 622 if ((is_smcd ? smcd_lgr_match(lgr, smcd, peer_gid) : 623 smcr_lgr_match(lgr, lcl, role, clcqpn)) && 624 !lgr->sync_err && 625 lgr->vlan_id == vlan_id && 626 (role == SMC_CLNT || 627 lgr->conns_num < SMC_RMBS_PER_LGR_MAX)) { 628 /* link group found */ 629 local_contact = SMC_REUSE_CONTACT; 630 conn->lgr = lgr; 631 smc_lgr_register_conn(conn); /* add smc conn to lgr */ 632 if (delayed_work_pending(&lgr->free_work)) 633 cancel_delayed_work(&lgr->free_work); 634 write_unlock_bh(&lgr->conns_lock); 635 break; 636 } 637 write_unlock_bh(&lgr->conns_lock); 638 } 639 spin_unlock_bh(&smc_lgr_list.lock); 640 641 if (role == SMC_CLNT && !srv_first_contact && 642 (local_contact == SMC_FIRST_CONTACT)) { 643 /* Server reuses a link group, but Client wants to start 644 * a new one 645 * send out_of_sync decline, reason synchr. error 646 */ 647 return -ENOLINK; 648 } 649 650 create: 651 if (local_contact == SMC_FIRST_CONTACT) { 652 rc = smc_lgr_create(smc, is_smcd, smcibdev, ibport, 653 lcl->id_for_peer, vlan_id, smcd, peer_gid); 654 if (rc) 655 goto out; 656 smc_lgr_register_conn(conn); /* add smc conn to lgr */ 657 } 658 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE; 659 conn->local_tx_ctrl.len = SMC_WR_TX_SIZE; 660 conn->urg_state = SMC_URG_READ; 661 if (is_smcd) { 662 conn->rx_off = sizeof(struct smcd_cdc_msg); 663 smcd_cdc_rx_init(conn); /* init tasklet for this conn */ 664 } 665 #ifndef KERNEL_HAS_ATOMIC64 666 spin_lock_init(&conn->acurs_lock); 667 #endif 668 669 out: 670 return rc ? rc : local_contact; 671 } 672 673 /* convert the RMB size into the compressed notation - minimum 16K. 674 * In contrast to plain ilog2, this rounds towards the next power of 2, 675 * so the socket application gets at least its desired sndbuf / rcvbuf size. 676 */ 677 static u8 smc_compress_bufsize(int size) 678 { 679 u8 compressed; 680 681 if (size <= SMC_BUF_MIN_SIZE) 682 return 0; 683 684 size = (size - 1) >> 14; 685 compressed = ilog2(size) + 1; 686 if (compressed >= SMC_RMBE_SIZES) 687 compressed = SMC_RMBE_SIZES - 1; 688 return compressed; 689 } 690 691 /* convert the RMB size from compressed notation into integer */ 692 int smc_uncompress_bufsize(u8 compressed) 693 { 694 u32 size; 695 696 size = 0x00000001 << (((int)compressed) + 14); 697 return (int)size; 698 } 699 700 /* try to reuse a sndbuf or rmb description slot for a certain 701 * buffer size; if not available, return NULL 702 */ 703 static struct smc_buf_desc *smc_buf_get_slot(int compressed_bufsize, 704 rwlock_t *lock, 705 struct list_head *buf_list) 706 { 707 struct smc_buf_desc *buf_slot; 708 709 read_lock_bh(lock); 710 list_for_each_entry(buf_slot, buf_list, list) { 711 if (cmpxchg(&buf_slot->used, 0, 1) == 0) { 712 read_unlock_bh(lock); 713 return buf_slot; 714 } 715 } 716 read_unlock_bh(lock); 717 return NULL; 718 } 719 720 /* one of the conditions for announcing a receiver's current window size is 721 * that it "results in a minimum increase in the window size of 10% of the 722 * receive buffer space" [RFC7609] 723 */ 724 static inline int smc_rmb_wnd_update_limit(int rmbe_size) 725 { 726 return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2); 727 } 728 729 static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr, 730 bool is_rmb, int bufsize) 731 { 732 struct smc_buf_desc *buf_desc; 733 struct smc_link *lnk; 734 int rc; 735 736 /* try to alloc a new buffer */ 737 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); 738 if (!buf_desc) 739 return ERR_PTR(-ENOMEM); 740 741 buf_desc->order = get_order(bufsize); 742 buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN | 743 __GFP_NOMEMALLOC | __GFP_COMP | 744 __GFP_NORETRY | __GFP_ZERO, 745 buf_desc->order); 746 if (!buf_desc->pages) { 747 kfree(buf_desc); 748 return ERR_PTR(-EAGAIN); 749 } 750 buf_desc->cpu_addr = (void *)page_address(buf_desc->pages); 751 752 /* build the sg table from the pages */ 753 lnk = &lgr->lnk[SMC_SINGLE_LINK]; 754 rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1, 755 GFP_KERNEL); 756 if (rc) { 757 smc_buf_free(lgr, is_rmb, buf_desc); 758 return ERR_PTR(rc); 759 } 760 sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl, 761 buf_desc->cpu_addr, bufsize); 762 763 /* map sg table to DMA address */ 764 rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc, 765 is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); 766 /* SMC protocol depends on mapping to one DMA address only */ 767 if (rc != 1) { 768 smc_buf_free(lgr, is_rmb, buf_desc); 769 return ERR_PTR(-EAGAIN); 770 } 771 772 /* create a new memory region for the RMB */ 773 if (is_rmb) { 774 rc = smc_ib_get_memory_region(lnk->roce_pd, 775 IB_ACCESS_REMOTE_WRITE | 776 IB_ACCESS_LOCAL_WRITE, 777 buf_desc); 778 if (rc) { 779 smc_buf_free(lgr, is_rmb, buf_desc); 780 return ERR_PTR(rc); 781 } 782 } 783 784 buf_desc->len = bufsize; 785 return buf_desc; 786 } 787 788 #define SMCD_DMBE_SIZES 7 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */ 789 790 static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr, 791 bool is_dmb, int bufsize) 792 { 793 struct smc_buf_desc *buf_desc; 794 int rc; 795 796 if (smc_compress_bufsize(bufsize) > SMCD_DMBE_SIZES) 797 return ERR_PTR(-EAGAIN); 798 799 /* try to alloc a new DMB */ 800 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); 801 if (!buf_desc) 802 return ERR_PTR(-ENOMEM); 803 if (is_dmb) { 804 rc = smc_ism_register_dmb(lgr, bufsize, buf_desc); 805 if (rc) { 806 kfree(buf_desc); 807 return ERR_PTR(-EAGAIN); 808 } 809 buf_desc->pages = virt_to_page(buf_desc->cpu_addr); 810 /* CDC header stored in buf. So, pretend it was smaller */ 811 buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg); 812 } else { 813 buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL | 814 __GFP_NOWARN | __GFP_NORETRY | 815 __GFP_NOMEMALLOC); 816 if (!buf_desc->cpu_addr) { 817 kfree(buf_desc); 818 return ERR_PTR(-EAGAIN); 819 } 820 buf_desc->len = bufsize; 821 } 822 return buf_desc; 823 } 824 825 static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) 826 { 827 struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM); 828 struct smc_connection *conn = &smc->conn; 829 struct smc_link_group *lgr = conn->lgr; 830 struct list_head *buf_list; 831 int bufsize, bufsize_short; 832 int sk_buf_size; 833 rwlock_t *lock; 834 835 if (is_rmb) 836 /* use socket recv buffer size (w/o overhead) as start value */ 837 sk_buf_size = smc->sk.sk_rcvbuf / 2; 838 else 839 /* use socket send buffer size (w/o overhead) as start value */ 840 sk_buf_size = smc->sk.sk_sndbuf / 2; 841 842 for (bufsize_short = smc_compress_bufsize(sk_buf_size); 843 bufsize_short >= 0; bufsize_short--) { 844 845 if (is_rmb) { 846 lock = &lgr->rmbs_lock; 847 buf_list = &lgr->rmbs[bufsize_short]; 848 } else { 849 lock = &lgr->sndbufs_lock; 850 buf_list = &lgr->sndbufs[bufsize_short]; 851 } 852 bufsize = smc_uncompress_bufsize(bufsize_short); 853 if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC) 854 continue; 855 856 /* check for reusable slot in the link group */ 857 buf_desc = smc_buf_get_slot(bufsize_short, lock, buf_list); 858 if (buf_desc) { 859 memset(buf_desc->cpu_addr, 0, bufsize); 860 break; /* found reusable slot */ 861 } 862 863 if (is_smcd) 864 buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize); 865 else 866 buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize); 867 868 if (PTR_ERR(buf_desc) == -ENOMEM) 869 break; 870 if (IS_ERR(buf_desc)) 871 continue; 872 873 buf_desc->used = 1; 874 write_lock_bh(lock); 875 list_add(&buf_desc->list, buf_list); 876 write_unlock_bh(lock); 877 break; /* found */ 878 } 879 880 if (IS_ERR(buf_desc)) 881 return -ENOMEM; 882 883 if (is_rmb) { 884 conn->rmb_desc = buf_desc; 885 conn->rmbe_size_short = bufsize_short; 886 smc->sk.sk_rcvbuf = bufsize * 2; 887 atomic_set(&conn->bytes_to_rcv, 0); 888 conn->rmbe_update_limit = 889 smc_rmb_wnd_update_limit(buf_desc->len); 890 if (is_smcd) 891 smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */ 892 } else { 893 conn->sndbuf_desc = buf_desc; 894 smc->sk.sk_sndbuf = bufsize * 2; 895 atomic_set(&conn->sndbuf_space, bufsize); 896 } 897 return 0; 898 } 899 900 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn) 901 { 902 struct smc_link_group *lgr = conn->lgr; 903 904 if (!conn->lgr || conn->lgr->is_smcd) 905 return; 906 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 907 conn->sndbuf_desc, DMA_TO_DEVICE); 908 } 909 910 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn) 911 { 912 struct smc_link_group *lgr = conn->lgr; 913 914 if (!conn->lgr || conn->lgr->is_smcd) 915 return; 916 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 917 conn->sndbuf_desc, DMA_TO_DEVICE); 918 } 919 920 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn) 921 { 922 struct smc_link_group *lgr = conn->lgr; 923 924 if (!conn->lgr || conn->lgr->is_smcd) 925 return; 926 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 927 conn->rmb_desc, DMA_FROM_DEVICE); 928 } 929 930 void smc_rmb_sync_sg_for_device(struct smc_connection *conn) 931 { 932 struct smc_link_group *lgr = conn->lgr; 933 934 if (!conn->lgr || conn->lgr->is_smcd) 935 return; 936 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 937 conn->rmb_desc, DMA_FROM_DEVICE); 938 } 939 940 /* create the send and receive buffer for an SMC socket; 941 * receive buffers are called RMBs; 942 * (even though the SMC protocol allows more than one RMB-element per RMB, 943 * the Linux implementation uses just one RMB-element per RMB, i.e. uses an 944 * extra RMB for every connection in a link group 945 */ 946 int smc_buf_create(struct smc_sock *smc, bool is_smcd) 947 { 948 int rc; 949 950 /* create send buffer */ 951 rc = __smc_buf_create(smc, is_smcd, false); 952 if (rc) 953 return rc; 954 /* create rmb */ 955 rc = __smc_buf_create(smc, is_smcd, true); 956 if (rc) 957 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc); 958 return rc; 959 } 960 961 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr) 962 { 963 int i; 964 965 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { 966 if (!test_and_set_bit(i, lgr->rtokens_used_mask)) 967 return i; 968 } 969 return -ENOSPC; 970 } 971 972 /* add a new rtoken from peer */ 973 int smc_rtoken_add(struct smc_link_group *lgr, __be64 nw_vaddr, __be32 nw_rkey) 974 { 975 u64 dma_addr = be64_to_cpu(nw_vaddr); 976 u32 rkey = ntohl(nw_rkey); 977 int i; 978 979 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { 980 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) && 981 (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) && 982 test_bit(i, lgr->rtokens_used_mask)) { 983 /* already in list */ 984 return i; 985 } 986 } 987 i = smc_rmb_reserve_rtoken_idx(lgr); 988 if (i < 0) 989 return i; 990 lgr->rtokens[i][SMC_SINGLE_LINK].rkey = rkey; 991 lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = dma_addr; 992 return i; 993 } 994 995 /* delete an rtoken */ 996 int smc_rtoken_delete(struct smc_link_group *lgr, __be32 nw_rkey) 997 { 998 u32 rkey = ntohl(nw_rkey); 999 int i; 1000 1001 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { 1002 if (lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey && 1003 test_bit(i, lgr->rtokens_used_mask)) { 1004 lgr->rtokens[i][SMC_SINGLE_LINK].rkey = 0; 1005 lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = 0; 1006 1007 clear_bit(i, lgr->rtokens_used_mask); 1008 return 0; 1009 } 1010 } 1011 return -ENOENT; 1012 } 1013 1014 /* save rkey and dma_addr received from peer during clc handshake */ 1015 int smc_rmb_rtoken_handling(struct smc_connection *conn, 1016 struct smc_clc_msg_accept_confirm *clc) 1017 { 1018 conn->rtoken_idx = smc_rtoken_add(conn->lgr, clc->rmb_dma_addr, 1019 clc->rmb_rkey); 1020 if (conn->rtoken_idx < 0) 1021 return conn->rtoken_idx; 1022 return 0; 1023 } 1024 1025 /* Called (from smc_exit) when module is removed */ 1026 void smc_core_exit(void) 1027 { 1028 struct smc_link_group *lgr, *lg; 1029 LIST_HEAD(lgr_freeing_list); 1030 1031 spin_lock_bh(&smc_lgr_list.lock); 1032 if (!list_empty(&smc_lgr_list.list)) 1033 list_splice_init(&smc_lgr_list.list, &lgr_freeing_list); 1034 spin_unlock_bh(&smc_lgr_list.lock); 1035 list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) { 1036 list_del_init(&lgr->list); 1037 if (!lgr->is_smcd) { 1038 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 1039 1040 if (lnk->state == SMC_LNK_ACTIVE) 1041 smc_llc_send_delete_link(lnk, SMC_LLC_REQ, 1042 false); 1043 smc_llc_link_inactive(lnk); 1044 } 1045 cancel_delayed_work_sync(&lgr->free_work); 1046 if (lgr->is_smcd) 1047 smc_ism_signal_shutdown(lgr); 1048 smc_lgr_free(lgr); /* free link group */ 1049 } 1050 } 1051