1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Basic Transport Functions exploiting Infiniband API 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 10 */ 11 12 #include <linux/socket.h> 13 #include <linux/if_vlan.h> 14 #include <linux/random.h> 15 #include <linux/workqueue.h> 16 #include <net/tcp.h> 17 #include <net/sock.h> 18 #include <rdma/ib_verbs.h> 19 20 #include "smc.h" 21 #include "smc_clc.h" 22 #include "smc_core.h" 23 #include "smc_ib.h" 24 #include "smc_wr.h" 25 #include "smc_llc.h" 26 #include "smc_cdc.h" 27 #include "smc_close.h" 28 29 #define SMC_LGR_NUM_INCR 256 30 #define SMC_LGR_FREE_DELAY_SERV (600 * HZ) 31 #define SMC_LGR_FREE_DELAY_CLNT (SMC_LGR_FREE_DELAY_SERV + 10) 32 33 static u32 smc_lgr_num; /* unique link group number */ 34 35 /* Register connection's alert token in our lookup structure. 36 * To use rbtrees we have to implement our own insert core. 37 * Requires @conns_lock 38 * @smc connection to register 39 * Returns 0 on success, != otherwise. 40 */ 41 static void smc_lgr_add_alert_token(struct smc_connection *conn) 42 { 43 struct rb_node **link, *parent = NULL; 44 u32 token = conn->alert_token_local; 45 46 link = &conn->lgr->conns_all.rb_node; 47 while (*link) { 48 struct smc_connection *cur = rb_entry(*link, 49 struct smc_connection, alert_node); 50 51 parent = *link; 52 if (cur->alert_token_local > token) 53 link = &parent->rb_left; 54 else 55 link = &parent->rb_right; 56 } 57 /* Put the new node there */ 58 rb_link_node(&conn->alert_node, parent, link); 59 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); 60 } 61 62 /* Register connection in link group by assigning an alert token 63 * registered in a search tree. 64 * Requires @conns_lock 65 * Note that '0' is a reserved value and not assigned. 66 */ 67 static void smc_lgr_register_conn(struct smc_connection *conn) 68 { 69 struct smc_sock *smc = container_of(conn, struct smc_sock, conn); 70 static atomic_t nexttoken = ATOMIC_INIT(0); 71 72 /* find a new alert_token_local value not yet used by some connection 73 * in this link group 74 */ 75 sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */ 76 while (!conn->alert_token_local) { 77 conn->alert_token_local = atomic_inc_return(&nexttoken); 78 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) 79 conn->alert_token_local = 0; 80 } 81 smc_lgr_add_alert_token(conn); 82 conn->lgr->conns_num++; 83 } 84 85 /* Unregister connection and reset the alert token of the given connection< 86 */ 87 static void __smc_lgr_unregister_conn(struct smc_connection *conn) 88 { 89 struct smc_sock *smc = container_of(conn, struct smc_sock, conn); 90 struct smc_link_group *lgr = conn->lgr; 91 92 rb_erase(&conn->alert_node, &lgr->conns_all); 93 lgr->conns_num--; 94 conn->alert_token_local = 0; 95 conn->lgr = NULL; 96 sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */ 97 } 98 99 /* Unregister connection and trigger lgr freeing if applicable 100 */ 101 static void smc_lgr_unregister_conn(struct smc_connection *conn) 102 { 103 struct smc_link_group *lgr = conn->lgr; 104 int reduced = 0; 105 106 write_lock_bh(&lgr->conns_lock); 107 if (conn->alert_token_local) { 108 reduced = 1; 109 __smc_lgr_unregister_conn(conn); 110 } 111 write_unlock_bh(&lgr->conns_lock); 112 if (!reduced || lgr->conns_num) 113 return; 114 /* client link group creation always follows the server link group 115 * creation. For client use a somewhat higher removal delay time, 116 * otherwise there is a risk of out-of-sync link groups. 117 */ 118 mod_delayed_work(system_wq, &lgr->free_work, 119 lgr->role == SMC_CLNT ? SMC_LGR_FREE_DELAY_CLNT : 120 SMC_LGR_FREE_DELAY_SERV); 121 } 122 123 static void smc_lgr_free_work(struct work_struct *work) 124 { 125 struct smc_link_group *lgr = container_of(to_delayed_work(work), 126 struct smc_link_group, 127 free_work); 128 bool conns; 129 130 spin_lock_bh(&smc_lgr_list.lock); 131 if (list_empty(&lgr->list)) 132 goto free; 133 read_lock_bh(&lgr->conns_lock); 134 conns = RB_EMPTY_ROOT(&lgr->conns_all); 135 read_unlock_bh(&lgr->conns_lock); 136 if (!conns) { /* number of lgr connections is no longer zero */ 137 spin_unlock_bh(&smc_lgr_list.lock); 138 return; 139 } 140 list_del_init(&lgr->list); /* remove from smc_lgr_list */ 141 free: 142 spin_unlock_bh(&smc_lgr_list.lock); 143 smc_lgr_free(lgr); 144 } 145 146 /* create a new SMC link group */ 147 static int smc_lgr_create(struct smc_sock *smc, __be32 peer_in_addr, 148 struct smc_ib_device *smcibdev, u8 ibport, 149 char *peer_systemid, unsigned short vlan_id) 150 { 151 struct smc_link_group *lgr; 152 struct smc_link *lnk; 153 u8 rndvec[3]; 154 int rc = 0; 155 int i; 156 157 lgr = kzalloc(sizeof(*lgr), GFP_KERNEL); 158 if (!lgr) { 159 rc = -ENOMEM; 160 goto out; 161 } 162 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; 163 lgr->sync_err = false; 164 lgr->daddr = peer_in_addr; 165 memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN); 166 lgr->vlan_id = vlan_id; 167 rwlock_init(&lgr->sndbufs_lock); 168 rwlock_init(&lgr->rmbs_lock); 169 for (i = 0; i < SMC_RMBE_SIZES; i++) { 170 INIT_LIST_HEAD(&lgr->sndbufs[i]); 171 INIT_LIST_HEAD(&lgr->rmbs[i]); 172 } 173 smc_lgr_num += SMC_LGR_NUM_INCR; 174 memcpy(&lgr->id, (u8 *)&smc_lgr_num, SMC_LGR_ID_SIZE); 175 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); 176 lgr->conns_all = RB_ROOT; 177 178 lnk = &lgr->lnk[SMC_SINGLE_LINK]; 179 /* initialize link */ 180 lnk->link_id = SMC_SINGLE_LINK; 181 lnk->smcibdev = smcibdev; 182 lnk->ibport = ibport; 183 lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu; 184 if (!smcibdev->initialized) 185 smc_ib_setup_per_ibdev(smcibdev); 186 get_random_bytes(rndvec, sizeof(rndvec)); 187 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + (rndvec[2] << 16); 188 rc = smc_wr_alloc_link_mem(lnk); 189 if (rc) 190 goto free_lgr; 191 rc = smc_ib_create_protection_domain(lnk); 192 if (rc) 193 goto free_link_mem; 194 rc = smc_ib_create_queue_pair(lnk); 195 if (rc) 196 goto dealloc_pd; 197 rc = smc_wr_create_link(lnk); 198 if (rc) 199 goto destroy_qp; 200 init_completion(&lnk->llc_confirm); 201 init_completion(&lnk->llc_confirm_resp); 202 203 smc->conn.lgr = lgr; 204 rwlock_init(&lgr->conns_lock); 205 spin_lock_bh(&smc_lgr_list.lock); 206 list_add(&lgr->list, &smc_lgr_list.list); 207 spin_unlock_bh(&smc_lgr_list.lock); 208 return 0; 209 210 destroy_qp: 211 smc_ib_destroy_queue_pair(lnk); 212 dealloc_pd: 213 smc_ib_dealloc_protection_domain(lnk); 214 free_link_mem: 215 smc_wr_free_link_mem(lnk); 216 free_lgr: 217 kfree(lgr); 218 out: 219 return rc; 220 } 221 222 static void smc_buf_unuse(struct smc_connection *conn) 223 { 224 if (conn->sndbuf_desc) { 225 conn->sndbuf_desc->used = 0; 226 conn->sndbuf_size = 0; 227 } 228 if (conn->rmb_desc) { 229 conn->rmb_desc->reused = true; 230 conn->rmb_desc->used = 0; 231 conn->rmbe_size = 0; 232 } 233 } 234 235 /* remove a finished connection from its link group */ 236 void smc_conn_free(struct smc_connection *conn) 237 { 238 if (!conn->lgr) 239 return; 240 smc_cdc_tx_dismiss_slots(conn); 241 smc_lgr_unregister_conn(conn); 242 smc_buf_unuse(conn); 243 } 244 245 static void smc_link_clear(struct smc_link *lnk) 246 { 247 lnk->peer_qpn = 0; 248 smc_ib_modify_qp_reset(lnk); 249 smc_wr_free_link(lnk); 250 smc_ib_destroy_queue_pair(lnk); 251 smc_ib_dealloc_protection_domain(lnk); 252 smc_wr_free_link_mem(lnk); 253 } 254 255 static void smc_buf_free(struct smc_buf_desc *buf_desc, struct smc_link *lnk, 256 bool is_rmb) 257 { 258 if (is_rmb) { 259 if (buf_desc->mr_rx[SMC_SINGLE_LINK]) 260 smc_ib_put_memory_region( 261 buf_desc->mr_rx[SMC_SINGLE_LINK]); 262 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, 263 DMA_FROM_DEVICE); 264 } else { 265 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, 266 DMA_TO_DEVICE); 267 } 268 sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]); 269 if (buf_desc->cpu_addr) 270 free_pages((unsigned long)buf_desc->cpu_addr, buf_desc->order); 271 kfree(buf_desc); 272 } 273 274 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb) 275 { 276 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 277 struct smc_buf_desc *buf_desc, *bf_desc; 278 struct list_head *buf_list; 279 int i; 280 281 for (i = 0; i < SMC_RMBE_SIZES; i++) { 282 if (is_rmb) 283 buf_list = &lgr->rmbs[i]; 284 else 285 buf_list = &lgr->sndbufs[i]; 286 list_for_each_entry_safe(buf_desc, bf_desc, buf_list, 287 list) { 288 list_del(&buf_desc->list); 289 smc_buf_free(buf_desc, lnk, is_rmb); 290 } 291 } 292 } 293 294 static void smc_lgr_free_bufs(struct smc_link_group *lgr) 295 { 296 /* free send buffers */ 297 __smc_lgr_free_bufs(lgr, false); 298 /* free rmbs */ 299 __smc_lgr_free_bufs(lgr, true); 300 } 301 302 /* remove a link group */ 303 void smc_lgr_free(struct smc_link_group *lgr) 304 { 305 smc_lgr_free_bufs(lgr); 306 smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]); 307 kfree(lgr); 308 } 309 310 /* terminate linkgroup abnormally */ 311 void smc_lgr_terminate(struct smc_link_group *lgr) 312 { 313 struct smc_connection *conn; 314 struct smc_sock *smc; 315 struct rb_node *node; 316 317 spin_lock_bh(&smc_lgr_list.lock); 318 if (list_empty(&lgr->list)) { 319 /* termination already triggered */ 320 spin_unlock_bh(&smc_lgr_list.lock); 321 return; 322 } 323 /* do not use this link group for new connections */ 324 list_del_init(&lgr->list); 325 spin_unlock_bh(&smc_lgr_list.lock); 326 327 write_lock_bh(&lgr->conns_lock); 328 node = rb_first(&lgr->conns_all); 329 while (node) { 330 conn = rb_entry(node, struct smc_connection, alert_node); 331 smc = container_of(conn, struct smc_sock, conn); 332 sock_hold(&smc->sk); /* sock_put in close work */ 333 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; 334 __smc_lgr_unregister_conn(conn); 335 write_unlock_bh(&lgr->conns_lock); 336 if (!schedule_work(&conn->close_work)) 337 sock_put(&smc->sk); 338 write_lock_bh(&lgr->conns_lock); 339 node = rb_first(&lgr->conns_all); 340 } 341 write_unlock_bh(&lgr->conns_lock); 342 wake_up(&lgr->lnk[SMC_SINGLE_LINK].wr_reg_wait); 343 } 344 345 /* Determine vlan of internal TCP socket. 346 * @vlan_id: address to store the determined vlan id into 347 */ 348 static int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id) 349 { 350 struct dst_entry *dst = sk_dst_get(clcsock->sk); 351 int rc = 0; 352 353 *vlan_id = 0; 354 if (!dst) { 355 rc = -ENOTCONN; 356 goto out; 357 } 358 if (!dst->dev) { 359 rc = -ENODEV; 360 goto out_rel; 361 } 362 363 if (is_vlan_dev(dst->dev)) 364 *vlan_id = vlan_dev_vlan_id(dst->dev); 365 366 out_rel: 367 dst_release(dst); 368 out: 369 return rc; 370 } 371 372 /* determine the link gid matching the vlan id of the link group */ 373 static int smc_link_determine_gid(struct smc_link_group *lgr) 374 { 375 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 376 struct ib_gid_attr gattr; 377 union ib_gid gid; 378 int i; 379 380 if (!lgr->vlan_id) { 381 lnk->gid = lnk->smcibdev->gid[lnk->ibport - 1]; 382 return 0; 383 } 384 385 for (i = 0; i < lnk->smcibdev->pattr[lnk->ibport - 1].gid_tbl_len; 386 i++) { 387 if (ib_query_gid(lnk->smcibdev->ibdev, lnk->ibport, i, &gid, 388 &gattr)) 389 continue; 390 if (gattr.ndev) { 391 if (is_vlan_dev(gattr.ndev) && 392 vlan_dev_vlan_id(gattr.ndev) == lgr->vlan_id) { 393 lnk->gid = gid; 394 dev_put(gattr.ndev); 395 return 0; 396 } 397 dev_put(gattr.ndev); 398 } 399 } 400 return -ENODEV; 401 } 402 403 /* create a new SMC connection (and a new link group if necessary) */ 404 int smc_conn_create(struct smc_sock *smc, __be32 peer_in_addr, 405 struct smc_ib_device *smcibdev, u8 ibport, 406 struct smc_clc_msg_local *lcl, int srv_first_contact) 407 { 408 struct smc_connection *conn = &smc->conn; 409 struct smc_link_group *lgr; 410 unsigned short vlan_id; 411 enum smc_lgr_role role; 412 int local_contact = SMC_FIRST_CONTACT; 413 int rc = 0; 414 415 role = smc->listen_smc ? SMC_SERV : SMC_CLNT; 416 rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id); 417 if (rc) 418 return rc; 419 420 if ((role == SMC_CLNT) && srv_first_contact) 421 /* create new link group as well */ 422 goto create; 423 424 /* determine if an existing link group can be reused */ 425 spin_lock_bh(&smc_lgr_list.lock); 426 list_for_each_entry(lgr, &smc_lgr_list.list, list) { 427 write_lock_bh(&lgr->conns_lock); 428 if (!memcmp(lgr->peer_systemid, lcl->id_for_peer, 429 SMC_SYSTEMID_LEN) && 430 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid, 431 SMC_GID_SIZE) && 432 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac, 433 sizeof(lcl->mac)) && 434 !lgr->sync_err && 435 (lgr->role == role) && 436 (lgr->vlan_id == vlan_id) && 437 ((role == SMC_CLNT) || 438 (lgr->conns_num < SMC_RMBS_PER_LGR_MAX))) { 439 /* link group found */ 440 local_contact = SMC_REUSE_CONTACT; 441 conn->lgr = lgr; 442 smc_lgr_register_conn(conn); /* add smc conn to lgr */ 443 write_unlock_bh(&lgr->conns_lock); 444 break; 445 } 446 write_unlock_bh(&lgr->conns_lock); 447 } 448 spin_unlock_bh(&smc_lgr_list.lock); 449 450 if (role == SMC_CLNT && !srv_first_contact && 451 (local_contact == SMC_FIRST_CONTACT)) { 452 /* Server reuses a link group, but Client wants to start 453 * a new one 454 * send out_of_sync decline, reason synchr. error 455 */ 456 return -ENOLINK; 457 } 458 459 create: 460 if (local_contact == SMC_FIRST_CONTACT) { 461 rc = smc_lgr_create(smc, peer_in_addr, smcibdev, ibport, 462 lcl->id_for_peer, vlan_id); 463 if (rc) 464 goto out; 465 smc_lgr_register_conn(conn); /* add smc conn to lgr */ 466 rc = smc_link_determine_gid(conn->lgr); 467 } 468 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE; 469 conn->local_tx_ctrl.len = SMC_WR_TX_SIZE; 470 #ifndef KERNEL_HAS_ATOMIC64 471 spin_lock_init(&conn->acurs_lock); 472 #endif 473 474 out: 475 return rc ? rc : local_contact; 476 } 477 478 /* try to reuse a sndbuf or rmb description slot for a certain 479 * buffer size; if not available, return NULL 480 */ 481 static inline 482 struct smc_buf_desc *smc_buf_get_slot(struct smc_link_group *lgr, 483 int compressed_bufsize, 484 rwlock_t *lock, 485 struct list_head *buf_list) 486 { 487 struct smc_buf_desc *buf_slot; 488 489 read_lock_bh(lock); 490 list_for_each_entry(buf_slot, buf_list, list) { 491 if (cmpxchg(&buf_slot->used, 0, 1) == 0) { 492 read_unlock_bh(lock); 493 return buf_slot; 494 } 495 } 496 read_unlock_bh(lock); 497 return NULL; 498 } 499 500 /* one of the conditions for announcing a receiver's current window size is 501 * that it "results in a minimum increase in the window size of 10% of the 502 * receive buffer space" [RFC7609] 503 */ 504 static inline int smc_rmb_wnd_update_limit(int rmbe_size) 505 { 506 return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2); 507 } 508 509 static struct smc_buf_desc *smc_new_buf_create(struct smc_link_group *lgr, 510 bool is_rmb, int bufsize) 511 { 512 struct smc_buf_desc *buf_desc; 513 struct smc_link *lnk; 514 int rc; 515 516 /* try to alloc a new buffer */ 517 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); 518 if (!buf_desc) 519 return ERR_PTR(-ENOMEM); 520 521 buf_desc->cpu_addr = 522 (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | 523 __GFP_NOMEMALLOC | 524 __GFP_NORETRY | __GFP_ZERO, 525 get_order(bufsize)); 526 if (!buf_desc->cpu_addr) { 527 kfree(buf_desc); 528 return ERR_PTR(-EAGAIN); 529 } 530 buf_desc->order = get_order(bufsize); 531 532 /* build the sg table from the pages */ 533 lnk = &lgr->lnk[SMC_SINGLE_LINK]; 534 rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1, 535 GFP_KERNEL); 536 if (rc) { 537 smc_buf_free(buf_desc, lnk, is_rmb); 538 return ERR_PTR(rc); 539 } 540 sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl, 541 buf_desc->cpu_addr, bufsize); 542 543 /* map sg table to DMA address */ 544 rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc, 545 is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); 546 /* SMC protocol depends on mapping to one DMA address only */ 547 if (rc != 1) { 548 smc_buf_free(buf_desc, lnk, is_rmb); 549 return ERR_PTR(-EAGAIN); 550 } 551 552 /* create a new memory region for the RMB */ 553 if (is_rmb) { 554 rc = smc_ib_get_memory_region(lnk->roce_pd, 555 IB_ACCESS_REMOTE_WRITE | 556 IB_ACCESS_LOCAL_WRITE, 557 buf_desc); 558 if (rc) { 559 smc_buf_free(buf_desc, lnk, is_rmb); 560 return ERR_PTR(rc); 561 } 562 } 563 564 return buf_desc; 565 } 566 567 static int __smc_buf_create(struct smc_sock *smc, bool is_rmb) 568 { 569 struct smc_connection *conn = &smc->conn; 570 struct smc_link_group *lgr = conn->lgr; 571 struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM); 572 struct list_head *buf_list; 573 int bufsize, bufsize_short; 574 int sk_buf_size; 575 rwlock_t *lock; 576 577 if (is_rmb) 578 /* use socket recv buffer size (w/o overhead) as start value */ 579 sk_buf_size = smc->sk.sk_rcvbuf / 2; 580 else 581 /* use socket send buffer size (w/o overhead) as start value */ 582 sk_buf_size = smc->sk.sk_sndbuf / 2; 583 584 for (bufsize_short = smc_compress_bufsize(sk_buf_size); 585 bufsize_short >= 0; bufsize_short--) { 586 587 if (is_rmb) { 588 lock = &lgr->rmbs_lock; 589 buf_list = &lgr->rmbs[bufsize_short]; 590 } else { 591 lock = &lgr->sndbufs_lock; 592 buf_list = &lgr->sndbufs[bufsize_short]; 593 } 594 bufsize = smc_uncompress_bufsize(bufsize_short); 595 if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC) 596 continue; 597 598 /* check for reusable slot in the link group */ 599 buf_desc = smc_buf_get_slot(lgr, bufsize_short, lock, buf_list); 600 if (buf_desc) { 601 memset(buf_desc->cpu_addr, 0, bufsize); 602 break; /* found reusable slot */ 603 } 604 605 buf_desc = smc_new_buf_create(lgr, is_rmb, bufsize); 606 if (PTR_ERR(buf_desc) == -ENOMEM) 607 break; 608 if (IS_ERR(buf_desc)) 609 continue; 610 611 buf_desc->used = 1; 612 write_lock_bh(lock); 613 list_add(&buf_desc->list, buf_list); 614 write_unlock_bh(lock); 615 break; /* found */ 616 } 617 618 if (IS_ERR(buf_desc)) 619 return -ENOMEM; 620 621 if (is_rmb) { 622 conn->rmb_desc = buf_desc; 623 conn->rmbe_size = bufsize; 624 conn->rmbe_size_short = bufsize_short; 625 smc->sk.sk_rcvbuf = bufsize * 2; 626 atomic_set(&conn->bytes_to_rcv, 0); 627 conn->rmbe_update_limit = smc_rmb_wnd_update_limit(bufsize); 628 } else { 629 conn->sndbuf_desc = buf_desc; 630 conn->sndbuf_size = bufsize; 631 smc->sk.sk_sndbuf = bufsize * 2; 632 atomic_set(&conn->sndbuf_space, bufsize); 633 } 634 return 0; 635 } 636 637 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn) 638 { 639 struct smc_link_group *lgr = conn->lgr; 640 641 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 642 conn->sndbuf_desc, DMA_TO_DEVICE); 643 } 644 645 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn) 646 { 647 struct smc_link_group *lgr = conn->lgr; 648 649 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 650 conn->sndbuf_desc, DMA_TO_DEVICE); 651 } 652 653 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn) 654 { 655 struct smc_link_group *lgr = conn->lgr; 656 657 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 658 conn->rmb_desc, DMA_FROM_DEVICE); 659 } 660 661 void smc_rmb_sync_sg_for_device(struct smc_connection *conn) 662 { 663 struct smc_link_group *lgr = conn->lgr; 664 665 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 666 conn->rmb_desc, DMA_FROM_DEVICE); 667 } 668 669 /* create the send and receive buffer for an SMC socket; 670 * receive buffers are called RMBs; 671 * (even though the SMC protocol allows more than one RMB-element per RMB, 672 * the Linux implementation uses just one RMB-element per RMB, i.e. uses an 673 * extra RMB for every connection in a link group 674 */ 675 int smc_buf_create(struct smc_sock *smc) 676 { 677 int rc; 678 679 /* create send buffer */ 680 rc = __smc_buf_create(smc, false); 681 if (rc) 682 return rc; 683 /* create rmb */ 684 rc = __smc_buf_create(smc, true); 685 if (rc) 686 smc_buf_free(smc->conn.sndbuf_desc, 687 &smc->conn.lgr->lnk[SMC_SINGLE_LINK], false); 688 return rc; 689 } 690 691 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr) 692 { 693 int i; 694 695 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { 696 if (!test_and_set_bit(i, lgr->rtokens_used_mask)) 697 return i; 698 } 699 return -ENOSPC; 700 } 701 702 /* save rkey and dma_addr received from peer during clc handshake */ 703 int smc_rmb_rtoken_handling(struct smc_connection *conn, 704 struct smc_clc_msg_accept_confirm *clc) 705 { 706 u64 dma_addr = be64_to_cpu(clc->rmb_dma_addr); 707 struct smc_link_group *lgr = conn->lgr; 708 u32 rkey = ntohl(clc->rmb_rkey); 709 int i; 710 711 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { 712 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) && 713 (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) && 714 test_bit(i, lgr->rtokens_used_mask)) { 715 conn->rtoken_idx = i; 716 return 0; 717 } 718 } 719 conn->rtoken_idx = smc_rmb_reserve_rtoken_idx(lgr); 720 if (conn->rtoken_idx < 0) 721 return conn->rtoken_idx; 722 lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].rkey = rkey; 723 lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].dma_addr = dma_addr; 724 return 0; 725 } 726