1 /* 2 * Copyright (c) 2005 Voltaire Inc. All rights reserved. 3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. 4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. 5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 6 * 7 * This Software is licensed under one of the following licenses: 8 * 9 * 1) under the terms of the "Common Public License 1.0" a copy of which is 10 * available from the Open Source Initiative, see 11 * http://www.opensource.org/licenses/cpl.php. 12 * 13 * 2) under the terms of the "The BSD License" a copy of which is 14 * available from the Open Source Initiative, see 15 * http://www.opensource.org/licenses/bsd-license.php. 16 * 17 * 3) under the terms of the "GNU General Public License (GPL) Version 2" a 18 * copy of which is available from the Open Source Initiative, see 19 * http://www.opensource.org/licenses/gpl-license.php. 20 * 21 * Licensee has the right to choose one of the above licenses. 22 * 23 * Redistributions of source code must retain the above copyright 24 * notice and one of the license notices. 25 * 26 * Redistributions in binary form must reproduce both the above copyright 27 * notice, one of the license notices in the documentation 28 * and/or other materials provided with the distribution. 29 * 30 */ 31 32 #include <linux/completion.h> 33 #include <linux/in.h> 34 #include <linux/in6.h> 35 #include <linux/mutex.h> 36 #include <linux/random.h> 37 #include <linux/idr.h> 38 #include <linux/inetdevice.h> 39 40 #include <net/tcp.h> 41 42 #include <rdma/rdma_cm.h> 43 #include <rdma/rdma_cm_ib.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/ib_cm.h> 46 #include <rdma/ib_sa.h> 47 #include <rdma/iw_cm.h> 48 49 MODULE_AUTHOR("Sean Hefty"); 50 MODULE_DESCRIPTION("Generic RDMA CM Agent"); 51 MODULE_LICENSE("Dual BSD/GPL"); 52 53 #define CMA_CM_RESPONSE_TIMEOUT 20 54 #define CMA_MAX_CM_RETRIES 15 55 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) 56 57 static void cma_add_one(struct ib_device *device); 58 static void cma_remove_one(struct ib_device *device); 59 60 static struct ib_client cma_client = { 61 .name = "cma", 62 .add = cma_add_one, 63 .remove = cma_remove_one 64 }; 65 66 static struct ib_sa_client sa_client; 67 static struct rdma_addr_client addr_client; 68 static LIST_HEAD(dev_list); 69 static LIST_HEAD(listen_any_list); 70 static DEFINE_MUTEX(lock); 71 static struct workqueue_struct *cma_wq; 72 static DEFINE_IDR(sdp_ps); 73 static DEFINE_IDR(tcp_ps); 74 static DEFINE_IDR(udp_ps); 75 static DEFINE_IDR(ipoib_ps); 76 static int next_port; 77 78 struct cma_device { 79 struct list_head list; 80 struct ib_device *device; 81 struct completion comp; 82 atomic_t refcount; 83 struct list_head id_list; 84 }; 85 86 enum cma_state { 87 CMA_IDLE, 88 CMA_ADDR_QUERY, 89 CMA_ADDR_RESOLVED, 90 CMA_ROUTE_QUERY, 91 CMA_ROUTE_RESOLVED, 92 CMA_CONNECT, 93 CMA_DISCONNECT, 94 CMA_ADDR_BOUND, 95 CMA_LISTEN, 96 CMA_DEVICE_REMOVAL, 97 CMA_DESTROYING 98 }; 99 100 struct rdma_bind_list { 101 struct idr *ps; 102 struct hlist_head owners; 103 unsigned short port; 104 }; 105 106 /* 107 * Device removal can occur at anytime, so we need extra handling to 108 * serialize notifying the user of device removal with other callbacks. 109 * We do this by disabling removal notification while a callback is in process, 110 * and reporting it after the callback completes. 111 */ 112 struct rdma_id_private { 113 struct rdma_cm_id id; 114 115 struct rdma_bind_list *bind_list; 116 struct hlist_node node; 117 struct list_head list; /* listen_any_list or cma_device.list */ 118 struct list_head listen_list; /* per device listens */ 119 struct cma_device *cma_dev; 120 struct list_head mc_list; 121 122 int internal_id; 123 enum cma_state state; 124 spinlock_t lock; 125 struct mutex qp_mutex; 126 127 struct completion comp; 128 atomic_t refcount; 129 wait_queue_head_t wait_remove; 130 atomic_t dev_remove; 131 132 int backlog; 133 int timeout_ms; 134 struct ib_sa_query *query; 135 int query_id; 136 union { 137 struct ib_cm_id *ib; 138 struct iw_cm_id *iw; 139 } cm_id; 140 141 u32 seq_num; 142 u32 qkey; 143 u32 qp_num; 144 u8 srq; 145 u8 tos; 146 }; 147 148 struct cma_multicast { 149 struct rdma_id_private *id_priv; 150 union { 151 struct ib_sa_multicast *ib; 152 } multicast; 153 struct list_head list; 154 void *context; 155 struct sockaddr addr; 156 u8 pad[sizeof(struct sockaddr_in6) - 157 sizeof(struct sockaddr)]; 158 }; 159 160 struct cma_work { 161 struct work_struct work; 162 struct rdma_id_private *id; 163 enum cma_state old_state; 164 enum cma_state new_state; 165 struct rdma_cm_event event; 166 }; 167 168 union cma_ip_addr { 169 struct in6_addr ip6; 170 struct { 171 __u32 pad[3]; 172 __u32 addr; 173 } ip4; 174 }; 175 176 struct cma_hdr { 177 u8 cma_version; 178 u8 ip_version; /* IP version: 7:4 */ 179 __u16 port; 180 union cma_ip_addr src_addr; 181 union cma_ip_addr dst_addr; 182 }; 183 184 struct sdp_hh { 185 u8 bsdh[16]; 186 u8 sdp_version; /* Major version: 7:4 */ 187 u8 ip_version; /* IP version: 7:4 */ 188 u8 sdp_specific1[10]; 189 __u16 port; 190 __u16 sdp_specific2; 191 union cma_ip_addr src_addr; 192 union cma_ip_addr dst_addr; 193 }; 194 195 struct sdp_hah { 196 u8 bsdh[16]; 197 u8 sdp_version; 198 }; 199 200 #define CMA_VERSION 0x00 201 #define SDP_MAJ_VERSION 0x2 202 203 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp) 204 { 205 unsigned long flags; 206 int ret; 207 208 spin_lock_irqsave(&id_priv->lock, flags); 209 ret = (id_priv->state == comp); 210 spin_unlock_irqrestore(&id_priv->lock, flags); 211 return ret; 212 } 213 214 static int cma_comp_exch(struct rdma_id_private *id_priv, 215 enum cma_state comp, enum cma_state exch) 216 { 217 unsigned long flags; 218 int ret; 219 220 spin_lock_irqsave(&id_priv->lock, flags); 221 if ((ret = (id_priv->state == comp))) 222 id_priv->state = exch; 223 spin_unlock_irqrestore(&id_priv->lock, flags); 224 return ret; 225 } 226 227 static enum cma_state cma_exch(struct rdma_id_private *id_priv, 228 enum cma_state exch) 229 { 230 unsigned long flags; 231 enum cma_state old; 232 233 spin_lock_irqsave(&id_priv->lock, flags); 234 old = id_priv->state; 235 id_priv->state = exch; 236 spin_unlock_irqrestore(&id_priv->lock, flags); 237 return old; 238 } 239 240 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr) 241 { 242 return hdr->ip_version >> 4; 243 } 244 245 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) 246 { 247 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); 248 } 249 250 static inline u8 sdp_get_majv(u8 sdp_version) 251 { 252 return sdp_version >> 4; 253 } 254 255 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh) 256 { 257 return hh->ip_version >> 4; 258 } 259 260 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver) 261 { 262 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF); 263 } 264 265 static inline int cma_is_ud_ps(enum rdma_port_space ps) 266 { 267 return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB); 268 } 269 270 static void cma_attach_to_dev(struct rdma_id_private *id_priv, 271 struct cma_device *cma_dev) 272 { 273 atomic_inc(&cma_dev->refcount); 274 id_priv->cma_dev = cma_dev; 275 id_priv->id.device = cma_dev->device; 276 list_add_tail(&id_priv->list, &cma_dev->id_list); 277 } 278 279 static inline void cma_deref_dev(struct cma_device *cma_dev) 280 { 281 if (atomic_dec_and_test(&cma_dev->refcount)) 282 complete(&cma_dev->comp); 283 } 284 285 static void cma_detach_from_dev(struct rdma_id_private *id_priv) 286 { 287 list_del(&id_priv->list); 288 cma_deref_dev(id_priv->cma_dev); 289 id_priv->cma_dev = NULL; 290 } 291 292 static int cma_set_qkey(struct ib_device *device, u8 port_num, 293 enum rdma_port_space ps, 294 struct rdma_dev_addr *dev_addr, u32 *qkey) 295 { 296 struct ib_sa_mcmember_rec rec; 297 int ret = 0; 298 299 switch (ps) { 300 case RDMA_PS_UDP: 301 *qkey = RDMA_UDP_QKEY; 302 break; 303 case RDMA_PS_IPOIB: 304 ib_addr_get_mgid(dev_addr, &rec.mgid); 305 ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec); 306 *qkey = be32_to_cpu(rec.qkey); 307 break; 308 default: 309 break; 310 } 311 return ret; 312 } 313 314 static int cma_acquire_dev(struct rdma_id_private *id_priv) 315 { 316 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 317 struct cma_device *cma_dev; 318 union ib_gid gid; 319 int ret = -ENODEV; 320 321 switch (rdma_node_get_transport(dev_addr->dev_type)) { 322 case RDMA_TRANSPORT_IB: 323 ib_addr_get_sgid(dev_addr, &gid); 324 break; 325 case RDMA_TRANSPORT_IWARP: 326 iw_addr_get_sgid(dev_addr, &gid); 327 break; 328 default: 329 return -ENODEV; 330 } 331 332 list_for_each_entry(cma_dev, &dev_list, list) { 333 ret = ib_find_cached_gid(cma_dev->device, &gid, 334 &id_priv->id.port_num, NULL); 335 if (!ret) { 336 ret = cma_set_qkey(cma_dev->device, 337 id_priv->id.port_num, 338 id_priv->id.ps, dev_addr, 339 &id_priv->qkey); 340 if (!ret) 341 cma_attach_to_dev(id_priv, cma_dev); 342 break; 343 } 344 } 345 return ret; 346 } 347 348 static void cma_deref_id(struct rdma_id_private *id_priv) 349 { 350 if (atomic_dec_and_test(&id_priv->refcount)) 351 complete(&id_priv->comp); 352 } 353 354 static int cma_disable_remove(struct rdma_id_private *id_priv, 355 enum cma_state state) 356 { 357 unsigned long flags; 358 int ret; 359 360 spin_lock_irqsave(&id_priv->lock, flags); 361 if (id_priv->state == state) { 362 atomic_inc(&id_priv->dev_remove); 363 ret = 0; 364 } else 365 ret = -EINVAL; 366 spin_unlock_irqrestore(&id_priv->lock, flags); 367 return ret; 368 } 369 370 static void cma_enable_remove(struct rdma_id_private *id_priv) 371 { 372 if (atomic_dec_and_test(&id_priv->dev_remove)) 373 wake_up(&id_priv->wait_remove); 374 } 375 376 static int cma_has_cm_dev(struct rdma_id_private *id_priv) 377 { 378 return (id_priv->id.device && id_priv->cm_id.ib); 379 } 380 381 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler, 382 void *context, enum rdma_port_space ps) 383 { 384 struct rdma_id_private *id_priv; 385 386 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); 387 if (!id_priv) 388 return ERR_PTR(-ENOMEM); 389 390 id_priv->state = CMA_IDLE; 391 id_priv->id.context = context; 392 id_priv->id.event_handler = event_handler; 393 id_priv->id.ps = ps; 394 spin_lock_init(&id_priv->lock); 395 mutex_init(&id_priv->qp_mutex); 396 init_completion(&id_priv->comp); 397 atomic_set(&id_priv->refcount, 1); 398 init_waitqueue_head(&id_priv->wait_remove); 399 atomic_set(&id_priv->dev_remove, 0); 400 INIT_LIST_HEAD(&id_priv->listen_list); 401 INIT_LIST_HEAD(&id_priv->mc_list); 402 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); 403 404 return &id_priv->id; 405 } 406 EXPORT_SYMBOL(rdma_create_id); 407 408 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 409 { 410 struct ib_qp_attr qp_attr; 411 int qp_attr_mask, ret; 412 413 qp_attr.qp_state = IB_QPS_INIT; 414 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 415 if (ret) 416 return ret; 417 418 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); 419 if (ret) 420 return ret; 421 422 qp_attr.qp_state = IB_QPS_RTR; 423 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); 424 if (ret) 425 return ret; 426 427 qp_attr.qp_state = IB_QPS_RTS; 428 qp_attr.sq_psn = 0; 429 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); 430 431 return ret; 432 } 433 434 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 435 { 436 struct ib_qp_attr qp_attr; 437 int qp_attr_mask, ret; 438 439 qp_attr.qp_state = IB_QPS_INIT; 440 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 441 if (ret) 442 return ret; 443 444 return ib_modify_qp(qp, &qp_attr, qp_attr_mask); 445 } 446 447 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, 448 struct ib_qp_init_attr *qp_init_attr) 449 { 450 struct rdma_id_private *id_priv; 451 struct ib_qp *qp; 452 int ret; 453 454 id_priv = container_of(id, struct rdma_id_private, id); 455 if (id->device != pd->device) 456 return -EINVAL; 457 458 qp = ib_create_qp(pd, qp_init_attr); 459 if (IS_ERR(qp)) 460 return PTR_ERR(qp); 461 462 if (cma_is_ud_ps(id_priv->id.ps)) 463 ret = cma_init_ud_qp(id_priv, qp); 464 else 465 ret = cma_init_conn_qp(id_priv, qp); 466 if (ret) 467 goto err; 468 469 id->qp = qp; 470 id_priv->qp_num = qp->qp_num; 471 id_priv->srq = (qp->srq != NULL); 472 return 0; 473 err: 474 ib_destroy_qp(qp); 475 return ret; 476 } 477 EXPORT_SYMBOL(rdma_create_qp); 478 479 void rdma_destroy_qp(struct rdma_cm_id *id) 480 { 481 struct rdma_id_private *id_priv; 482 483 id_priv = container_of(id, struct rdma_id_private, id); 484 mutex_lock(&id_priv->qp_mutex); 485 ib_destroy_qp(id_priv->id.qp); 486 id_priv->id.qp = NULL; 487 mutex_unlock(&id_priv->qp_mutex); 488 } 489 EXPORT_SYMBOL(rdma_destroy_qp); 490 491 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv) 492 { 493 struct ib_qp_attr qp_attr; 494 int qp_attr_mask, ret; 495 496 mutex_lock(&id_priv->qp_mutex); 497 if (!id_priv->id.qp) { 498 ret = 0; 499 goto out; 500 } 501 502 /* Need to update QP attributes from default values. */ 503 qp_attr.qp_state = IB_QPS_INIT; 504 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 505 if (ret) 506 goto out; 507 508 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 509 if (ret) 510 goto out; 511 512 qp_attr.qp_state = IB_QPS_RTR; 513 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 514 if (ret) 515 goto out; 516 517 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 518 out: 519 mutex_unlock(&id_priv->qp_mutex); 520 return ret; 521 } 522 523 static int cma_modify_qp_rts(struct rdma_id_private *id_priv) 524 { 525 struct ib_qp_attr qp_attr; 526 int qp_attr_mask, ret; 527 528 mutex_lock(&id_priv->qp_mutex); 529 if (!id_priv->id.qp) { 530 ret = 0; 531 goto out; 532 } 533 534 qp_attr.qp_state = IB_QPS_RTS; 535 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 536 if (ret) 537 goto out; 538 539 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 540 out: 541 mutex_unlock(&id_priv->qp_mutex); 542 return ret; 543 } 544 545 static int cma_modify_qp_err(struct rdma_id_private *id_priv) 546 { 547 struct ib_qp_attr qp_attr; 548 int ret; 549 550 mutex_lock(&id_priv->qp_mutex); 551 if (!id_priv->id.qp) { 552 ret = 0; 553 goto out; 554 } 555 556 qp_attr.qp_state = IB_QPS_ERR; 557 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); 558 out: 559 mutex_unlock(&id_priv->qp_mutex); 560 return ret; 561 } 562 563 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, 564 struct ib_qp_attr *qp_attr, int *qp_attr_mask) 565 { 566 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 567 int ret; 568 569 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, 570 ib_addr_get_pkey(dev_addr), 571 &qp_attr->pkey_index); 572 if (ret) 573 return ret; 574 575 qp_attr->port_num = id_priv->id.port_num; 576 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; 577 578 if (cma_is_ud_ps(id_priv->id.ps)) { 579 qp_attr->qkey = id_priv->qkey; 580 *qp_attr_mask |= IB_QP_QKEY; 581 } else { 582 qp_attr->qp_access_flags = 0; 583 *qp_attr_mask |= IB_QP_ACCESS_FLAGS; 584 } 585 return 0; 586 } 587 588 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, 589 int *qp_attr_mask) 590 { 591 struct rdma_id_private *id_priv; 592 int ret = 0; 593 594 id_priv = container_of(id, struct rdma_id_private, id); 595 switch (rdma_node_get_transport(id_priv->id.device->node_type)) { 596 case RDMA_TRANSPORT_IB: 597 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps)) 598 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); 599 else 600 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, 601 qp_attr_mask); 602 if (qp_attr->qp_state == IB_QPS_RTR) 603 qp_attr->rq_psn = id_priv->seq_num; 604 break; 605 case RDMA_TRANSPORT_IWARP: 606 if (!id_priv->cm_id.iw) { 607 qp_attr->qp_access_flags = 0; 608 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; 609 } else 610 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, 611 qp_attr_mask); 612 break; 613 default: 614 ret = -ENOSYS; 615 break; 616 } 617 618 return ret; 619 } 620 EXPORT_SYMBOL(rdma_init_qp_attr); 621 622 static inline int cma_zero_addr(struct sockaddr *addr) 623 { 624 struct in6_addr *ip6; 625 626 if (addr->sa_family == AF_INET) 627 return ZERONET(((struct sockaddr_in *) addr)->sin_addr.s_addr); 628 else { 629 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr; 630 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] | 631 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0; 632 } 633 } 634 635 static inline int cma_loopback_addr(struct sockaddr *addr) 636 { 637 return LOOPBACK(((struct sockaddr_in *) addr)->sin_addr.s_addr); 638 } 639 640 static inline int cma_any_addr(struct sockaddr *addr) 641 { 642 return cma_zero_addr(addr) || cma_loopback_addr(addr); 643 } 644 645 static inline __be16 cma_port(struct sockaddr *addr) 646 { 647 if (addr->sa_family == AF_INET) 648 return ((struct sockaddr_in *) addr)->sin_port; 649 else 650 return ((struct sockaddr_in6 *) addr)->sin6_port; 651 } 652 653 static inline int cma_any_port(struct sockaddr *addr) 654 { 655 return !cma_port(addr); 656 } 657 658 static int cma_get_net_info(void *hdr, enum rdma_port_space ps, 659 u8 *ip_ver, __u16 *port, 660 union cma_ip_addr **src, union cma_ip_addr **dst) 661 { 662 switch (ps) { 663 case RDMA_PS_SDP: 664 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) != 665 SDP_MAJ_VERSION) 666 return -EINVAL; 667 668 *ip_ver = sdp_get_ip_ver(hdr); 669 *port = ((struct sdp_hh *) hdr)->port; 670 *src = &((struct sdp_hh *) hdr)->src_addr; 671 *dst = &((struct sdp_hh *) hdr)->dst_addr; 672 break; 673 default: 674 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION) 675 return -EINVAL; 676 677 *ip_ver = cma_get_ip_ver(hdr); 678 *port = ((struct cma_hdr *) hdr)->port; 679 *src = &((struct cma_hdr *) hdr)->src_addr; 680 *dst = &((struct cma_hdr *) hdr)->dst_addr; 681 break; 682 } 683 684 if (*ip_ver != 4 && *ip_ver != 6) 685 return -EINVAL; 686 return 0; 687 } 688 689 static void cma_save_net_info(struct rdma_addr *addr, 690 struct rdma_addr *listen_addr, 691 u8 ip_ver, __u16 port, 692 union cma_ip_addr *src, union cma_ip_addr *dst) 693 { 694 struct sockaddr_in *listen4, *ip4; 695 struct sockaddr_in6 *listen6, *ip6; 696 697 switch (ip_ver) { 698 case 4: 699 listen4 = (struct sockaddr_in *) &listen_addr->src_addr; 700 ip4 = (struct sockaddr_in *) &addr->src_addr; 701 ip4->sin_family = listen4->sin_family; 702 ip4->sin_addr.s_addr = dst->ip4.addr; 703 ip4->sin_port = listen4->sin_port; 704 705 ip4 = (struct sockaddr_in *) &addr->dst_addr; 706 ip4->sin_family = listen4->sin_family; 707 ip4->sin_addr.s_addr = src->ip4.addr; 708 ip4->sin_port = port; 709 break; 710 case 6: 711 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr; 712 ip6 = (struct sockaddr_in6 *) &addr->src_addr; 713 ip6->sin6_family = listen6->sin6_family; 714 ip6->sin6_addr = dst->ip6; 715 ip6->sin6_port = listen6->sin6_port; 716 717 ip6 = (struct sockaddr_in6 *) &addr->dst_addr; 718 ip6->sin6_family = listen6->sin6_family; 719 ip6->sin6_addr = src->ip6; 720 ip6->sin6_port = port; 721 break; 722 default: 723 break; 724 } 725 } 726 727 static inline int cma_user_data_offset(enum rdma_port_space ps) 728 { 729 switch (ps) { 730 case RDMA_PS_SDP: 731 return 0; 732 default: 733 return sizeof(struct cma_hdr); 734 } 735 } 736 737 static void cma_cancel_route(struct rdma_id_private *id_priv) 738 { 739 switch (rdma_node_get_transport(id_priv->id.device->node_type)) { 740 case RDMA_TRANSPORT_IB: 741 if (id_priv->query) 742 ib_sa_cancel_query(id_priv->query_id, id_priv->query); 743 break; 744 default: 745 break; 746 } 747 } 748 749 static void cma_cancel_listens(struct rdma_id_private *id_priv) 750 { 751 struct rdma_id_private *dev_id_priv; 752 753 /* 754 * Remove from listen_any_list to prevent added devices from spawning 755 * additional listen requests. 756 */ 757 mutex_lock(&lock); 758 list_del(&id_priv->list); 759 760 while (!list_empty(&id_priv->listen_list)) { 761 dev_id_priv = list_entry(id_priv->listen_list.next, 762 struct rdma_id_private, listen_list); 763 /* sync with device removal to avoid duplicate destruction */ 764 list_del_init(&dev_id_priv->list); 765 list_del(&dev_id_priv->listen_list); 766 mutex_unlock(&lock); 767 768 rdma_destroy_id(&dev_id_priv->id); 769 mutex_lock(&lock); 770 } 771 mutex_unlock(&lock); 772 } 773 774 static void cma_cancel_operation(struct rdma_id_private *id_priv, 775 enum cma_state state) 776 { 777 switch (state) { 778 case CMA_ADDR_QUERY: 779 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); 780 break; 781 case CMA_ROUTE_QUERY: 782 cma_cancel_route(id_priv); 783 break; 784 case CMA_LISTEN: 785 if (cma_any_addr(&id_priv->id.route.addr.src_addr) && 786 !id_priv->cma_dev) 787 cma_cancel_listens(id_priv); 788 break; 789 default: 790 break; 791 } 792 } 793 794 static void cma_release_port(struct rdma_id_private *id_priv) 795 { 796 struct rdma_bind_list *bind_list = id_priv->bind_list; 797 798 if (!bind_list) 799 return; 800 801 mutex_lock(&lock); 802 hlist_del(&id_priv->node); 803 if (hlist_empty(&bind_list->owners)) { 804 idr_remove(bind_list->ps, bind_list->port); 805 kfree(bind_list); 806 } 807 mutex_unlock(&lock); 808 } 809 810 static void cma_leave_mc_groups(struct rdma_id_private *id_priv) 811 { 812 struct cma_multicast *mc; 813 814 while (!list_empty(&id_priv->mc_list)) { 815 mc = container_of(id_priv->mc_list.next, 816 struct cma_multicast, list); 817 list_del(&mc->list); 818 ib_sa_free_multicast(mc->multicast.ib); 819 kfree(mc); 820 } 821 } 822 823 void rdma_destroy_id(struct rdma_cm_id *id) 824 { 825 struct rdma_id_private *id_priv; 826 enum cma_state state; 827 828 id_priv = container_of(id, struct rdma_id_private, id); 829 state = cma_exch(id_priv, CMA_DESTROYING); 830 cma_cancel_operation(id_priv, state); 831 832 mutex_lock(&lock); 833 if (id_priv->cma_dev) { 834 mutex_unlock(&lock); 835 switch (rdma_node_get_transport(id->device->node_type)) { 836 case RDMA_TRANSPORT_IB: 837 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) 838 ib_destroy_cm_id(id_priv->cm_id.ib); 839 break; 840 case RDMA_TRANSPORT_IWARP: 841 if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw)) 842 iw_destroy_cm_id(id_priv->cm_id.iw); 843 break; 844 default: 845 break; 846 } 847 cma_leave_mc_groups(id_priv); 848 mutex_lock(&lock); 849 cma_detach_from_dev(id_priv); 850 } 851 mutex_unlock(&lock); 852 853 cma_release_port(id_priv); 854 cma_deref_id(id_priv); 855 wait_for_completion(&id_priv->comp); 856 857 if (id_priv->internal_id) 858 cma_deref_id(id_priv->id.context); 859 860 kfree(id_priv->id.route.path_rec); 861 kfree(id_priv); 862 } 863 EXPORT_SYMBOL(rdma_destroy_id); 864 865 static int cma_rep_recv(struct rdma_id_private *id_priv) 866 { 867 int ret; 868 869 ret = cma_modify_qp_rtr(id_priv); 870 if (ret) 871 goto reject; 872 873 ret = cma_modify_qp_rts(id_priv); 874 if (ret) 875 goto reject; 876 877 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); 878 if (ret) 879 goto reject; 880 881 return 0; 882 reject: 883 cma_modify_qp_err(id_priv); 884 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, 885 NULL, 0, NULL, 0); 886 return ret; 887 } 888 889 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data) 890 { 891 if (id_priv->id.ps == RDMA_PS_SDP && 892 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) != 893 SDP_MAJ_VERSION) 894 return -EINVAL; 895 896 return 0; 897 } 898 899 static void cma_set_rep_event_data(struct rdma_cm_event *event, 900 struct ib_cm_rep_event_param *rep_data, 901 void *private_data) 902 { 903 event->param.conn.private_data = private_data; 904 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; 905 event->param.conn.responder_resources = rep_data->responder_resources; 906 event->param.conn.initiator_depth = rep_data->initiator_depth; 907 event->param.conn.flow_control = rep_data->flow_control; 908 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; 909 event->param.conn.srq = rep_data->srq; 910 event->param.conn.qp_num = rep_data->remote_qpn; 911 } 912 913 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 914 { 915 struct rdma_id_private *id_priv = cm_id->context; 916 struct rdma_cm_event event; 917 int ret = 0; 918 919 if (cma_disable_remove(id_priv, CMA_CONNECT)) 920 return 0; 921 922 memset(&event, 0, sizeof event); 923 switch (ib_event->event) { 924 case IB_CM_REQ_ERROR: 925 case IB_CM_REP_ERROR: 926 event.event = RDMA_CM_EVENT_UNREACHABLE; 927 event.status = -ETIMEDOUT; 928 break; 929 case IB_CM_REP_RECEIVED: 930 event.status = cma_verify_rep(id_priv, ib_event->private_data); 931 if (event.status) 932 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 933 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) { 934 event.status = cma_rep_recv(id_priv); 935 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : 936 RDMA_CM_EVENT_ESTABLISHED; 937 } else 938 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; 939 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, 940 ib_event->private_data); 941 break; 942 case IB_CM_RTU_RECEIVED: 943 case IB_CM_USER_ESTABLISHED: 944 event.event = RDMA_CM_EVENT_ESTABLISHED; 945 break; 946 case IB_CM_DREQ_ERROR: 947 event.status = -ETIMEDOUT; /* fall through */ 948 case IB_CM_DREQ_RECEIVED: 949 case IB_CM_DREP_RECEIVED: 950 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT)) 951 goto out; 952 event.event = RDMA_CM_EVENT_DISCONNECTED; 953 break; 954 case IB_CM_TIMEWAIT_EXIT: 955 case IB_CM_MRA_RECEIVED: 956 /* ignore event */ 957 goto out; 958 case IB_CM_REJ_RECEIVED: 959 cma_modify_qp_err(id_priv); 960 event.status = ib_event->param.rej_rcvd.reason; 961 event.event = RDMA_CM_EVENT_REJECTED; 962 event.param.conn.private_data = ib_event->private_data; 963 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; 964 break; 965 default: 966 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d", 967 ib_event->event); 968 goto out; 969 } 970 971 ret = id_priv->id.event_handler(&id_priv->id, &event); 972 if (ret) { 973 /* Destroy the CM ID by returning a non-zero value. */ 974 id_priv->cm_id.ib = NULL; 975 cma_exch(id_priv, CMA_DESTROYING); 976 cma_enable_remove(id_priv); 977 rdma_destroy_id(&id_priv->id); 978 return ret; 979 } 980 out: 981 cma_enable_remove(id_priv); 982 return ret; 983 } 984 985 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, 986 struct ib_cm_event *ib_event) 987 { 988 struct rdma_id_private *id_priv; 989 struct rdma_cm_id *id; 990 struct rdma_route *rt; 991 union cma_ip_addr *src, *dst; 992 __u16 port; 993 u8 ip_ver; 994 995 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 996 &ip_ver, &port, &src, &dst)) 997 goto err; 998 999 id = rdma_create_id(listen_id->event_handler, listen_id->context, 1000 listen_id->ps); 1001 if (IS_ERR(id)) 1002 goto err; 1003 1004 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 1005 ip_ver, port, src, dst); 1006 1007 rt = &id->route; 1008 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; 1009 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths, 1010 GFP_KERNEL); 1011 if (!rt->path_rec) 1012 goto destroy_id; 1013 1014 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path; 1015 if (rt->num_paths == 2) 1016 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; 1017 1018 ib_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); 1019 ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); 1020 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey)); 1021 rt->addr.dev_addr.dev_type = RDMA_NODE_IB_CA; 1022 1023 id_priv = container_of(id, struct rdma_id_private, id); 1024 id_priv->state = CMA_CONNECT; 1025 return id_priv; 1026 1027 destroy_id: 1028 rdma_destroy_id(id); 1029 err: 1030 return NULL; 1031 } 1032 1033 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id, 1034 struct ib_cm_event *ib_event) 1035 { 1036 struct rdma_id_private *id_priv; 1037 struct rdma_cm_id *id; 1038 union cma_ip_addr *src, *dst; 1039 __u16 port; 1040 u8 ip_ver; 1041 int ret; 1042 1043 id = rdma_create_id(listen_id->event_handler, listen_id->context, 1044 listen_id->ps); 1045 if (IS_ERR(id)) 1046 return NULL; 1047 1048 1049 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 1050 &ip_ver, &port, &src, &dst)) 1051 goto err; 1052 1053 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 1054 ip_ver, port, src, dst); 1055 1056 ret = rdma_translate_ip(&id->route.addr.src_addr, 1057 &id->route.addr.dev_addr); 1058 if (ret) 1059 goto err; 1060 1061 id_priv = container_of(id, struct rdma_id_private, id); 1062 id_priv->state = CMA_CONNECT; 1063 return id_priv; 1064 err: 1065 rdma_destroy_id(id); 1066 return NULL; 1067 } 1068 1069 static void cma_set_req_event_data(struct rdma_cm_event *event, 1070 struct ib_cm_req_event_param *req_data, 1071 void *private_data, int offset) 1072 { 1073 event->param.conn.private_data = private_data + offset; 1074 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; 1075 event->param.conn.responder_resources = req_data->responder_resources; 1076 event->param.conn.initiator_depth = req_data->initiator_depth; 1077 event->param.conn.flow_control = req_data->flow_control; 1078 event->param.conn.retry_count = req_data->retry_count; 1079 event->param.conn.rnr_retry_count = req_data->rnr_retry_count; 1080 event->param.conn.srq = req_data->srq; 1081 event->param.conn.qp_num = req_data->remote_qpn; 1082 } 1083 1084 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 1085 { 1086 struct rdma_id_private *listen_id, *conn_id; 1087 struct rdma_cm_event event; 1088 int offset, ret; 1089 1090 listen_id = cm_id->context; 1091 if (cma_disable_remove(listen_id, CMA_LISTEN)) 1092 return -ECONNABORTED; 1093 1094 memset(&event, 0, sizeof event); 1095 offset = cma_user_data_offset(listen_id->id.ps); 1096 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1097 if (cma_is_ud_ps(listen_id->id.ps)) { 1098 conn_id = cma_new_udp_id(&listen_id->id, ib_event); 1099 event.param.ud.private_data = ib_event->private_data + offset; 1100 event.param.ud.private_data_len = 1101 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; 1102 } else { 1103 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 1104 conn_id = cma_new_conn_id(&listen_id->id, ib_event); 1105 cma_set_req_event_data(&event, &ib_event->param.req_rcvd, 1106 ib_event->private_data, offset); 1107 } 1108 if (!conn_id) { 1109 ret = -ENOMEM; 1110 goto out; 1111 } 1112 1113 atomic_inc(&conn_id->dev_remove); 1114 mutex_lock(&lock); 1115 ret = cma_acquire_dev(conn_id); 1116 mutex_unlock(&lock); 1117 if (ret) 1118 goto release_conn_id; 1119 1120 conn_id->cm_id.ib = cm_id; 1121 cm_id->context = conn_id; 1122 cm_id->cm_handler = cma_ib_handler; 1123 1124 ret = conn_id->id.event_handler(&conn_id->id, &event); 1125 if (!ret) 1126 goto out; 1127 1128 /* Destroy the CM ID by returning a non-zero value. */ 1129 conn_id->cm_id.ib = NULL; 1130 1131 release_conn_id: 1132 cma_exch(conn_id, CMA_DESTROYING); 1133 cma_enable_remove(conn_id); 1134 rdma_destroy_id(&conn_id->id); 1135 1136 out: 1137 cma_enable_remove(listen_id); 1138 return ret; 1139 } 1140 1141 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr) 1142 { 1143 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr))); 1144 } 1145 1146 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr, 1147 struct ib_cm_compare_data *compare) 1148 { 1149 struct cma_hdr *cma_data, *cma_mask; 1150 struct sdp_hh *sdp_data, *sdp_mask; 1151 __u32 ip4_addr; 1152 struct in6_addr ip6_addr; 1153 1154 memset(compare, 0, sizeof *compare); 1155 cma_data = (void *) compare->data; 1156 cma_mask = (void *) compare->mask; 1157 sdp_data = (void *) compare->data; 1158 sdp_mask = (void *) compare->mask; 1159 1160 switch (addr->sa_family) { 1161 case AF_INET: 1162 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr; 1163 if (ps == RDMA_PS_SDP) { 1164 sdp_set_ip_ver(sdp_data, 4); 1165 sdp_set_ip_ver(sdp_mask, 0xF); 1166 sdp_data->dst_addr.ip4.addr = ip4_addr; 1167 sdp_mask->dst_addr.ip4.addr = ~0; 1168 } else { 1169 cma_set_ip_ver(cma_data, 4); 1170 cma_set_ip_ver(cma_mask, 0xF); 1171 cma_data->dst_addr.ip4.addr = ip4_addr; 1172 cma_mask->dst_addr.ip4.addr = ~0; 1173 } 1174 break; 1175 case AF_INET6: 1176 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr; 1177 if (ps == RDMA_PS_SDP) { 1178 sdp_set_ip_ver(sdp_data, 6); 1179 sdp_set_ip_ver(sdp_mask, 0xF); 1180 sdp_data->dst_addr.ip6 = ip6_addr; 1181 memset(&sdp_mask->dst_addr.ip6, 0xFF, 1182 sizeof sdp_mask->dst_addr.ip6); 1183 } else { 1184 cma_set_ip_ver(cma_data, 6); 1185 cma_set_ip_ver(cma_mask, 0xF); 1186 cma_data->dst_addr.ip6 = ip6_addr; 1187 memset(&cma_mask->dst_addr.ip6, 0xFF, 1188 sizeof cma_mask->dst_addr.ip6); 1189 } 1190 break; 1191 default: 1192 break; 1193 } 1194 } 1195 1196 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) 1197 { 1198 struct rdma_id_private *id_priv = iw_id->context; 1199 struct rdma_cm_event event; 1200 struct sockaddr_in *sin; 1201 int ret = 0; 1202 1203 if (cma_disable_remove(id_priv, CMA_CONNECT)) 1204 return 0; 1205 1206 memset(&event, 0, sizeof event); 1207 switch (iw_event->event) { 1208 case IW_CM_EVENT_CLOSE: 1209 event.event = RDMA_CM_EVENT_DISCONNECTED; 1210 break; 1211 case IW_CM_EVENT_CONNECT_REPLY: 1212 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1213 *sin = iw_event->local_addr; 1214 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr; 1215 *sin = iw_event->remote_addr; 1216 switch (iw_event->status) { 1217 case 0: 1218 event.event = RDMA_CM_EVENT_ESTABLISHED; 1219 break; 1220 case -ECONNRESET: 1221 case -ECONNREFUSED: 1222 event.event = RDMA_CM_EVENT_REJECTED; 1223 break; 1224 case -ETIMEDOUT: 1225 event.event = RDMA_CM_EVENT_UNREACHABLE; 1226 break; 1227 default: 1228 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 1229 break; 1230 } 1231 break; 1232 case IW_CM_EVENT_ESTABLISHED: 1233 event.event = RDMA_CM_EVENT_ESTABLISHED; 1234 break; 1235 default: 1236 BUG_ON(1); 1237 } 1238 1239 event.status = iw_event->status; 1240 event.param.conn.private_data = iw_event->private_data; 1241 event.param.conn.private_data_len = iw_event->private_data_len; 1242 ret = id_priv->id.event_handler(&id_priv->id, &event); 1243 if (ret) { 1244 /* Destroy the CM ID by returning a non-zero value. */ 1245 id_priv->cm_id.iw = NULL; 1246 cma_exch(id_priv, CMA_DESTROYING); 1247 cma_enable_remove(id_priv); 1248 rdma_destroy_id(&id_priv->id); 1249 return ret; 1250 } 1251 1252 cma_enable_remove(id_priv); 1253 return ret; 1254 } 1255 1256 static int iw_conn_req_handler(struct iw_cm_id *cm_id, 1257 struct iw_cm_event *iw_event) 1258 { 1259 struct rdma_cm_id *new_cm_id; 1260 struct rdma_id_private *listen_id, *conn_id; 1261 struct sockaddr_in *sin; 1262 struct net_device *dev = NULL; 1263 struct rdma_cm_event event; 1264 int ret; 1265 1266 listen_id = cm_id->context; 1267 if (cma_disable_remove(listen_id, CMA_LISTEN)) 1268 return -ECONNABORTED; 1269 1270 /* Create a new RDMA id for the new IW CM ID */ 1271 new_cm_id = rdma_create_id(listen_id->id.event_handler, 1272 listen_id->id.context, 1273 RDMA_PS_TCP); 1274 if (!new_cm_id) { 1275 ret = -ENOMEM; 1276 goto out; 1277 } 1278 conn_id = container_of(new_cm_id, struct rdma_id_private, id); 1279 atomic_inc(&conn_id->dev_remove); 1280 conn_id->state = CMA_CONNECT; 1281 1282 dev = ip_dev_find(iw_event->local_addr.sin_addr.s_addr); 1283 if (!dev) { 1284 ret = -EADDRNOTAVAIL; 1285 cma_enable_remove(conn_id); 1286 rdma_destroy_id(new_cm_id); 1287 goto out; 1288 } 1289 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL); 1290 if (ret) { 1291 cma_enable_remove(conn_id); 1292 rdma_destroy_id(new_cm_id); 1293 goto out; 1294 } 1295 1296 mutex_lock(&lock); 1297 ret = cma_acquire_dev(conn_id); 1298 mutex_unlock(&lock); 1299 if (ret) { 1300 cma_enable_remove(conn_id); 1301 rdma_destroy_id(new_cm_id); 1302 goto out; 1303 } 1304 1305 conn_id->cm_id.iw = cm_id; 1306 cm_id->context = conn_id; 1307 cm_id->cm_handler = cma_iw_handler; 1308 1309 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr; 1310 *sin = iw_event->local_addr; 1311 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr; 1312 *sin = iw_event->remote_addr; 1313 1314 memset(&event, 0, sizeof event); 1315 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1316 event.param.conn.private_data = iw_event->private_data; 1317 event.param.conn.private_data_len = iw_event->private_data_len; 1318 ret = conn_id->id.event_handler(&conn_id->id, &event); 1319 if (ret) { 1320 /* User wants to destroy the CM ID */ 1321 conn_id->cm_id.iw = NULL; 1322 cma_exch(conn_id, CMA_DESTROYING); 1323 cma_enable_remove(conn_id); 1324 rdma_destroy_id(&conn_id->id); 1325 } 1326 1327 out: 1328 if (dev) 1329 dev_put(dev); 1330 cma_enable_remove(listen_id); 1331 return ret; 1332 } 1333 1334 static int cma_ib_listen(struct rdma_id_private *id_priv) 1335 { 1336 struct ib_cm_compare_data compare_data; 1337 struct sockaddr *addr; 1338 __be64 svc_id; 1339 int ret; 1340 1341 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler, 1342 id_priv); 1343 if (IS_ERR(id_priv->cm_id.ib)) 1344 return PTR_ERR(id_priv->cm_id.ib); 1345 1346 addr = &id_priv->id.route.addr.src_addr; 1347 svc_id = cma_get_service_id(id_priv->id.ps, addr); 1348 if (cma_any_addr(addr)) 1349 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL); 1350 else { 1351 cma_set_compare_data(id_priv->id.ps, addr, &compare_data); 1352 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data); 1353 } 1354 1355 if (ret) { 1356 ib_destroy_cm_id(id_priv->cm_id.ib); 1357 id_priv->cm_id.ib = NULL; 1358 } 1359 1360 return ret; 1361 } 1362 1363 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 1364 { 1365 int ret; 1366 struct sockaddr_in *sin; 1367 1368 id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, 1369 iw_conn_req_handler, 1370 id_priv); 1371 if (IS_ERR(id_priv->cm_id.iw)) 1372 return PTR_ERR(id_priv->cm_id.iw); 1373 1374 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1375 id_priv->cm_id.iw->local_addr = *sin; 1376 1377 ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 1378 1379 if (ret) { 1380 iw_destroy_cm_id(id_priv->cm_id.iw); 1381 id_priv->cm_id.iw = NULL; 1382 } 1383 1384 return ret; 1385 } 1386 1387 static int cma_listen_handler(struct rdma_cm_id *id, 1388 struct rdma_cm_event *event) 1389 { 1390 struct rdma_id_private *id_priv = id->context; 1391 1392 id->context = id_priv->id.context; 1393 id->event_handler = id_priv->id.event_handler; 1394 return id_priv->id.event_handler(id, event); 1395 } 1396 1397 static void cma_listen_on_dev(struct rdma_id_private *id_priv, 1398 struct cma_device *cma_dev) 1399 { 1400 struct rdma_id_private *dev_id_priv; 1401 struct rdma_cm_id *id; 1402 int ret; 1403 1404 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps); 1405 if (IS_ERR(id)) 1406 return; 1407 1408 dev_id_priv = container_of(id, struct rdma_id_private, id); 1409 1410 dev_id_priv->state = CMA_ADDR_BOUND; 1411 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr, 1412 ip_addr_size(&id_priv->id.route.addr.src_addr)); 1413 1414 cma_attach_to_dev(dev_id_priv, cma_dev); 1415 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); 1416 atomic_inc(&id_priv->refcount); 1417 dev_id_priv->internal_id = 1; 1418 1419 ret = rdma_listen(id, id_priv->backlog); 1420 if (ret) 1421 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, " 1422 "listening on device %s", ret, cma_dev->device->name); 1423 } 1424 1425 static void cma_listen_on_all(struct rdma_id_private *id_priv) 1426 { 1427 struct cma_device *cma_dev; 1428 1429 mutex_lock(&lock); 1430 list_add_tail(&id_priv->list, &listen_any_list); 1431 list_for_each_entry(cma_dev, &dev_list, list) 1432 cma_listen_on_dev(id_priv, cma_dev); 1433 mutex_unlock(&lock); 1434 } 1435 1436 static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af) 1437 { 1438 struct sockaddr_in addr_in; 1439 1440 memset(&addr_in, 0, sizeof addr_in); 1441 addr_in.sin_family = af; 1442 return rdma_bind_addr(id, (struct sockaddr *) &addr_in); 1443 } 1444 1445 int rdma_listen(struct rdma_cm_id *id, int backlog) 1446 { 1447 struct rdma_id_private *id_priv; 1448 int ret; 1449 1450 id_priv = container_of(id, struct rdma_id_private, id); 1451 if (id_priv->state == CMA_IDLE) { 1452 ret = cma_bind_any(id, AF_INET); 1453 if (ret) 1454 return ret; 1455 } 1456 1457 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN)) 1458 return -EINVAL; 1459 1460 id_priv->backlog = backlog; 1461 if (id->device) { 1462 switch (rdma_node_get_transport(id->device->node_type)) { 1463 case RDMA_TRANSPORT_IB: 1464 ret = cma_ib_listen(id_priv); 1465 if (ret) 1466 goto err; 1467 break; 1468 case RDMA_TRANSPORT_IWARP: 1469 ret = cma_iw_listen(id_priv, backlog); 1470 if (ret) 1471 goto err; 1472 break; 1473 default: 1474 ret = -ENOSYS; 1475 goto err; 1476 } 1477 } else 1478 cma_listen_on_all(id_priv); 1479 1480 return 0; 1481 err: 1482 id_priv->backlog = 0; 1483 cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND); 1484 return ret; 1485 } 1486 EXPORT_SYMBOL(rdma_listen); 1487 1488 void rdma_set_service_type(struct rdma_cm_id *id, int tos) 1489 { 1490 struct rdma_id_private *id_priv; 1491 1492 id_priv = container_of(id, struct rdma_id_private, id); 1493 id_priv->tos = (u8) tos; 1494 } 1495 EXPORT_SYMBOL(rdma_set_service_type); 1496 1497 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec, 1498 void *context) 1499 { 1500 struct cma_work *work = context; 1501 struct rdma_route *route; 1502 1503 route = &work->id->id.route; 1504 1505 if (!status) { 1506 route->num_paths = 1; 1507 *route->path_rec = *path_rec; 1508 } else { 1509 work->old_state = CMA_ROUTE_QUERY; 1510 work->new_state = CMA_ADDR_RESOLVED; 1511 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; 1512 work->event.status = status; 1513 } 1514 1515 queue_work(cma_wq, &work->work); 1516 } 1517 1518 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, 1519 struct cma_work *work) 1520 { 1521 struct rdma_addr *addr = &id_priv->id.route.addr; 1522 struct ib_sa_path_rec path_rec; 1523 ib_sa_comp_mask comp_mask; 1524 struct sockaddr_in6 *sin6; 1525 1526 memset(&path_rec, 0, sizeof path_rec); 1527 ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid); 1528 ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid); 1529 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr)); 1530 path_rec.numb_path = 1; 1531 path_rec.reversible = 1; 1532 path_rec.service_id = cma_get_service_id(id_priv->id.ps, &addr->dst_addr); 1533 1534 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | 1535 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | 1536 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; 1537 1538 if (addr->src_addr.sa_family == AF_INET) { 1539 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); 1540 comp_mask |= IB_SA_PATH_REC_QOS_CLASS; 1541 } else { 1542 sin6 = (struct sockaddr_in6 *) &addr->src_addr; 1543 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); 1544 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 1545 } 1546 1547 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, 1548 id_priv->id.port_num, &path_rec, 1549 comp_mask, timeout_ms, 1550 GFP_KERNEL, cma_query_handler, 1551 work, &id_priv->query); 1552 1553 return (id_priv->query_id < 0) ? id_priv->query_id : 0; 1554 } 1555 1556 static void cma_work_handler(struct work_struct *_work) 1557 { 1558 struct cma_work *work = container_of(_work, struct cma_work, work); 1559 struct rdma_id_private *id_priv = work->id; 1560 int destroy = 0; 1561 1562 atomic_inc(&id_priv->dev_remove); 1563 if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) 1564 goto out; 1565 1566 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 1567 cma_exch(id_priv, CMA_DESTROYING); 1568 destroy = 1; 1569 } 1570 out: 1571 cma_enable_remove(id_priv); 1572 cma_deref_id(id_priv); 1573 if (destroy) 1574 rdma_destroy_id(&id_priv->id); 1575 kfree(work); 1576 } 1577 1578 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) 1579 { 1580 struct rdma_route *route = &id_priv->id.route; 1581 struct cma_work *work; 1582 int ret; 1583 1584 work = kzalloc(sizeof *work, GFP_KERNEL); 1585 if (!work) 1586 return -ENOMEM; 1587 1588 work->id = id_priv; 1589 INIT_WORK(&work->work, cma_work_handler); 1590 work->old_state = CMA_ROUTE_QUERY; 1591 work->new_state = CMA_ROUTE_RESOLVED; 1592 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1593 1594 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); 1595 if (!route->path_rec) { 1596 ret = -ENOMEM; 1597 goto err1; 1598 } 1599 1600 ret = cma_query_ib_route(id_priv, timeout_ms, work); 1601 if (ret) 1602 goto err2; 1603 1604 return 0; 1605 err2: 1606 kfree(route->path_rec); 1607 route->path_rec = NULL; 1608 err1: 1609 kfree(work); 1610 return ret; 1611 } 1612 1613 int rdma_set_ib_paths(struct rdma_cm_id *id, 1614 struct ib_sa_path_rec *path_rec, int num_paths) 1615 { 1616 struct rdma_id_private *id_priv; 1617 int ret; 1618 1619 id_priv = container_of(id, struct rdma_id_private, id); 1620 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED)) 1621 return -EINVAL; 1622 1623 id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL); 1624 if (!id->route.path_rec) { 1625 ret = -ENOMEM; 1626 goto err; 1627 } 1628 1629 memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths); 1630 return 0; 1631 err: 1632 cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED); 1633 return ret; 1634 } 1635 EXPORT_SYMBOL(rdma_set_ib_paths); 1636 1637 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) 1638 { 1639 struct cma_work *work; 1640 1641 work = kzalloc(sizeof *work, GFP_KERNEL); 1642 if (!work) 1643 return -ENOMEM; 1644 1645 work->id = id_priv; 1646 INIT_WORK(&work->work, cma_work_handler); 1647 work->old_state = CMA_ROUTE_QUERY; 1648 work->new_state = CMA_ROUTE_RESOLVED; 1649 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1650 queue_work(cma_wq, &work->work); 1651 return 0; 1652 } 1653 1654 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) 1655 { 1656 struct rdma_id_private *id_priv; 1657 int ret; 1658 1659 id_priv = container_of(id, struct rdma_id_private, id); 1660 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY)) 1661 return -EINVAL; 1662 1663 atomic_inc(&id_priv->refcount); 1664 switch (rdma_node_get_transport(id->device->node_type)) { 1665 case RDMA_TRANSPORT_IB: 1666 ret = cma_resolve_ib_route(id_priv, timeout_ms); 1667 break; 1668 case RDMA_TRANSPORT_IWARP: 1669 ret = cma_resolve_iw_route(id_priv, timeout_ms); 1670 break; 1671 default: 1672 ret = -ENOSYS; 1673 break; 1674 } 1675 if (ret) 1676 goto err; 1677 1678 return 0; 1679 err: 1680 cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED); 1681 cma_deref_id(id_priv); 1682 return ret; 1683 } 1684 EXPORT_SYMBOL(rdma_resolve_route); 1685 1686 static int cma_bind_loopback(struct rdma_id_private *id_priv) 1687 { 1688 struct cma_device *cma_dev; 1689 struct ib_port_attr port_attr; 1690 union ib_gid gid; 1691 u16 pkey; 1692 int ret; 1693 u8 p; 1694 1695 mutex_lock(&lock); 1696 if (list_empty(&dev_list)) { 1697 ret = -ENODEV; 1698 goto out; 1699 } 1700 list_for_each_entry(cma_dev, &dev_list, list) 1701 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p) 1702 if (!ib_query_port(cma_dev->device, p, &port_attr) && 1703 port_attr.state == IB_PORT_ACTIVE) 1704 goto port_found; 1705 1706 p = 1; 1707 cma_dev = list_entry(dev_list.next, struct cma_device, list); 1708 1709 port_found: 1710 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid); 1711 if (ret) 1712 goto out; 1713 1714 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); 1715 if (ret) 1716 goto out; 1717 1718 ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); 1719 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); 1720 id_priv->id.port_num = p; 1721 cma_attach_to_dev(id_priv, cma_dev); 1722 out: 1723 mutex_unlock(&lock); 1724 return ret; 1725 } 1726 1727 static void addr_handler(int status, struct sockaddr *src_addr, 1728 struct rdma_dev_addr *dev_addr, void *context) 1729 { 1730 struct rdma_id_private *id_priv = context; 1731 struct rdma_cm_event event; 1732 1733 memset(&event, 0, sizeof event); 1734 atomic_inc(&id_priv->dev_remove); 1735 1736 /* 1737 * Grab mutex to block rdma_destroy_id() from removing the device while 1738 * we're trying to acquire it. 1739 */ 1740 mutex_lock(&lock); 1741 if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) { 1742 mutex_unlock(&lock); 1743 goto out; 1744 } 1745 1746 if (!status && !id_priv->cma_dev) 1747 status = cma_acquire_dev(id_priv); 1748 mutex_unlock(&lock); 1749 1750 if (status) { 1751 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND)) 1752 goto out; 1753 event.event = RDMA_CM_EVENT_ADDR_ERROR; 1754 event.status = status; 1755 } else { 1756 memcpy(&id_priv->id.route.addr.src_addr, src_addr, 1757 ip_addr_size(src_addr)); 1758 event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 1759 } 1760 1761 if (id_priv->id.event_handler(&id_priv->id, &event)) { 1762 cma_exch(id_priv, CMA_DESTROYING); 1763 cma_enable_remove(id_priv); 1764 cma_deref_id(id_priv); 1765 rdma_destroy_id(&id_priv->id); 1766 return; 1767 } 1768 out: 1769 cma_enable_remove(id_priv); 1770 cma_deref_id(id_priv); 1771 } 1772 1773 static int cma_resolve_loopback(struct rdma_id_private *id_priv) 1774 { 1775 struct cma_work *work; 1776 struct sockaddr_in *src_in, *dst_in; 1777 union ib_gid gid; 1778 int ret; 1779 1780 work = kzalloc(sizeof *work, GFP_KERNEL); 1781 if (!work) 1782 return -ENOMEM; 1783 1784 if (!id_priv->cma_dev) { 1785 ret = cma_bind_loopback(id_priv); 1786 if (ret) 1787 goto err; 1788 } 1789 1790 ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); 1791 ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); 1792 1793 if (cma_zero_addr(&id_priv->id.route.addr.src_addr)) { 1794 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr; 1795 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr; 1796 src_in->sin_family = dst_in->sin_family; 1797 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr; 1798 } 1799 1800 work->id = id_priv; 1801 INIT_WORK(&work->work, cma_work_handler); 1802 work->old_state = CMA_ADDR_QUERY; 1803 work->new_state = CMA_ADDR_RESOLVED; 1804 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 1805 queue_work(cma_wq, &work->work); 1806 return 0; 1807 err: 1808 kfree(work); 1809 return ret; 1810 } 1811 1812 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 1813 struct sockaddr *dst_addr) 1814 { 1815 if (src_addr && src_addr->sa_family) 1816 return rdma_bind_addr(id, src_addr); 1817 else 1818 return cma_bind_any(id, dst_addr->sa_family); 1819 } 1820 1821 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 1822 struct sockaddr *dst_addr, int timeout_ms) 1823 { 1824 struct rdma_id_private *id_priv; 1825 int ret; 1826 1827 id_priv = container_of(id, struct rdma_id_private, id); 1828 if (id_priv->state == CMA_IDLE) { 1829 ret = cma_bind_addr(id, src_addr, dst_addr); 1830 if (ret) 1831 return ret; 1832 } 1833 1834 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY)) 1835 return -EINVAL; 1836 1837 atomic_inc(&id_priv->refcount); 1838 memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr)); 1839 if (cma_any_addr(dst_addr)) 1840 ret = cma_resolve_loopback(id_priv); 1841 else 1842 ret = rdma_resolve_ip(&addr_client, &id->route.addr.src_addr, 1843 dst_addr, &id->route.addr.dev_addr, 1844 timeout_ms, addr_handler, id_priv); 1845 if (ret) 1846 goto err; 1847 1848 return 0; 1849 err: 1850 cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND); 1851 cma_deref_id(id_priv); 1852 return ret; 1853 } 1854 EXPORT_SYMBOL(rdma_resolve_addr); 1855 1856 static void cma_bind_port(struct rdma_bind_list *bind_list, 1857 struct rdma_id_private *id_priv) 1858 { 1859 struct sockaddr_in *sin; 1860 1861 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1862 sin->sin_port = htons(bind_list->port); 1863 id_priv->bind_list = bind_list; 1864 hlist_add_head(&id_priv->node, &bind_list->owners); 1865 } 1866 1867 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv, 1868 unsigned short snum) 1869 { 1870 struct rdma_bind_list *bind_list; 1871 int port, ret; 1872 1873 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 1874 if (!bind_list) 1875 return -ENOMEM; 1876 1877 do { 1878 ret = idr_get_new_above(ps, bind_list, snum, &port); 1879 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); 1880 1881 if (ret) 1882 goto err1; 1883 1884 if (port != snum) { 1885 ret = -EADDRNOTAVAIL; 1886 goto err2; 1887 } 1888 1889 bind_list->ps = ps; 1890 bind_list->port = (unsigned short) port; 1891 cma_bind_port(bind_list, id_priv); 1892 return 0; 1893 err2: 1894 idr_remove(ps, port); 1895 err1: 1896 kfree(bind_list); 1897 return ret; 1898 } 1899 1900 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv) 1901 { 1902 struct rdma_bind_list *bind_list; 1903 int port, ret, low, high; 1904 1905 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 1906 if (!bind_list) 1907 return -ENOMEM; 1908 1909 retry: 1910 /* FIXME: add proper port randomization per like inet_csk_get_port */ 1911 do { 1912 ret = idr_get_new_above(ps, bind_list, next_port, &port); 1913 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); 1914 1915 if (ret) 1916 goto err1; 1917 1918 inet_get_local_port_range(&low, &high); 1919 if (port > high) { 1920 if (next_port != low) { 1921 idr_remove(ps, port); 1922 next_port = low; 1923 goto retry; 1924 } 1925 ret = -EADDRNOTAVAIL; 1926 goto err2; 1927 } 1928 1929 if (port == high) 1930 next_port = low; 1931 else 1932 next_port = port + 1; 1933 1934 bind_list->ps = ps; 1935 bind_list->port = (unsigned short) port; 1936 cma_bind_port(bind_list, id_priv); 1937 return 0; 1938 err2: 1939 idr_remove(ps, port); 1940 err1: 1941 kfree(bind_list); 1942 return ret; 1943 } 1944 1945 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv) 1946 { 1947 struct rdma_id_private *cur_id; 1948 struct sockaddr_in *sin, *cur_sin; 1949 struct rdma_bind_list *bind_list; 1950 struct hlist_node *node; 1951 unsigned short snum; 1952 1953 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1954 snum = ntohs(sin->sin_port); 1955 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) 1956 return -EACCES; 1957 1958 bind_list = idr_find(ps, snum); 1959 if (!bind_list) 1960 return cma_alloc_port(ps, id_priv, snum); 1961 1962 /* 1963 * We don't support binding to any address if anyone is bound to 1964 * a specific address on the same port. 1965 */ 1966 if (cma_any_addr(&id_priv->id.route.addr.src_addr)) 1967 return -EADDRNOTAVAIL; 1968 1969 hlist_for_each_entry(cur_id, node, &bind_list->owners, node) { 1970 if (cma_any_addr(&cur_id->id.route.addr.src_addr)) 1971 return -EADDRNOTAVAIL; 1972 1973 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr; 1974 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr) 1975 return -EADDRINUSE; 1976 } 1977 1978 cma_bind_port(bind_list, id_priv); 1979 return 0; 1980 } 1981 1982 static int cma_get_port(struct rdma_id_private *id_priv) 1983 { 1984 struct idr *ps; 1985 int ret; 1986 1987 switch (id_priv->id.ps) { 1988 case RDMA_PS_SDP: 1989 ps = &sdp_ps; 1990 break; 1991 case RDMA_PS_TCP: 1992 ps = &tcp_ps; 1993 break; 1994 case RDMA_PS_UDP: 1995 ps = &udp_ps; 1996 break; 1997 case RDMA_PS_IPOIB: 1998 ps = &ipoib_ps; 1999 break; 2000 default: 2001 return -EPROTONOSUPPORT; 2002 } 2003 2004 mutex_lock(&lock); 2005 if (cma_any_port(&id_priv->id.route.addr.src_addr)) 2006 ret = cma_alloc_any_port(ps, id_priv); 2007 else 2008 ret = cma_use_port(ps, id_priv); 2009 mutex_unlock(&lock); 2010 2011 return ret; 2012 } 2013 2014 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 2015 { 2016 struct rdma_id_private *id_priv; 2017 int ret; 2018 2019 if (addr->sa_family != AF_INET) 2020 return -EAFNOSUPPORT; 2021 2022 id_priv = container_of(id, struct rdma_id_private, id); 2023 if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND)) 2024 return -EINVAL; 2025 2026 if (!cma_any_addr(addr)) { 2027 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr); 2028 if (ret) 2029 goto err1; 2030 2031 mutex_lock(&lock); 2032 ret = cma_acquire_dev(id_priv); 2033 mutex_unlock(&lock); 2034 if (ret) 2035 goto err1; 2036 } 2037 2038 memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr)); 2039 ret = cma_get_port(id_priv); 2040 if (ret) 2041 goto err2; 2042 2043 return 0; 2044 err2: 2045 if (!cma_any_addr(addr)) { 2046 mutex_lock(&lock); 2047 cma_detach_from_dev(id_priv); 2048 mutex_unlock(&lock); 2049 } 2050 err1: 2051 cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE); 2052 return ret; 2053 } 2054 EXPORT_SYMBOL(rdma_bind_addr); 2055 2056 static int cma_format_hdr(void *hdr, enum rdma_port_space ps, 2057 struct rdma_route *route) 2058 { 2059 struct sockaddr_in *src4, *dst4; 2060 struct cma_hdr *cma_hdr; 2061 struct sdp_hh *sdp_hdr; 2062 2063 src4 = (struct sockaddr_in *) &route->addr.src_addr; 2064 dst4 = (struct sockaddr_in *) &route->addr.dst_addr; 2065 2066 switch (ps) { 2067 case RDMA_PS_SDP: 2068 sdp_hdr = hdr; 2069 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) 2070 return -EINVAL; 2071 sdp_set_ip_ver(sdp_hdr, 4); 2072 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2073 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2074 sdp_hdr->port = src4->sin_port; 2075 break; 2076 default: 2077 cma_hdr = hdr; 2078 cma_hdr->cma_version = CMA_VERSION; 2079 cma_set_ip_ver(cma_hdr, 4); 2080 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2081 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2082 cma_hdr->port = src4->sin_port; 2083 break; 2084 } 2085 return 0; 2086 } 2087 2088 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, 2089 struct ib_cm_event *ib_event) 2090 { 2091 struct rdma_id_private *id_priv = cm_id->context; 2092 struct rdma_cm_event event; 2093 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; 2094 int ret = 0; 2095 2096 if (cma_disable_remove(id_priv, CMA_CONNECT)) 2097 return 0; 2098 2099 memset(&event, 0, sizeof event); 2100 switch (ib_event->event) { 2101 case IB_CM_SIDR_REQ_ERROR: 2102 event.event = RDMA_CM_EVENT_UNREACHABLE; 2103 event.status = -ETIMEDOUT; 2104 break; 2105 case IB_CM_SIDR_REP_RECEIVED: 2106 event.param.ud.private_data = ib_event->private_data; 2107 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; 2108 if (rep->status != IB_SIDR_SUCCESS) { 2109 event.event = RDMA_CM_EVENT_UNREACHABLE; 2110 event.status = ib_event->param.sidr_rep_rcvd.status; 2111 break; 2112 } 2113 if (id_priv->qkey != rep->qkey) { 2114 event.event = RDMA_CM_EVENT_UNREACHABLE; 2115 event.status = -EINVAL; 2116 break; 2117 } 2118 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, 2119 id_priv->id.route.path_rec, 2120 &event.param.ud.ah_attr); 2121 event.param.ud.qp_num = rep->qpn; 2122 event.param.ud.qkey = rep->qkey; 2123 event.event = RDMA_CM_EVENT_ESTABLISHED; 2124 event.status = 0; 2125 break; 2126 default: 2127 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d", 2128 ib_event->event); 2129 goto out; 2130 } 2131 2132 ret = id_priv->id.event_handler(&id_priv->id, &event); 2133 if (ret) { 2134 /* Destroy the CM ID by returning a non-zero value. */ 2135 id_priv->cm_id.ib = NULL; 2136 cma_exch(id_priv, CMA_DESTROYING); 2137 cma_enable_remove(id_priv); 2138 rdma_destroy_id(&id_priv->id); 2139 return ret; 2140 } 2141 out: 2142 cma_enable_remove(id_priv); 2143 return ret; 2144 } 2145 2146 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, 2147 struct rdma_conn_param *conn_param) 2148 { 2149 struct ib_cm_sidr_req_param req; 2150 struct rdma_route *route; 2151 int ret; 2152 2153 req.private_data_len = sizeof(struct cma_hdr) + 2154 conn_param->private_data_len; 2155 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2156 if (!req.private_data) 2157 return -ENOMEM; 2158 2159 if (conn_param->private_data && conn_param->private_data_len) 2160 memcpy((void *) req.private_data + sizeof(struct cma_hdr), 2161 conn_param->private_data, conn_param->private_data_len); 2162 2163 route = &id_priv->id.route; 2164 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route); 2165 if (ret) 2166 goto out; 2167 2168 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, 2169 cma_sidr_rep_handler, id_priv); 2170 if (IS_ERR(id_priv->cm_id.ib)) { 2171 ret = PTR_ERR(id_priv->cm_id.ib); 2172 goto out; 2173 } 2174 2175 req.path = route->path_rec; 2176 req.service_id = cma_get_service_id(id_priv->id.ps, 2177 &route->addr.dst_addr); 2178 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); 2179 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2180 2181 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); 2182 if (ret) { 2183 ib_destroy_cm_id(id_priv->cm_id.ib); 2184 id_priv->cm_id.ib = NULL; 2185 } 2186 out: 2187 kfree(req.private_data); 2188 return ret; 2189 } 2190 2191 static int cma_connect_ib(struct rdma_id_private *id_priv, 2192 struct rdma_conn_param *conn_param) 2193 { 2194 struct ib_cm_req_param req; 2195 struct rdma_route *route; 2196 void *private_data; 2197 int offset, ret; 2198 2199 memset(&req, 0, sizeof req); 2200 offset = cma_user_data_offset(id_priv->id.ps); 2201 req.private_data_len = offset + conn_param->private_data_len; 2202 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2203 if (!private_data) 2204 return -ENOMEM; 2205 2206 if (conn_param->private_data && conn_param->private_data_len) 2207 memcpy(private_data + offset, conn_param->private_data, 2208 conn_param->private_data_len); 2209 2210 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler, 2211 id_priv); 2212 if (IS_ERR(id_priv->cm_id.ib)) { 2213 ret = PTR_ERR(id_priv->cm_id.ib); 2214 goto out; 2215 } 2216 2217 route = &id_priv->id.route; 2218 ret = cma_format_hdr(private_data, id_priv->id.ps, route); 2219 if (ret) 2220 goto out; 2221 req.private_data = private_data; 2222 2223 req.primary_path = &route->path_rec[0]; 2224 if (route->num_paths == 2) 2225 req.alternate_path = &route->path_rec[1]; 2226 2227 req.service_id = cma_get_service_id(id_priv->id.ps, 2228 &route->addr.dst_addr); 2229 req.qp_num = id_priv->qp_num; 2230 req.qp_type = IB_QPT_RC; 2231 req.starting_psn = id_priv->seq_num; 2232 req.responder_resources = conn_param->responder_resources; 2233 req.initiator_depth = conn_param->initiator_depth; 2234 req.flow_control = conn_param->flow_control; 2235 req.retry_count = conn_param->retry_count; 2236 req.rnr_retry_count = conn_param->rnr_retry_count; 2237 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 2238 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 2239 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2240 req.srq = id_priv->srq ? 1 : 0; 2241 2242 ret = ib_send_cm_req(id_priv->cm_id.ib, &req); 2243 out: 2244 if (ret && !IS_ERR(id_priv->cm_id.ib)) { 2245 ib_destroy_cm_id(id_priv->cm_id.ib); 2246 id_priv->cm_id.ib = NULL; 2247 } 2248 2249 kfree(private_data); 2250 return ret; 2251 } 2252 2253 static int cma_connect_iw(struct rdma_id_private *id_priv, 2254 struct rdma_conn_param *conn_param) 2255 { 2256 struct iw_cm_id *cm_id; 2257 struct sockaddr_in* sin; 2258 int ret; 2259 struct iw_cm_conn_param iw_param; 2260 2261 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); 2262 if (IS_ERR(cm_id)) { 2263 ret = PTR_ERR(cm_id); 2264 goto out; 2265 } 2266 2267 id_priv->cm_id.iw = cm_id; 2268 2269 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr; 2270 cm_id->local_addr = *sin; 2271 2272 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr; 2273 cm_id->remote_addr = *sin; 2274 2275 ret = cma_modify_qp_rtr(id_priv); 2276 if (ret) 2277 goto out; 2278 2279 iw_param.ord = conn_param->initiator_depth; 2280 iw_param.ird = conn_param->responder_resources; 2281 iw_param.private_data = conn_param->private_data; 2282 iw_param.private_data_len = conn_param->private_data_len; 2283 if (id_priv->id.qp) 2284 iw_param.qpn = id_priv->qp_num; 2285 else 2286 iw_param.qpn = conn_param->qp_num; 2287 ret = iw_cm_connect(cm_id, &iw_param); 2288 out: 2289 if (ret && !IS_ERR(cm_id)) { 2290 iw_destroy_cm_id(cm_id); 2291 id_priv->cm_id.iw = NULL; 2292 } 2293 return ret; 2294 } 2295 2296 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2297 { 2298 struct rdma_id_private *id_priv; 2299 int ret; 2300 2301 id_priv = container_of(id, struct rdma_id_private, id); 2302 if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT)) 2303 return -EINVAL; 2304 2305 if (!id->qp) { 2306 id_priv->qp_num = conn_param->qp_num; 2307 id_priv->srq = conn_param->srq; 2308 } 2309 2310 switch (rdma_node_get_transport(id->device->node_type)) { 2311 case RDMA_TRANSPORT_IB: 2312 if (cma_is_ud_ps(id->ps)) 2313 ret = cma_resolve_ib_udp(id_priv, conn_param); 2314 else 2315 ret = cma_connect_ib(id_priv, conn_param); 2316 break; 2317 case RDMA_TRANSPORT_IWARP: 2318 ret = cma_connect_iw(id_priv, conn_param); 2319 break; 2320 default: 2321 ret = -ENOSYS; 2322 break; 2323 } 2324 if (ret) 2325 goto err; 2326 2327 return 0; 2328 err: 2329 cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED); 2330 return ret; 2331 } 2332 EXPORT_SYMBOL(rdma_connect); 2333 2334 static int cma_accept_ib(struct rdma_id_private *id_priv, 2335 struct rdma_conn_param *conn_param) 2336 { 2337 struct ib_cm_rep_param rep; 2338 struct ib_qp_attr qp_attr; 2339 int qp_attr_mask, ret; 2340 2341 if (id_priv->id.qp) { 2342 ret = cma_modify_qp_rtr(id_priv); 2343 if (ret) 2344 goto out; 2345 2346 qp_attr.qp_state = IB_QPS_RTS; 2347 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, &qp_attr, 2348 &qp_attr_mask); 2349 if (ret) 2350 goto out; 2351 2352 qp_attr.max_rd_atomic = conn_param->initiator_depth; 2353 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 2354 if (ret) 2355 goto out; 2356 } 2357 2358 memset(&rep, 0, sizeof rep); 2359 rep.qp_num = id_priv->qp_num; 2360 rep.starting_psn = id_priv->seq_num; 2361 rep.private_data = conn_param->private_data; 2362 rep.private_data_len = conn_param->private_data_len; 2363 rep.responder_resources = conn_param->responder_resources; 2364 rep.initiator_depth = conn_param->initiator_depth; 2365 rep.failover_accepted = 0; 2366 rep.flow_control = conn_param->flow_control; 2367 rep.rnr_retry_count = conn_param->rnr_retry_count; 2368 rep.srq = id_priv->srq ? 1 : 0; 2369 2370 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); 2371 out: 2372 return ret; 2373 } 2374 2375 static int cma_accept_iw(struct rdma_id_private *id_priv, 2376 struct rdma_conn_param *conn_param) 2377 { 2378 struct iw_cm_conn_param iw_param; 2379 int ret; 2380 2381 ret = cma_modify_qp_rtr(id_priv); 2382 if (ret) 2383 return ret; 2384 2385 iw_param.ord = conn_param->initiator_depth; 2386 iw_param.ird = conn_param->responder_resources; 2387 iw_param.private_data = conn_param->private_data; 2388 iw_param.private_data_len = conn_param->private_data_len; 2389 if (id_priv->id.qp) { 2390 iw_param.qpn = id_priv->qp_num; 2391 } else 2392 iw_param.qpn = conn_param->qp_num; 2393 2394 return iw_cm_accept(id_priv->cm_id.iw, &iw_param); 2395 } 2396 2397 static int cma_send_sidr_rep(struct rdma_id_private *id_priv, 2398 enum ib_cm_sidr_status status, 2399 const void *private_data, int private_data_len) 2400 { 2401 struct ib_cm_sidr_rep_param rep; 2402 2403 memset(&rep, 0, sizeof rep); 2404 rep.status = status; 2405 if (status == IB_SIDR_SUCCESS) { 2406 rep.qp_num = id_priv->qp_num; 2407 rep.qkey = id_priv->qkey; 2408 } 2409 rep.private_data = private_data; 2410 rep.private_data_len = private_data_len; 2411 2412 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); 2413 } 2414 2415 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2416 { 2417 struct rdma_id_private *id_priv; 2418 int ret; 2419 2420 id_priv = container_of(id, struct rdma_id_private, id); 2421 if (!cma_comp(id_priv, CMA_CONNECT)) 2422 return -EINVAL; 2423 2424 if (!id->qp && conn_param) { 2425 id_priv->qp_num = conn_param->qp_num; 2426 id_priv->srq = conn_param->srq; 2427 } 2428 2429 switch (rdma_node_get_transport(id->device->node_type)) { 2430 case RDMA_TRANSPORT_IB: 2431 if (cma_is_ud_ps(id->ps)) 2432 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 2433 conn_param->private_data, 2434 conn_param->private_data_len); 2435 else if (conn_param) 2436 ret = cma_accept_ib(id_priv, conn_param); 2437 else 2438 ret = cma_rep_recv(id_priv); 2439 break; 2440 case RDMA_TRANSPORT_IWARP: 2441 ret = cma_accept_iw(id_priv, conn_param); 2442 break; 2443 default: 2444 ret = -ENOSYS; 2445 break; 2446 } 2447 2448 if (ret) 2449 goto reject; 2450 2451 return 0; 2452 reject: 2453 cma_modify_qp_err(id_priv); 2454 rdma_reject(id, NULL, 0); 2455 return ret; 2456 } 2457 EXPORT_SYMBOL(rdma_accept); 2458 2459 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) 2460 { 2461 struct rdma_id_private *id_priv; 2462 int ret; 2463 2464 id_priv = container_of(id, struct rdma_id_private, id); 2465 if (!cma_has_cm_dev(id_priv)) 2466 return -EINVAL; 2467 2468 switch (id->device->node_type) { 2469 case RDMA_NODE_IB_CA: 2470 ret = ib_cm_notify(id_priv->cm_id.ib, event); 2471 break; 2472 default: 2473 ret = 0; 2474 break; 2475 } 2476 return ret; 2477 } 2478 EXPORT_SYMBOL(rdma_notify); 2479 2480 int rdma_reject(struct rdma_cm_id *id, const void *private_data, 2481 u8 private_data_len) 2482 { 2483 struct rdma_id_private *id_priv; 2484 int ret; 2485 2486 id_priv = container_of(id, struct rdma_id_private, id); 2487 if (!cma_has_cm_dev(id_priv)) 2488 return -EINVAL; 2489 2490 switch (rdma_node_get_transport(id->device->node_type)) { 2491 case RDMA_TRANSPORT_IB: 2492 if (cma_is_ud_ps(id->ps)) 2493 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 2494 private_data, private_data_len); 2495 else 2496 ret = ib_send_cm_rej(id_priv->cm_id.ib, 2497 IB_CM_REJ_CONSUMER_DEFINED, NULL, 2498 0, private_data, private_data_len); 2499 break; 2500 case RDMA_TRANSPORT_IWARP: 2501 ret = iw_cm_reject(id_priv->cm_id.iw, 2502 private_data, private_data_len); 2503 break; 2504 default: 2505 ret = -ENOSYS; 2506 break; 2507 } 2508 return ret; 2509 } 2510 EXPORT_SYMBOL(rdma_reject); 2511 2512 int rdma_disconnect(struct rdma_cm_id *id) 2513 { 2514 struct rdma_id_private *id_priv; 2515 int ret; 2516 2517 id_priv = container_of(id, struct rdma_id_private, id); 2518 if (!cma_has_cm_dev(id_priv)) 2519 return -EINVAL; 2520 2521 switch (rdma_node_get_transport(id->device->node_type)) { 2522 case RDMA_TRANSPORT_IB: 2523 ret = cma_modify_qp_err(id_priv); 2524 if (ret) 2525 goto out; 2526 /* Initiate or respond to a disconnect. */ 2527 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) 2528 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); 2529 break; 2530 case RDMA_TRANSPORT_IWARP: 2531 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); 2532 break; 2533 default: 2534 ret = -EINVAL; 2535 break; 2536 } 2537 out: 2538 return ret; 2539 } 2540 EXPORT_SYMBOL(rdma_disconnect); 2541 2542 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) 2543 { 2544 struct rdma_id_private *id_priv; 2545 struct cma_multicast *mc = multicast->context; 2546 struct rdma_cm_event event; 2547 int ret; 2548 2549 id_priv = mc->id_priv; 2550 if (cma_disable_remove(id_priv, CMA_ADDR_BOUND) && 2551 cma_disable_remove(id_priv, CMA_ADDR_RESOLVED)) 2552 return 0; 2553 2554 mutex_lock(&id_priv->qp_mutex); 2555 if (!status && id_priv->id.qp) 2556 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, 2557 multicast->rec.mlid); 2558 mutex_unlock(&id_priv->qp_mutex); 2559 2560 memset(&event, 0, sizeof event); 2561 event.status = status; 2562 event.param.ud.private_data = mc->context; 2563 if (!status) { 2564 event.event = RDMA_CM_EVENT_MULTICAST_JOIN; 2565 ib_init_ah_from_mcmember(id_priv->id.device, 2566 id_priv->id.port_num, &multicast->rec, 2567 &event.param.ud.ah_attr); 2568 event.param.ud.qp_num = 0xFFFFFF; 2569 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); 2570 } else 2571 event.event = RDMA_CM_EVENT_MULTICAST_ERROR; 2572 2573 ret = id_priv->id.event_handler(&id_priv->id, &event); 2574 if (ret) { 2575 cma_exch(id_priv, CMA_DESTROYING); 2576 cma_enable_remove(id_priv); 2577 rdma_destroy_id(&id_priv->id); 2578 return 0; 2579 } 2580 2581 cma_enable_remove(id_priv); 2582 return 0; 2583 } 2584 2585 static void cma_set_mgid(struct rdma_id_private *id_priv, 2586 struct sockaddr *addr, union ib_gid *mgid) 2587 { 2588 unsigned char mc_map[MAX_ADDR_LEN]; 2589 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2590 struct sockaddr_in *sin = (struct sockaddr_in *) addr; 2591 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; 2592 2593 if (cma_any_addr(addr)) { 2594 memset(mgid, 0, sizeof *mgid); 2595 } else if ((addr->sa_family == AF_INET6) && 2596 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) == 2597 0xFF10A01B)) { 2598 /* IPv6 address is an SA assigned MGID. */ 2599 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 2600 } else { 2601 ip_ib_mc_map(sin->sin_addr.s_addr, mc_map); 2602 if (id_priv->id.ps == RDMA_PS_UDP) 2603 mc_map[7] = 0x01; /* Use RDMA CM signature */ 2604 mc_map[8] = ib_addr_get_pkey(dev_addr) >> 8; 2605 mc_map[9] = (unsigned char) ib_addr_get_pkey(dev_addr); 2606 *mgid = *(union ib_gid *) (mc_map + 4); 2607 } 2608 } 2609 2610 static int cma_join_ib_multicast(struct rdma_id_private *id_priv, 2611 struct cma_multicast *mc) 2612 { 2613 struct ib_sa_mcmember_rec rec; 2614 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2615 ib_sa_comp_mask comp_mask; 2616 int ret; 2617 2618 ib_addr_get_mgid(dev_addr, &rec.mgid); 2619 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, 2620 &rec.mgid, &rec); 2621 if (ret) 2622 return ret; 2623 2624 cma_set_mgid(id_priv, &mc->addr, &rec.mgid); 2625 if (id_priv->id.ps == RDMA_PS_UDP) 2626 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 2627 ib_addr_get_sgid(dev_addr, &rec.port_gid); 2628 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 2629 rec.join_state = 1; 2630 2631 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 2632 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | 2633 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | 2634 IB_SA_MCMEMBER_REC_FLOW_LABEL | 2635 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; 2636 2637 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, 2638 id_priv->id.port_num, &rec, 2639 comp_mask, GFP_KERNEL, 2640 cma_ib_mc_handler, mc); 2641 if (IS_ERR(mc->multicast.ib)) 2642 return PTR_ERR(mc->multicast.ib); 2643 2644 return 0; 2645 } 2646 2647 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, 2648 void *context) 2649 { 2650 struct rdma_id_private *id_priv; 2651 struct cma_multicast *mc; 2652 int ret; 2653 2654 id_priv = container_of(id, struct rdma_id_private, id); 2655 if (!cma_comp(id_priv, CMA_ADDR_BOUND) && 2656 !cma_comp(id_priv, CMA_ADDR_RESOLVED)) 2657 return -EINVAL; 2658 2659 mc = kmalloc(sizeof *mc, GFP_KERNEL); 2660 if (!mc) 2661 return -ENOMEM; 2662 2663 memcpy(&mc->addr, addr, ip_addr_size(addr)); 2664 mc->context = context; 2665 mc->id_priv = id_priv; 2666 2667 spin_lock(&id_priv->lock); 2668 list_add(&mc->list, &id_priv->mc_list); 2669 spin_unlock(&id_priv->lock); 2670 2671 switch (rdma_node_get_transport(id->device->node_type)) { 2672 case RDMA_TRANSPORT_IB: 2673 ret = cma_join_ib_multicast(id_priv, mc); 2674 break; 2675 default: 2676 ret = -ENOSYS; 2677 break; 2678 } 2679 2680 if (ret) { 2681 spin_lock_irq(&id_priv->lock); 2682 list_del(&mc->list); 2683 spin_unlock_irq(&id_priv->lock); 2684 kfree(mc); 2685 } 2686 return ret; 2687 } 2688 EXPORT_SYMBOL(rdma_join_multicast); 2689 2690 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) 2691 { 2692 struct rdma_id_private *id_priv; 2693 struct cma_multicast *mc; 2694 2695 id_priv = container_of(id, struct rdma_id_private, id); 2696 spin_lock_irq(&id_priv->lock); 2697 list_for_each_entry(mc, &id_priv->mc_list, list) { 2698 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) { 2699 list_del(&mc->list); 2700 spin_unlock_irq(&id_priv->lock); 2701 2702 if (id->qp) 2703 ib_detach_mcast(id->qp, 2704 &mc->multicast.ib->rec.mgid, 2705 mc->multicast.ib->rec.mlid); 2706 ib_sa_free_multicast(mc->multicast.ib); 2707 kfree(mc); 2708 return; 2709 } 2710 } 2711 spin_unlock_irq(&id_priv->lock); 2712 } 2713 EXPORT_SYMBOL(rdma_leave_multicast); 2714 2715 static void cma_add_one(struct ib_device *device) 2716 { 2717 struct cma_device *cma_dev; 2718 struct rdma_id_private *id_priv; 2719 2720 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); 2721 if (!cma_dev) 2722 return; 2723 2724 cma_dev->device = device; 2725 2726 init_completion(&cma_dev->comp); 2727 atomic_set(&cma_dev->refcount, 1); 2728 INIT_LIST_HEAD(&cma_dev->id_list); 2729 ib_set_client_data(device, &cma_client, cma_dev); 2730 2731 mutex_lock(&lock); 2732 list_add_tail(&cma_dev->list, &dev_list); 2733 list_for_each_entry(id_priv, &listen_any_list, list) 2734 cma_listen_on_dev(id_priv, cma_dev); 2735 mutex_unlock(&lock); 2736 } 2737 2738 static int cma_remove_id_dev(struct rdma_id_private *id_priv) 2739 { 2740 struct rdma_cm_event event; 2741 enum cma_state state; 2742 2743 /* Record that we want to remove the device */ 2744 state = cma_exch(id_priv, CMA_DEVICE_REMOVAL); 2745 if (state == CMA_DESTROYING) 2746 return 0; 2747 2748 cma_cancel_operation(id_priv, state); 2749 wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove)); 2750 2751 /* Check for destruction from another callback. */ 2752 if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL)) 2753 return 0; 2754 2755 memset(&event, 0, sizeof event); 2756 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; 2757 return id_priv->id.event_handler(&id_priv->id, &event); 2758 } 2759 2760 static void cma_process_remove(struct cma_device *cma_dev) 2761 { 2762 struct rdma_id_private *id_priv; 2763 int ret; 2764 2765 mutex_lock(&lock); 2766 while (!list_empty(&cma_dev->id_list)) { 2767 id_priv = list_entry(cma_dev->id_list.next, 2768 struct rdma_id_private, list); 2769 2770 list_del(&id_priv->listen_list); 2771 list_del_init(&id_priv->list); 2772 atomic_inc(&id_priv->refcount); 2773 mutex_unlock(&lock); 2774 2775 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); 2776 cma_deref_id(id_priv); 2777 if (ret) 2778 rdma_destroy_id(&id_priv->id); 2779 2780 mutex_lock(&lock); 2781 } 2782 mutex_unlock(&lock); 2783 2784 cma_deref_dev(cma_dev); 2785 wait_for_completion(&cma_dev->comp); 2786 } 2787 2788 static void cma_remove_one(struct ib_device *device) 2789 { 2790 struct cma_device *cma_dev; 2791 2792 cma_dev = ib_get_client_data(device, &cma_client); 2793 if (!cma_dev) 2794 return; 2795 2796 mutex_lock(&lock); 2797 list_del(&cma_dev->list); 2798 mutex_unlock(&lock); 2799 2800 cma_process_remove(cma_dev); 2801 kfree(cma_dev); 2802 } 2803 2804 static int cma_init(void) 2805 { 2806 int ret, low, high, remaining; 2807 2808 get_random_bytes(&next_port, sizeof next_port); 2809 inet_get_local_port_range(&low, &high); 2810 remaining = (high - low) + 1; 2811 next_port = ((unsigned int) next_port % remaining) + low; 2812 2813 cma_wq = create_singlethread_workqueue("rdma_cm"); 2814 if (!cma_wq) 2815 return -ENOMEM; 2816 2817 ib_sa_register_client(&sa_client); 2818 rdma_addr_register_client(&addr_client); 2819 2820 ret = ib_register_client(&cma_client); 2821 if (ret) 2822 goto err; 2823 return 0; 2824 2825 err: 2826 rdma_addr_unregister_client(&addr_client); 2827 ib_sa_unregister_client(&sa_client); 2828 destroy_workqueue(cma_wq); 2829 return ret; 2830 } 2831 2832 static void cma_cleanup(void) 2833 { 2834 ib_unregister_client(&cma_client); 2835 rdma_addr_unregister_client(&addr_client); 2836 ib_sa_unregister_client(&sa_client); 2837 destroy_workqueue(cma_wq); 2838 idr_destroy(&sdp_ps); 2839 idr_destroy(&tcp_ps); 2840 idr_destroy(&udp_ps); 2841 idr_destroy(&ipoib_ps); 2842 } 2843 2844 module_init(cma_init); 2845 module_exit(cma_cleanup); 2846