1 /* 2 * Copyright (c) 2005 Voltaire Inc. All rights reserved. 3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. 4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. 5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 6 * 7 * This Software is licensed under one of the following licenses: 8 * 9 * 1) under the terms of the "Common Public License 1.0" a copy of which is 10 * available from the Open Source Initiative, see 11 * http://www.opensource.org/licenses/cpl.php. 12 * 13 * 2) under the terms of the "The BSD License" a copy of which is 14 * available from the Open Source Initiative, see 15 * http://www.opensource.org/licenses/bsd-license.php. 16 * 17 * 3) under the terms of the "GNU General Public License (GPL) Version 2" a 18 * copy of which is available from the Open Source Initiative, see 19 * http://www.opensource.org/licenses/gpl-license.php. 20 * 21 * Licensee has the right to choose one of the above licenses. 22 * 23 * Redistributions of source code must retain the above copyright 24 * notice and one of the license notices. 25 * 26 * Redistributions in binary form must reproduce both the above copyright 27 * notice, one of the license notices in the documentation 28 * and/or other materials provided with the distribution. 29 * 30 */ 31 32 #include <linux/completion.h> 33 #include <linux/in.h> 34 #include <linux/in6.h> 35 #include <linux/mutex.h> 36 #include <linux/random.h> 37 #include <linux/idr.h> 38 #include <linux/inetdevice.h> 39 40 #include <net/tcp.h> 41 42 #include <rdma/rdma_cm.h> 43 #include <rdma/rdma_cm_ib.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/ib_cm.h> 46 #include <rdma/ib_sa.h> 47 #include <rdma/iw_cm.h> 48 49 MODULE_AUTHOR("Sean Hefty"); 50 MODULE_DESCRIPTION("Generic RDMA CM Agent"); 51 MODULE_LICENSE("Dual BSD/GPL"); 52 53 #define CMA_CM_RESPONSE_TIMEOUT 20 54 #define CMA_MAX_CM_RETRIES 15 55 56 static void cma_add_one(struct ib_device *device); 57 static void cma_remove_one(struct ib_device *device); 58 59 static struct ib_client cma_client = { 60 .name = "cma", 61 .add = cma_add_one, 62 .remove = cma_remove_one 63 }; 64 65 static struct ib_sa_client sa_client; 66 static struct rdma_addr_client addr_client; 67 static LIST_HEAD(dev_list); 68 static LIST_HEAD(listen_any_list); 69 static DEFINE_MUTEX(lock); 70 static struct workqueue_struct *cma_wq; 71 static DEFINE_IDR(sdp_ps); 72 static DEFINE_IDR(tcp_ps); 73 static DEFINE_IDR(udp_ps); 74 static DEFINE_IDR(ipoib_ps); 75 static int next_port; 76 77 struct cma_device { 78 struct list_head list; 79 struct ib_device *device; 80 struct completion comp; 81 atomic_t refcount; 82 struct list_head id_list; 83 }; 84 85 enum cma_state { 86 CMA_IDLE, 87 CMA_ADDR_QUERY, 88 CMA_ADDR_RESOLVED, 89 CMA_ROUTE_QUERY, 90 CMA_ROUTE_RESOLVED, 91 CMA_CONNECT, 92 CMA_DISCONNECT, 93 CMA_ADDR_BOUND, 94 CMA_LISTEN, 95 CMA_DEVICE_REMOVAL, 96 CMA_DESTROYING 97 }; 98 99 struct rdma_bind_list { 100 struct idr *ps; 101 struct hlist_head owners; 102 unsigned short port; 103 }; 104 105 /* 106 * Device removal can occur at anytime, so we need extra handling to 107 * serialize notifying the user of device removal with other callbacks. 108 * We do this by disabling removal notification while a callback is in process, 109 * and reporting it after the callback completes. 110 */ 111 struct rdma_id_private { 112 struct rdma_cm_id id; 113 114 struct rdma_bind_list *bind_list; 115 struct hlist_node node; 116 struct list_head list; 117 struct list_head listen_list; 118 struct cma_device *cma_dev; 119 struct list_head mc_list; 120 121 enum cma_state state; 122 spinlock_t lock; 123 struct completion comp; 124 atomic_t refcount; 125 wait_queue_head_t wait_remove; 126 atomic_t dev_remove; 127 128 int backlog; 129 int timeout_ms; 130 struct ib_sa_query *query; 131 int query_id; 132 union { 133 struct ib_cm_id *ib; 134 struct iw_cm_id *iw; 135 } cm_id; 136 137 u32 seq_num; 138 u32 qkey; 139 u32 qp_num; 140 u8 srq; 141 }; 142 143 struct cma_multicast { 144 struct rdma_id_private *id_priv; 145 union { 146 struct ib_sa_multicast *ib; 147 } multicast; 148 struct list_head list; 149 void *context; 150 struct sockaddr addr; 151 u8 pad[sizeof(struct sockaddr_in6) - 152 sizeof(struct sockaddr)]; 153 }; 154 155 struct cma_work { 156 struct work_struct work; 157 struct rdma_id_private *id; 158 enum cma_state old_state; 159 enum cma_state new_state; 160 struct rdma_cm_event event; 161 }; 162 163 union cma_ip_addr { 164 struct in6_addr ip6; 165 struct { 166 __u32 pad[3]; 167 __u32 addr; 168 } ip4; 169 }; 170 171 struct cma_hdr { 172 u8 cma_version; 173 u8 ip_version; /* IP version: 7:4 */ 174 __u16 port; 175 union cma_ip_addr src_addr; 176 union cma_ip_addr dst_addr; 177 }; 178 179 struct sdp_hh { 180 u8 bsdh[16]; 181 u8 sdp_version; /* Major version: 7:4 */ 182 u8 ip_version; /* IP version: 7:4 */ 183 u8 sdp_specific1[10]; 184 __u16 port; 185 __u16 sdp_specific2; 186 union cma_ip_addr src_addr; 187 union cma_ip_addr dst_addr; 188 }; 189 190 struct sdp_hah { 191 u8 bsdh[16]; 192 u8 sdp_version; 193 }; 194 195 #define CMA_VERSION 0x00 196 #define SDP_MAJ_VERSION 0x2 197 198 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp) 199 { 200 unsigned long flags; 201 int ret; 202 203 spin_lock_irqsave(&id_priv->lock, flags); 204 ret = (id_priv->state == comp); 205 spin_unlock_irqrestore(&id_priv->lock, flags); 206 return ret; 207 } 208 209 static int cma_comp_exch(struct rdma_id_private *id_priv, 210 enum cma_state comp, enum cma_state exch) 211 { 212 unsigned long flags; 213 int ret; 214 215 spin_lock_irqsave(&id_priv->lock, flags); 216 if ((ret = (id_priv->state == comp))) 217 id_priv->state = exch; 218 spin_unlock_irqrestore(&id_priv->lock, flags); 219 return ret; 220 } 221 222 static enum cma_state cma_exch(struct rdma_id_private *id_priv, 223 enum cma_state exch) 224 { 225 unsigned long flags; 226 enum cma_state old; 227 228 spin_lock_irqsave(&id_priv->lock, flags); 229 old = id_priv->state; 230 id_priv->state = exch; 231 spin_unlock_irqrestore(&id_priv->lock, flags); 232 return old; 233 } 234 235 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr) 236 { 237 return hdr->ip_version >> 4; 238 } 239 240 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) 241 { 242 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); 243 } 244 245 static inline u8 sdp_get_majv(u8 sdp_version) 246 { 247 return sdp_version >> 4; 248 } 249 250 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh) 251 { 252 return hh->ip_version >> 4; 253 } 254 255 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver) 256 { 257 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF); 258 } 259 260 static inline int cma_is_ud_ps(enum rdma_port_space ps) 261 { 262 return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB); 263 } 264 265 static void cma_attach_to_dev(struct rdma_id_private *id_priv, 266 struct cma_device *cma_dev) 267 { 268 atomic_inc(&cma_dev->refcount); 269 id_priv->cma_dev = cma_dev; 270 id_priv->id.device = cma_dev->device; 271 list_add_tail(&id_priv->list, &cma_dev->id_list); 272 } 273 274 static inline void cma_deref_dev(struct cma_device *cma_dev) 275 { 276 if (atomic_dec_and_test(&cma_dev->refcount)) 277 complete(&cma_dev->comp); 278 } 279 280 static void cma_detach_from_dev(struct rdma_id_private *id_priv) 281 { 282 list_del(&id_priv->list); 283 cma_deref_dev(id_priv->cma_dev); 284 id_priv->cma_dev = NULL; 285 } 286 287 static int cma_set_qkey(struct ib_device *device, u8 port_num, 288 enum rdma_port_space ps, 289 struct rdma_dev_addr *dev_addr, u32 *qkey) 290 { 291 struct ib_sa_mcmember_rec rec; 292 int ret = 0; 293 294 switch (ps) { 295 case RDMA_PS_UDP: 296 *qkey = RDMA_UDP_QKEY; 297 break; 298 case RDMA_PS_IPOIB: 299 ib_addr_get_mgid(dev_addr, &rec.mgid); 300 ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec); 301 *qkey = be32_to_cpu(rec.qkey); 302 break; 303 default: 304 break; 305 } 306 return ret; 307 } 308 309 static int cma_acquire_dev(struct rdma_id_private *id_priv) 310 { 311 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 312 struct cma_device *cma_dev; 313 union ib_gid gid; 314 int ret = -ENODEV; 315 316 switch (rdma_node_get_transport(dev_addr->dev_type)) { 317 case RDMA_TRANSPORT_IB: 318 ib_addr_get_sgid(dev_addr, &gid); 319 break; 320 case RDMA_TRANSPORT_IWARP: 321 iw_addr_get_sgid(dev_addr, &gid); 322 break; 323 default: 324 return -ENODEV; 325 } 326 327 list_for_each_entry(cma_dev, &dev_list, list) { 328 ret = ib_find_cached_gid(cma_dev->device, &gid, 329 &id_priv->id.port_num, NULL); 330 if (!ret) { 331 ret = cma_set_qkey(cma_dev->device, 332 id_priv->id.port_num, 333 id_priv->id.ps, dev_addr, 334 &id_priv->qkey); 335 if (!ret) 336 cma_attach_to_dev(id_priv, cma_dev); 337 break; 338 } 339 } 340 return ret; 341 } 342 343 static void cma_deref_id(struct rdma_id_private *id_priv) 344 { 345 if (atomic_dec_and_test(&id_priv->refcount)) 346 complete(&id_priv->comp); 347 } 348 349 static void cma_release_remove(struct rdma_id_private *id_priv) 350 { 351 if (atomic_dec_and_test(&id_priv->dev_remove)) 352 wake_up(&id_priv->wait_remove); 353 } 354 355 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler, 356 void *context, enum rdma_port_space ps) 357 { 358 struct rdma_id_private *id_priv; 359 360 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); 361 if (!id_priv) 362 return ERR_PTR(-ENOMEM); 363 364 id_priv->state = CMA_IDLE; 365 id_priv->id.context = context; 366 id_priv->id.event_handler = event_handler; 367 id_priv->id.ps = ps; 368 spin_lock_init(&id_priv->lock); 369 init_completion(&id_priv->comp); 370 atomic_set(&id_priv->refcount, 1); 371 init_waitqueue_head(&id_priv->wait_remove); 372 atomic_set(&id_priv->dev_remove, 0); 373 INIT_LIST_HEAD(&id_priv->listen_list); 374 INIT_LIST_HEAD(&id_priv->mc_list); 375 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); 376 377 return &id_priv->id; 378 } 379 EXPORT_SYMBOL(rdma_create_id); 380 381 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 382 { 383 struct ib_qp_attr qp_attr; 384 int qp_attr_mask, ret; 385 386 qp_attr.qp_state = IB_QPS_INIT; 387 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 388 if (ret) 389 return ret; 390 391 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); 392 if (ret) 393 return ret; 394 395 qp_attr.qp_state = IB_QPS_RTR; 396 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); 397 if (ret) 398 return ret; 399 400 qp_attr.qp_state = IB_QPS_RTS; 401 qp_attr.sq_psn = 0; 402 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); 403 404 return ret; 405 } 406 407 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 408 { 409 struct ib_qp_attr qp_attr; 410 int qp_attr_mask, ret; 411 412 qp_attr.qp_state = IB_QPS_INIT; 413 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 414 if (ret) 415 return ret; 416 417 return ib_modify_qp(qp, &qp_attr, qp_attr_mask); 418 } 419 420 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, 421 struct ib_qp_init_attr *qp_init_attr) 422 { 423 struct rdma_id_private *id_priv; 424 struct ib_qp *qp; 425 int ret; 426 427 id_priv = container_of(id, struct rdma_id_private, id); 428 if (id->device != pd->device) 429 return -EINVAL; 430 431 qp = ib_create_qp(pd, qp_init_attr); 432 if (IS_ERR(qp)) 433 return PTR_ERR(qp); 434 435 if (cma_is_ud_ps(id_priv->id.ps)) 436 ret = cma_init_ud_qp(id_priv, qp); 437 else 438 ret = cma_init_conn_qp(id_priv, qp); 439 if (ret) 440 goto err; 441 442 id->qp = qp; 443 id_priv->qp_num = qp->qp_num; 444 id_priv->srq = (qp->srq != NULL); 445 return 0; 446 err: 447 ib_destroy_qp(qp); 448 return ret; 449 } 450 EXPORT_SYMBOL(rdma_create_qp); 451 452 void rdma_destroy_qp(struct rdma_cm_id *id) 453 { 454 ib_destroy_qp(id->qp); 455 } 456 EXPORT_SYMBOL(rdma_destroy_qp); 457 458 static int cma_modify_qp_rtr(struct rdma_cm_id *id) 459 { 460 struct ib_qp_attr qp_attr; 461 int qp_attr_mask, ret; 462 463 if (!id->qp) 464 return 0; 465 466 /* Need to update QP attributes from default values. */ 467 qp_attr.qp_state = IB_QPS_INIT; 468 ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask); 469 if (ret) 470 return ret; 471 472 ret = ib_modify_qp(id->qp, &qp_attr, qp_attr_mask); 473 if (ret) 474 return ret; 475 476 qp_attr.qp_state = IB_QPS_RTR; 477 ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask); 478 if (ret) 479 return ret; 480 481 return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask); 482 } 483 484 static int cma_modify_qp_rts(struct rdma_cm_id *id) 485 { 486 struct ib_qp_attr qp_attr; 487 int qp_attr_mask, ret; 488 489 if (!id->qp) 490 return 0; 491 492 qp_attr.qp_state = IB_QPS_RTS; 493 ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask); 494 if (ret) 495 return ret; 496 497 return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask); 498 } 499 500 static int cma_modify_qp_err(struct rdma_cm_id *id) 501 { 502 struct ib_qp_attr qp_attr; 503 504 if (!id->qp) 505 return 0; 506 507 qp_attr.qp_state = IB_QPS_ERR; 508 return ib_modify_qp(id->qp, &qp_attr, IB_QP_STATE); 509 } 510 511 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, 512 struct ib_qp_attr *qp_attr, int *qp_attr_mask) 513 { 514 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 515 int ret; 516 517 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, 518 ib_addr_get_pkey(dev_addr), 519 &qp_attr->pkey_index); 520 if (ret) 521 return ret; 522 523 qp_attr->port_num = id_priv->id.port_num; 524 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; 525 526 if (cma_is_ud_ps(id_priv->id.ps)) { 527 qp_attr->qkey = id_priv->qkey; 528 *qp_attr_mask |= IB_QP_QKEY; 529 } else { 530 qp_attr->qp_access_flags = 0; 531 *qp_attr_mask |= IB_QP_ACCESS_FLAGS; 532 } 533 return 0; 534 } 535 536 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, 537 int *qp_attr_mask) 538 { 539 struct rdma_id_private *id_priv; 540 int ret = 0; 541 542 id_priv = container_of(id, struct rdma_id_private, id); 543 switch (rdma_node_get_transport(id_priv->id.device->node_type)) { 544 case RDMA_TRANSPORT_IB: 545 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps)) 546 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); 547 else 548 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, 549 qp_attr_mask); 550 if (qp_attr->qp_state == IB_QPS_RTR) 551 qp_attr->rq_psn = id_priv->seq_num; 552 break; 553 case RDMA_TRANSPORT_IWARP: 554 if (!id_priv->cm_id.iw) { 555 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE; 556 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; 557 } else 558 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, 559 qp_attr_mask); 560 break; 561 default: 562 ret = -ENOSYS; 563 break; 564 } 565 566 return ret; 567 } 568 EXPORT_SYMBOL(rdma_init_qp_attr); 569 570 static inline int cma_zero_addr(struct sockaddr *addr) 571 { 572 struct in6_addr *ip6; 573 574 if (addr->sa_family == AF_INET) 575 return ZERONET(((struct sockaddr_in *) addr)->sin_addr.s_addr); 576 else { 577 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr; 578 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] | 579 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0; 580 } 581 } 582 583 static inline int cma_loopback_addr(struct sockaddr *addr) 584 { 585 return LOOPBACK(((struct sockaddr_in *) addr)->sin_addr.s_addr); 586 } 587 588 static inline int cma_any_addr(struct sockaddr *addr) 589 { 590 return cma_zero_addr(addr) || cma_loopback_addr(addr); 591 } 592 593 static inline __be16 cma_port(struct sockaddr *addr) 594 { 595 if (addr->sa_family == AF_INET) 596 return ((struct sockaddr_in *) addr)->sin_port; 597 else 598 return ((struct sockaddr_in6 *) addr)->sin6_port; 599 } 600 601 static inline int cma_any_port(struct sockaddr *addr) 602 { 603 return !cma_port(addr); 604 } 605 606 static int cma_get_net_info(void *hdr, enum rdma_port_space ps, 607 u8 *ip_ver, __u16 *port, 608 union cma_ip_addr **src, union cma_ip_addr **dst) 609 { 610 switch (ps) { 611 case RDMA_PS_SDP: 612 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) != 613 SDP_MAJ_VERSION) 614 return -EINVAL; 615 616 *ip_ver = sdp_get_ip_ver(hdr); 617 *port = ((struct sdp_hh *) hdr)->port; 618 *src = &((struct sdp_hh *) hdr)->src_addr; 619 *dst = &((struct sdp_hh *) hdr)->dst_addr; 620 break; 621 default: 622 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION) 623 return -EINVAL; 624 625 *ip_ver = cma_get_ip_ver(hdr); 626 *port = ((struct cma_hdr *) hdr)->port; 627 *src = &((struct cma_hdr *) hdr)->src_addr; 628 *dst = &((struct cma_hdr *) hdr)->dst_addr; 629 break; 630 } 631 632 if (*ip_ver != 4 && *ip_ver != 6) 633 return -EINVAL; 634 return 0; 635 } 636 637 static void cma_save_net_info(struct rdma_addr *addr, 638 struct rdma_addr *listen_addr, 639 u8 ip_ver, __u16 port, 640 union cma_ip_addr *src, union cma_ip_addr *dst) 641 { 642 struct sockaddr_in *listen4, *ip4; 643 struct sockaddr_in6 *listen6, *ip6; 644 645 switch (ip_ver) { 646 case 4: 647 listen4 = (struct sockaddr_in *) &listen_addr->src_addr; 648 ip4 = (struct sockaddr_in *) &addr->src_addr; 649 ip4->sin_family = listen4->sin_family; 650 ip4->sin_addr.s_addr = dst->ip4.addr; 651 ip4->sin_port = listen4->sin_port; 652 653 ip4 = (struct sockaddr_in *) &addr->dst_addr; 654 ip4->sin_family = listen4->sin_family; 655 ip4->sin_addr.s_addr = src->ip4.addr; 656 ip4->sin_port = port; 657 break; 658 case 6: 659 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr; 660 ip6 = (struct sockaddr_in6 *) &addr->src_addr; 661 ip6->sin6_family = listen6->sin6_family; 662 ip6->sin6_addr = dst->ip6; 663 ip6->sin6_port = listen6->sin6_port; 664 665 ip6 = (struct sockaddr_in6 *) &addr->dst_addr; 666 ip6->sin6_family = listen6->sin6_family; 667 ip6->sin6_addr = src->ip6; 668 ip6->sin6_port = port; 669 break; 670 default: 671 break; 672 } 673 } 674 675 static inline int cma_user_data_offset(enum rdma_port_space ps) 676 { 677 switch (ps) { 678 case RDMA_PS_SDP: 679 return 0; 680 default: 681 return sizeof(struct cma_hdr); 682 } 683 } 684 685 static void cma_cancel_route(struct rdma_id_private *id_priv) 686 { 687 switch (rdma_node_get_transport(id_priv->id.device->node_type)) { 688 case RDMA_TRANSPORT_IB: 689 if (id_priv->query) 690 ib_sa_cancel_query(id_priv->query_id, id_priv->query); 691 break; 692 default: 693 break; 694 } 695 } 696 697 static inline int cma_internal_listen(struct rdma_id_private *id_priv) 698 { 699 return (id_priv->state == CMA_LISTEN) && id_priv->cma_dev && 700 cma_any_addr(&id_priv->id.route.addr.src_addr); 701 } 702 703 static void cma_destroy_listen(struct rdma_id_private *id_priv) 704 { 705 cma_exch(id_priv, CMA_DESTROYING); 706 707 if (id_priv->cma_dev) { 708 switch (rdma_node_get_transport(id_priv->id.device->node_type)) { 709 case RDMA_TRANSPORT_IB: 710 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) 711 ib_destroy_cm_id(id_priv->cm_id.ib); 712 break; 713 case RDMA_TRANSPORT_IWARP: 714 if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw)) 715 iw_destroy_cm_id(id_priv->cm_id.iw); 716 break; 717 default: 718 break; 719 } 720 cma_detach_from_dev(id_priv); 721 } 722 list_del(&id_priv->listen_list); 723 724 cma_deref_id(id_priv); 725 wait_for_completion(&id_priv->comp); 726 727 kfree(id_priv); 728 } 729 730 static void cma_cancel_listens(struct rdma_id_private *id_priv) 731 { 732 struct rdma_id_private *dev_id_priv; 733 734 mutex_lock(&lock); 735 list_del(&id_priv->list); 736 737 while (!list_empty(&id_priv->listen_list)) { 738 dev_id_priv = list_entry(id_priv->listen_list.next, 739 struct rdma_id_private, listen_list); 740 cma_destroy_listen(dev_id_priv); 741 } 742 mutex_unlock(&lock); 743 } 744 745 static void cma_cancel_operation(struct rdma_id_private *id_priv, 746 enum cma_state state) 747 { 748 switch (state) { 749 case CMA_ADDR_QUERY: 750 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); 751 break; 752 case CMA_ROUTE_QUERY: 753 cma_cancel_route(id_priv); 754 break; 755 case CMA_LISTEN: 756 if (cma_any_addr(&id_priv->id.route.addr.src_addr) && 757 !id_priv->cma_dev) 758 cma_cancel_listens(id_priv); 759 break; 760 default: 761 break; 762 } 763 } 764 765 static void cma_release_port(struct rdma_id_private *id_priv) 766 { 767 struct rdma_bind_list *bind_list = id_priv->bind_list; 768 769 if (!bind_list) 770 return; 771 772 mutex_lock(&lock); 773 hlist_del(&id_priv->node); 774 if (hlist_empty(&bind_list->owners)) { 775 idr_remove(bind_list->ps, bind_list->port); 776 kfree(bind_list); 777 } 778 mutex_unlock(&lock); 779 } 780 781 static void cma_leave_mc_groups(struct rdma_id_private *id_priv) 782 { 783 struct cma_multicast *mc; 784 785 while (!list_empty(&id_priv->mc_list)) { 786 mc = container_of(id_priv->mc_list.next, 787 struct cma_multicast, list); 788 list_del(&mc->list); 789 ib_sa_free_multicast(mc->multicast.ib); 790 kfree(mc); 791 } 792 } 793 794 void rdma_destroy_id(struct rdma_cm_id *id) 795 { 796 struct rdma_id_private *id_priv; 797 enum cma_state state; 798 799 id_priv = container_of(id, struct rdma_id_private, id); 800 state = cma_exch(id_priv, CMA_DESTROYING); 801 cma_cancel_operation(id_priv, state); 802 803 mutex_lock(&lock); 804 if (id_priv->cma_dev) { 805 mutex_unlock(&lock); 806 switch (rdma_node_get_transport(id->device->node_type)) { 807 case RDMA_TRANSPORT_IB: 808 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) 809 ib_destroy_cm_id(id_priv->cm_id.ib); 810 break; 811 case RDMA_TRANSPORT_IWARP: 812 if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw)) 813 iw_destroy_cm_id(id_priv->cm_id.iw); 814 break; 815 default: 816 break; 817 } 818 cma_leave_mc_groups(id_priv); 819 mutex_lock(&lock); 820 cma_detach_from_dev(id_priv); 821 } 822 mutex_unlock(&lock); 823 824 cma_release_port(id_priv); 825 cma_deref_id(id_priv); 826 wait_for_completion(&id_priv->comp); 827 828 kfree(id_priv->id.route.path_rec); 829 kfree(id_priv); 830 } 831 EXPORT_SYMBOL(rdma_destroy_id); 832 833 static int cma_rep_recv(struct rdma_id_private *id_priv) 834 { 835 int ret; 836 837 ret = cma_modify_qp_rtr(&id_priv->id); 838 if (ret) 839 goto reject; 840 841 ret = cma_modify_qp_rts(&id_priv->id); 842 if (ret) 843 goto reject; 844 845 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); 846 if (ret) 847 goto reject; 848 849 return 0; 850 reject: 851 cma_modify_qp_err(&id_priv->id); 852 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, 853 NULL, 0, NULL, 0); 854 return ret; 855 } 856 857 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data) 858 { 859 if (id_priv->id.ps == RDMA_PS_SDP && 860 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) != 861 SDP_MAJ_VERSION) 862 return -EINVAL; 863 864 return 0; 865 } 866 867 static void cma_set_rep_event_data(struct rdma_cm_event *event, 868 struct ib_cm_rep_event_param *rep_data, 869 void *private_data) 870 { 871 event->param.conn.private_data = private_data; 872 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; 873 event->param.conn.responder_resources = rep_data->responder_resources; 874 event->param.conn.initiator_depth = rep_data->initiator_depth; 875 event->param.conn.flow_control = rep_data->flow_control; 876 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; 877 event->param.conn.srq = rep_data->srq; 878 event->param.conn.qp_num = rep_data->remote_qpn; 879 } 880 881 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 882 { 883 struct rdma_id_private *id_priv = cm_id->context; 884 struct rdma_cm_event event; 885 int ret = 0; 886 887 atomic_inc(&id_priv->dev_remove); 888 if (!cma_comp(id_priv, CMA_CONNECT)) 889 goto out; 890 891 memset(&event, 0, sizeof event); 892 switch (ib_event->event) { 893 case IB_CM_REQ_ERROR: 894 case IB_CM_REP_ERROR: 895 event.event = RDMA_CM_EVENT_UNREACHABLE; 896 event.status = -ETIMEDOUT; 897 break; 898 case IB_CM_REP_RECEIVED: 899 event.status = cma_verify_rep(id_priv, ib_event->private_data); 900 if (event.status) 901 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 902 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) { 903 event.status = cma_rep_recv(id_priv); 904 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : 905 RDMA_CM_EVENT_ESTABLISHED; 906 } else 907 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; 908 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, 909 ib_event->private_data); 910 break; 911 case IB_CM_RTU_RECEIVED: 912 case IB_CM_USER_ESTABLISHED: 913 event.event = RDMA_CM_EVENT_ESTABLISHED; 914 break; 915 case IB_CM_DREQ_ERROR: 916 event.status = -ETIMEDOUT; /* fall through */ 917 case IB_CM_DREQ_RECEIVED: 918 case IB_CM_DREP_RECEIVED: 919 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT)) 920 goto out; 921 event.event = RDMA_CM_EVENT_DISCONNECTED; 922 break; 923 case IB_CM_TIMEWAIT_EXIT: 924 case IB_CM_MRA_RECEIVED: 925 /* ignore event */ 926 goto out; 927 case IB_CM_REJ_RECEIVED: 928 cma_modify_qp_err(&id_priv->id); 929 event.status = ib_event->param.rej_rcvd.reason; 930 event.event = RDMA_CM_EVENT_REJECTED; 931 event.param.conn.private_data = ib_event->private_data; 932 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; 933 break; 934 default: 935 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d", 936 ib_event->event); 937 goto out; 938 } 939 940 ret = id_priv->id.event_handler(&id_priv->id, &event); 941 if (ret) { 942 /* Destroy the CM ID by returning a non-zero value. */ 943 id_priv->cm_id.ib = NULL; 944 cma_exch(id_priv, CMA_DESTROYING); 945 cma_release_remove(id_priv); 946 rdma_destroy_id(&id_priv->id); 947 return ret; 948 } 949 out: 950 cma_release_remove(id_priv); 951 return ret; 952 } 953 954 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, 955 struct ib_cm_event *ib_event) 956 { 957 struct rdma_id_private *id_priv; 958 struct rdma_cm_id *id; 959 struct rdma_route *rt; 960 union cma_ip_addr *src, *dst; 961 __u16 port; 962 u8 ip_ver; 963 964 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 965 &ip_ver, &port, &src, &dst)) 966 goto err; 967 968 id = rdma_create_id(listen_id->event_handler, listen_id->context, 969 listen_id->ps); 970 if (IS_ERR(id)) 971 goto err; 972 973 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 974 ip_ver, port, src, dst); 975 976 rt = &id->route; 977 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; 978 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths, 979 GFP_KERNEL); 980 if (!rt->path_rec) 981 goto destroy_id; 982 983 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path; 984 if (rt->num_paths == 2) 985 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; 986 987 ib_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); 988 ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); 989 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey)); 990 rt->addr.dev_addr.dev_type = RDMA_NODE_IB_CA; 991 992 id_priv = container_of(id, struct rdma_id_private, id); 993 id_priv->state = CMA_CONNECT; 994 return id_priv; 995 996 destroy_id: 997 rdma_destroy_id(id); 998 err: 999 return NULL; 1000 } 1001 1002 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id, 1003 struct ib_cm_event *ib_event) 1004 { 1005 struct rdma_id_private *id_priv; 1006 struct rdma_cm_id *id; 1007 union cma_ip_addr *src, *dst; 1008 __u16 port; 1009 u8 ip_ver; 1010 int ret; 1011 1012 id = rdma_create_id(listen_id->event_handler, listen_id->context, 1013 listen_id->ps); 1014 if (IS_ERR(id)) 1015 return NULL; 1016 1017 1018 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 1019 &ip_ver, &port, &src, &dst)) 1020 goto err; 1021 1022 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 1023 ip_ver, port, src, dst); 1024 1025 ret = rdma_translate_ip(&id->route.addr.src_addr, 1026 &id->route.addr.dev_addr); 1027 if (ret) 1028 goto err; 1029 1030 id_priv = container_of(id, struct rdma_id_private, id); 1031 id_priv->state = CMA_CONNECT; 1032 return id_priv; 1033 err: 1034 rdma_destroy_id(id); 1035 return NULL; 1036 } 1037 1038 static void cma_set_req_event_data(struct rdma_cm_event *event, 1039 struct ib_cm_req_event_param *req_data, 1040 void *private_data, int offset) 1041 { 1042 event->param.conn.private_data = private_data + offset; 1043 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; 1044 event->param.conn.responder_resources = req_data->responder_resources; 1045 event->param.conn.initiator_depth = req_data->initiator_depth; 1046 event->param.conn.flow_control = req_data->flow_control; 1047 event->param.conn.retry_count = req_data->retry_count; 1048 event->param.conn.rnr_retry_count = req_data->rnr_retry_count; 1049 event->param.conn.srq = req_data->srq; 1050 event->param.conn.qp_num = req_data->remote_qpn; 1051 } 1052 1053 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 1054 { 1055 struct rdma_id_private *listen_id, *conn_id; 1056 struct rdma_cm_event event; 1057 int offset, ret; 1058 1059 listen_id = cm_id->context; 1060 atomic_inc(&listen_id->dev_remove); 1061 if (!cma_comp(listen_id, CMA_LISTEN)) { 1062 ret = -ECONNABORTED; 1063 goto out; 1064 } 1065 1066 memset(&event, 0, sizeof event); 1067 offset = cma_user_data_offset(listen_id->id.ps); 1068 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1069 if (cma_is_ud_ps(listen_id->id.ps)) { 1070 conn_id = cma_new_udp_id(&listen_id->id, ib_event); 1071 event.param.ud.private_data = ib_event->private_data + offset; 1072 event.param.ud.private_data_len = 1073 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; 1074 } else { 1075 conn_id = cma_new_conn_id(&listen_id->id, ib_event); 1076 cma_set_req_event_data(&event, &ib_event->param.req_rcvd, 1077 ib_event->private_data, offset); 1078 } 1079 if (!conn_id) { 1080 ret = -ENOMEM; 1081 goto out; 1082 } 1083 1084 atomic_inc(&conn_id->dev_remove); 1085 mutex_lock(&lock); 1086 ret = cma_acquire_dev(conn_id); 1087 mutex_unlock(&lock); 1088 if (ret) 1089 goto release_conn_id; 1090 1091 conn_id->cm_id.ib = cm_id; 1092 cm_id->context = conn_id; 1093 cm_id->cm_handler = cma_ib_handler; 1094 1095 ret = conn_id->id.event_handler(&conn_id->id, &event); 1096 if (!ret) 1097 goto out; 1098 1099 /* Destroy the CM ID by returning a non-zero value. */ 1100 conn_id->cm_id.ib = NULL; 1101 1102 release_conn_id: 1103 cma_exch(conn_id, CMA_DESTROYING); 1104 cma_release_remove(conn_id); 1105 rdma_destroy_id(&conn_id->id); 1106 1107 out: 1108 cma_release_remove(listen_id); 1109 return ret; 1110 } 1111 1112 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr) 1113 { 1114 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr))); 1115 } 1116 1117 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr, 1118 struct ib_cm_compare_data *compare) 1119 { 1120 struct cma_hdr *cma_data, *cma_mask; 1121 struct sdp_hh *sdp_data, *sdp_mask; 1122 __u32 ip4_addr; 1123 struct in6_addr ip6_addr; 1124 1125 memset(compare, 0, sizeof *compare); 1126 cma_data = (void *) compare->data; 1127 cma_mask = (void *) compare->mask; 1128 sdp_data = (void *) compare->data; 1129 sdp_mask = (void *) compare->mask; 1130 1131 switch (addr->sa_family) { 1132 case AF_INET: 1133 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr; 1134 if (ps == RDMA_PS_SDP) { 1135 sdp_set_ip_ver(sdp_data, 4); 1136 sdp_set_ip_ver(sdp_mask, 0xF); 1137 sdp_data->dst_addr.ip4.addr = ip4_addr; 1138 sdp_mask->dst_addr.ip4.addr = ~0; 1139 } else { 1140 cma_set_ip_ver(cma_data, 4); 1141 cma_set_ip_ver(cma_mask, 0xF); 1142 cma_data->dst_addr.ip4.addr = ip4_addr; 1143 cma_mask->dst_addr.ip4.addr = ~0; 1144 } 1145 break; 1146 case AF_INET6: 1147 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr; 1148 if (ps == RDMA_PS_SDP) { 1149 sdp_set_ip_ver(sdp_data, 6); 1150 sdp_set_ip_ver(sdp_mask, 0xF); 1151 sdp_data->dst_addr.ip6 = ip6_addr; 1152 memset(&sdp_mask->dst_addr.ip6, 0xFF, 1153 sizeof sdp_mask->dst_addr.ip6); 1154 } else { 1155 cma_set_ip_ver(cma_data, 6); 1156 cma_set_ip_ver(cma_mask, 0xF); 1157 cma_data->dst_addr.ip6 = ip6_addr; 1158 memset(&cma_mask->dst_addr.ip6, 0xFF, 1159 sizeof cma_mask->dst_addr.ip6); 1160 } 1161 break; 1162 default: 1163 break; 1164 } 1165 } 1166 1167 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) 1168 { 1169 struct rdma_id_private *id_priv = iw_id->context; 1170 struct rdma_cm_event event; 1171 struct sockaddr_in *sin; 1172 int ret = 0; 1173 1174 memset(&event, 0, sizeof event); 1175 atomic_inc(&id_priv->dev_remove); 1176 1177 switch (iw_event->event) { 1178 case IW_CM_EVENT_CLOSE: 1179 event.event = RDMA_CM_EVENT_DISCONNECTED; 1180 break; 1181 case IW_CM_EVENT_CONNECT_REPLY: 1182 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1183 *sin = iw_event->local_addr; 1184 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr; 1185 *sin = iw_event->remote_addr; 1186 switch (iw_event->status) { 1187 case 0: 1188 event.event = RDMA_CM_EVENT_ESTABLISHED; 1189 break; 1190 case -ECONNRESET: 1191 case -ECONNREFUSED: 1192 event.event = RDMA_CM_EVENT_REJECTED; 1193 break; 1194 case -ETIMEDOUT: 1195 event.event = RDMA_CM_EVENT_UNREACHABLE; 1196 break; 1197 default: 1198 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 1199 break; 1200 } 1201 break; 1202 case IW_CM_EVENT_ESTABLISHED: 1203 event.event = RDMA_CM_EVENT_ESTABLISHED; 1204 break; 1205 default: 1206 BUG_ON(1); 1207 } 1208 1209 event.status = iw_event->status; 1210 event.param.conn.private_data = iw_event->private_data; 1211 event.param.conn.private_data_len = iw_event->private_data_len; 1212 ret = id_priv->id.event_handler(&id_priv->id, &event); 1213 if (ret) { 1214 /* Destroy the CM ID by returning a non-zero value. */ 1215 id_priv->cm_id.iw = NULL; 1216 cma_exch(id_priv, CMA_DESTROYING); 1217 cma_release_remove(id_priv); 1218 rdma_destroy_id(&id_priv->id); 1219 return ret; 1220 } 1221 1222 cma_release_remove(id_priv); 1223 return ret; 1224 } 1225 1226 static int iw_conn_req_handler(struct iw_cm_id *cm_id, 1227 struct iw_cm_event *iw_event) 1228 { 1229 struct rdma_cm_id *new_cm_id; 1230 struct rdma_id_private *listen_id, *conn_id; 1231 struct sockaddr_in *sin; 1232 struct net_device *dev = NULL; 1233 struct rdma_cm_event event; 1234 int ret; 1235 1236 listen_id = cm_id->context; 1237 atomic_inc(&listen_id->dev_remove); 1238 if (!cma_comp(listen_id, CMA_LISTEN)) { 1239 ret = -ECONNABORTED; 1240 goto out; 1241 } 1242 1243 /* Create a new RDMA id for the new IW CM ID */ 1244 new_cm_id = rdma_create_id(listen_id->id.event_handler, 1245 listen_id->id.context, 1246 RDMA_PS_TCP); 1247 if (!new_cm_id) { 1248 ret = -ENOMEM; 1249 goto out; 1250 } 1251 conn_id = container_of(new_cm_id, struct rdma_id_private, id); 1252 atomic_inc(&conn_id->dev_remove); 1253 conn_id->state = CMA_CONNECT; 1254 1255 dev = ip_dev_find(iw_event->local_addr.sin_addr.s_addr); 1256 if (!dev) { 1257 ret = -EADDRNOTAVAIL; 1258 cma_release_remove(conn_id); 1259 rdma_destroy_id(new_cm_id); 1260 goto out; 1261 } 1262 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL); 1263 if (ret) { 1264 cma_release_remove(conn_id); 1265 rdma_destroy_id(new_cm_id); 1266 goto out; 1267 } 1268 1269 mutex_lock(&lock); 1270 ret = cma_acquire_dev(conn_id); 1271 mutex_unlock(&lock); 1272 if (ret) { 1273 cma_release_remove(conn_id); 1274 rdma_destroy_id(new_cm_id); 1275 goto out; 1276 } 1277 1278 conn_id->cm_id.iw = cm_id; 1279 cm_id->context = conn_id; 1280 cm_id->cm_handler = cma_iw_handler; 1281 1282 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr; 1283 *sin = iw_event->local_addr; 1284 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr; 1285 *sin = iw_event->remote_addr; 1286 1287 memset(&event, 0, sizeof event); 1288 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1289 event.param.conn.private_data = iw_event->private_data; 1290 event.param.conn.private_data_len = iw_event->private_data_len; 1291 ret = conn_id->id.event_handler(&conn_id->id, &event); 1292 if (ret) { 1293 /* User wants to destroy the CM ID */ 1294 conn_id->cm_id.iw = NULL; 1295 cma_exch(conn_id, CMA_DESTROYING); 1296 cma_release_remove(conn_id); 1297 rdma_destroy_id(&conn_id->id); 1298 } 1299 1300 out: 1301 if (dev) 1302 dev_put(dev); 1303 cma_release_remove(listen_id); 1304 return ret; 1305 } 1306 1307 static int cma_ib_listen(struct rdma_id_private *id_priv) 1308 { 1309 struct ib_cm_compare_data compare_data; 1310 struct sockaddr *addr; 1311 __be64 svc_id; 1312 int ret; 1313 1314 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler, 1315 id_priv); 1316 if (IS_ERR(id_priv->cm_id.ib)) 1317 return PTR_ERR(id_priv->cm_id.ib); 1318 1319 addr = &id_priv->id.route.addr.src_addr; 1320 svc_id = cma_get_service_id(id_priv->id.ps, addr); 1321 if (cma_any_addr(addr)) 1322 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL); 1323 else { 1324 cma_set_compare_data(id_priv->id.ps, addr, &compare_data); 1325 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data); 1326 } 1327 1328 if (ret) { 1329 ib_destroy_cm_id(id_priv->cm_id.ib); 1330 id_priv->cm_id.ib = NULL; 1331 } 1332 1333 return ret; 1334 } 1335 1336 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 1337 { 1338 int ret; 1339 struct sockaddr_in *sin; 1340 1341 id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, 1342 iw_conn_req_handler, 1343 id_priv); 1344 if (IS_ERR(id_priv->cm_id.iw)) 1345 return PTR_ERR(id_priv->cm_id.iw); 1346 1347 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1348 id_priv->cm_id.iw->local_addr = *sin; 1349 1350 ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 1351 1352 if (ret) { 1353 iw_destroy_cm_id(id_priv->cm_id.iw); 1354 id_priv->cm_id.iw = NULL; 1355 } 1356 1357 return ret; 1358 } 1359 1360 static int cma_listen_handler(struct rdma_cm_id *id, 1361 struct rdma_cm_event *event) 1362 { 1363 struct rdma_id_private *id_priv = id->context; 1364 1365 id->context = id_priv->id.context; 1366 id->event_handler = id_priv->id.event_handler; 1367 return id_priv->id.event_handler(id, event); 1368 } 1369 1370 static void cma_listen_on_dev(struct rdma_id_private *id_priv, 1371 struct cma_device *cma_dev) 1372 { 1373 struct rdma_id_private *dev_id_priv; 1374 struct rdma_cm_id *id; 1375 int ret; 1376 1377 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps); 1378 if (IS_ERR(id)) 1379 return; 1380 1381 dev_id_priv = container_of(id, struct rdma_id_private, id); 1382 1383 dev_id_priv->state = CMA_ADDR_BOUND; 1384 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr, 1385 ip_addr_size(&id_priv->id.route.addr.src_addr)); 1386 1387 cma_attach_to_dev(dev_id_priv, cma_dev); 1388 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); 1389 1390 ret = rdma_listen(id, id_priv->backlog); 1391 if (ret) 1392 goto err; 1393 1394 return; 1395 err: 1396 cma_destroy_listen(dev_id_priv); 1397 } 1398 1399 static void cma_listen_on_all(struct rdma_id_private *id_priv) 1400 { 1401 struct cma_device *cma_dev; 1402 1403 mutex_lock(&lock); 1404 list_add_tail(&id_priv->list, &listen_any_list); 1405 list_for_each_entry(cma_dev, &dev_list, list) 1406 cma_listen_on_dev(id_priv, cma_dev); 1407 mutex_unlock(&lock); 1408 } 1409 1410 static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af) 1411 { 1412 struct sockaddr_in addr_in; 1413 1414 memset(&addr_in, 0, sizeof addr_in); 1415 addr_in.sin_family = af; 1416 return rdma_bind_addr(id, (struct sockaddr *) &addr_in); 1417 } 1418 1419 int rdma_listen(struct rdma_cm_id *id, int backlog) 1420 { 1421 struct rdma_id_private *id_priv; 1422 int ret; 1423 1424 id_priv = container_of(id, struct rdma_id_private, id); 1425 if (id_priv->state == CMA_IDLE) { 1426 ret = cma_bind_any(id, AF_INET); 1427 if (ret) 1428 return ret; 1429 } 1430 1431 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN)) 1432 return -EINVAL; 1433 1434 id_priv->backlog = backlog; 1435 if (id->device) { 1436 switch (rdma_node_get_transport(id->device->node_type)) { 1437 case RDMA_TRANSPORT_IB: 1438 ret = cma_ib_listen(id_priv); 1439 if (ret) 1440 goto err; 1441 break; 1442 case RDMA_TRANSPORT_IWARP: 1443 ret = cma_iw_listen(id_priv, backlog); 1444 if (ret) 1445 goto err; 1446 break; 1447 default: 1448 ret = -ENOSYS; 1449 goto err; 1450 } 1451 } else 1452 cma_listen_on_all(id_priv); 1453 1454 return 0; 1455 err: 1456 id_priv->backlog = 0; 1457 cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND); 1458 return ret; 1459 } 1460 EXPORT_SYMBOL(rdma_listen); 1461 1462 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec, 1463 void *context) 1464 { 1465 struct cma_work *work = context; 1466 struct rdma_route *route; 1467 1468 route = &work->id->id.route; 1469 1470 if (!status) { 1471 route->num_paths = 1; 1472 *route->path_rec = *path_rec; 1473 } else { 1474 work->old_state = CMA_ROUTE_QUERY; 1475 work->new_state = CMA_ADDR_RESOLVED; 1476 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; 1477 work->event.status = status; 1478 } 1479 1480 queue_work(cma_wq, &work->work); 1481 } 1482 1483 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, 1484 struct cma_work *work) 1485 { 1486 struct rdma_dev_addr *addr = &id_priv->id.route.addr.dev_addr; 1487 struct ib_sa_path_rec path_rec; 1488 1489 memset(&path_rec, 0, sizeof path_rec); 1490 ib_addr_get_sgid(addr, &path_rec.sgid); 1491 ib_addr_get_dgid(addr, &path_rec.dgid); 1492 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr)); 1493 path_rec.numb_path = 1; 1494 path_rec.reversible = 1; 1495 1496 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, 1497 id_priv->id.port_num, &path_rec, 1498 IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | 1499 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | 1500 IB_SA_PATH_REC_REVERSIBLE, 1501 timeout_ms, GFP_KERNEL, 1502 cma_query_handler, work, &id_priv->query); 1503 1504 return (id_priv->query_id < 0) ? id_priv->query_id : 0; 1505 } 1506 1507 static void cma_work_handler(struct work_struct *_work) 1508 { 1509 struct cma_work *work = container_of(_work, struct cma_work, work); 1510 struct rdma_id_private *id_priv = work->id; 1511 int destroy = 0; 1512 1513 atomic_inc(&id_priv->dev_remove); 1514 if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) 1515 goto out; 1516 1517 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 1518 cma_exch(id_priv, CMA_DESTROYING); 1519 destroy = 1; 1520 } 1521 out: 1522 cma_release_remove(id_priv); 1523 cma_deref_id(id_priv); 1524 if (destroy) 1525 rdma_destroy_id(&id_priv->id); 1526 kfree(work); 1527 } 1528 1529 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) 1530 { 1531 struct rdma_route *route = &id_priv->id.route; 1532 struct cma_work *work; 1533 int ret; 1534 1535 work = kzalloc(sizeof *work, GFP_KERNEL); 1536 if (!work) 1537 return -ENOMEM; 1538 1539 work->id = id_priv; 1540 INIT_WORK(&work->work, cma_work_handler); 1541 work->old_state = CMA_ROUTE_QUERY; 1542 work->new_state = CMA_ROUTE_RESOLVED; 1543 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1544 1545 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); 1546 if (!route->path_rec) { 1547 ret = -ENOMEM; 1548 goto err1; 1549 } 1550 1551 ret = cma_query_ib_route(id_priv, timeout_ms, work); 1552 if (ret) 1553 goto err2; 1554 1555 return 0; 1556 err2: 1557 kfree(route->path_rec); 1558 route->path_rec = NULL; 1559 err1: 1560 kfree(work); 1561 return ret; 1562 } 1563 1564 int rdma_set_ib_paths(struct rdma_cm_id *id, 1565 struct ib_sa_path_rec *path_rec, int num_paths) 1566 { 1567 struct rdma_id_private *id_priv; 1568 int ret; 1569 1570 id_priv = container_of(id, struct rdma_id_private, id); 1571 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED)) 1572 return -EINVAL; 1573 1574 id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL); 1575 if (!id->route.path_rec) { 1576 ret = -ENOMEM; 1577 goto err; 1578 } 1579 1580 memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths); 1581 return 0; 1582 err: 1583 cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED); 1584 return ret; 1585 } 1586 EXPORT_SYMBOL(rdma_set_ib_paths); 1587 1588 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) 1589 { 1590 struct cma_work *work; 1591 1592 work = kzalloc(sizeof *work, GFP_KERNEL); 1593 if (!work) 1594 return -ENOMEM; 1595 1596 work->id = id_priv; 1597 INIT_WORK(&work->work, cma_work_handler); 1598 work->old_state = CMA_ROUTE_QUERY; 1599 work->new_state = CMA_ROUTE_RESOLVED; 1600 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1601 queue_work(cma_wq, &work->work); 1602 return 0; 1603 } 1604 1605 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) 1606 { 1607 struct rdma_id_private *id_priv; 1608 int ret; 1609 1610 id_priv = container_of(id, struct rdma_id_private, id); 1611 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY)) 1612 return -EINVAL; 1613 1614 atomic_inc(&id_priv->refcount); 1615 switch (rdma_node_get_transport(id->device->node_type)) { 1616 case RDMA_TRANSPORT_IB: 1617 ret = cma_resolve_ib_route(id_priv, timeout_ms); 1618 break; 1619 case RDMA_TRANSPORT_IWARP: 1620 ret = cma_resolve_iw_route(id_priv, timeout_ms); 1621 break; 1622 default: 1623 ret = -ENOSYS; 1624 break; 1625 } 1626 if (ret) 1627 goto err; 1628 1629 return 0; 1630 err: 1631 cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED); 1632 cma_deref_id(id_priv); 1633 return ret; 1634 } 1635 EXPORT_SYMBOL(rdma_resolve_route); 1636 1637 static int cma_bind_loopback(struct rdma_id_private *id_priv) 1638 { 1639 struct cma_device *cma_dev; 1640 struct ib_port_attr port_attr; 1641 union ib_gid gid; 1642 u16 pkey; 1643 int ret; 1644 u8 p; 1645 1646 mutex_lock(&lock); 1647 if (list_empty(&dev_list)) { 1648 ret = -ENODEV; 1649 goto out; 1650 } 1651 list_for_each_entry(cma_dev, &dev_list, list) 1652 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p) 1653 if (!ib_query_port(cma_dev->device, p, &port_attr) && 1654 port_attr.state == IB_PORT_ACTIVE) 1655 goto port_found; 1656 1657 p = 1; 1658 cma_dev = list_entry(dev_list.next, struct cma_device, list); 1659 1660 port_found: 1661 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid); 1662 if (ret) 1663 goto out; 1664 1665 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); 1666 if (ret) 1667 goto out; 1668 1669 ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); 1670 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); 1671 id_priv->id.port_num = p; 1672 cma_attach_to_dev(id_priv, cma_dev); 1673 out: 1674 mutex_unlock(&lock); 1675 return ret; 1676 } 1677 1678 static void addr_handler(int status, struct sockaddr *src_addr, 1679 struct rdma_dev_addr *dev_addr, void *context) 1680 { 1681 struct rdma_id_private *id_priv = context; 1682 struct rdma_cm_event event; 1683 1684 memset(&event, 0, sizeof event); 1685 atomic_inc(&id_priv->dev_remove); 1686 1687 /* 1688 * Grab mutex to block rdma_destroy_id() from removing the device while 1689 * we're trying to acquire it. 1690 */ 1691 mutex_lock(&lock); 1692 if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) { 1693 mutex_unlock(&lock); 1694 goto out; 1695 } 1696 1697 if (!status && !id_priv->cma_dev) 1698 status = cma_acquire_dev(id_priv); 1699 mutex_unlock(&lock); 1700 1701 if (status) { 1702 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND)) 1703 goto out; 1704 event.event = RDMA_CM_EVENT_ADDR_ERROR; 1705 event.status = status; 1706 } else { 1707 memcpy(&id_priv->id.route.addr.src_addr, src_addr, 1708 ip_addr_size(src_addr)); 1709 event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 1710 } 1711 1712 if (id_priv->id.event_handler(&id_priv->id, &event)) { 1713 cma_exch(id_priv, CMA_DESTROYING); 1714 cma_release_remove(id_priv); 1715 cma_deref_id(id_priv); 1716 rdma_destroy_id(&id_priv->id); 1717 return; 1718 } 1719 out: 1720 cma_release_remove(id_priv); 1721 cma_deref_id(id_priv); 1722 } 1723 1724 static int cma_resolve_loopback(struct rdma_id_private *id_priv) 1725 { 1726 struct cma_work *work; 1727 struct sockaddr_in *src_in, *dst_in; 1728 union ib_gid gid; 1729 int ret; 1730 1731 work = kzalloc(sizeof *work, GFP_KERNEL); 1732 if (!work) 1733 return -ENOMEM; 1734 1735 if (!id_priv->cma_dev) { 1736 ret = cma_bind_loopback(id_priv); 1737 if (ret) 1738 goto err; 1739 } 1740 1741 ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); 1742 ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); 1743 1744 if (cma_zero_addr(&id_priv->id.route.addr.src_addr)) { 1745 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr; 1746 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr; 1747 src_in->sin_family = dst_in->sin_family; 1748 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr; 1749 } 1750 1751 work->id = id_priv; 1752 INIT_WORK(&work->work, cma_work_handler); 1753 work->old_state = CMA_ADDR_QUERY; 1754 work->new_state = CMA_ADDR_RESOLVED; 1755 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 1756 queue_work(cma_wq, &work->work); 1757 return 0; 1758 err: 1759 kfree(work); 1760 return ret; 1761 } 1762 1763 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 1764 struct sockaddr *dst_addr) 1765 { 1766 if (src_addr && src_addr->sa_family) 1767 return rdma_bind_addr(id, src_addr); 1768 else 1769 return cma_bind_any(id, dst_addr->sa_family); 1770 } 1771 1772 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 1773 struct sockaddr *dst_addr, int timeout_ms) 1774 { 1775 struct rdma_id_private *id_priv; 1776 int ret; 1777 1778 id_priv = container_of(id, struct rdma_id_private, id); 1779 if (id_priv->state == CMA_IDLE) { 1780 ret = cma_bind_addr(id, src_addr, dst_addr); 1781 if (ret) 1782 return ret; 1783 } 1784 1785 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY)) 1786 return -EINVAL; 1787 1788 atomic_inc(&id_priv->refcount); 1789 memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr)); 1790 if (cma_any_addr(dst_addr)) 1791 ret = cma_resolve_loopback(id_priv); 1792 else 1793 ret = rdma_resolve_ip(&addr_client, &id->route.addr.src_addr, 1794 dst_addr, &id->route.addr.dev_addr, 1795 timeout_ms, addr_handler, id_priv); 1796 if (ret) 1797 goto err; 1798 1799 return 0; 1800 err: 1801 cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND); 1802 cma_deref_id(id_priv); 1803 return ret; 1804 } 1805 EXPORT_SYMBOL(rdma_resolve_addr); 1806 1807 static void cma_bind_port(struct rdma_bind_list *bind_list, 1808 struct rdma_id_private *id_priv) 1809 { 1810 struct sockaddr_in *sin; 1811 1812 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1813 sin->sin_port = htons(bind_list->port); 1814 id_priv->bind_list = bind_list; 1815 hlist_add_head(&id_priv->node, &bind_list->owners); 1816 } 1817 1818 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv, 1819 unsigned short snum) 1820 { 1821 struct rdma_bind_list *bind_list; 1822 int port, ret; 1823 1824 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 1825 if (!bind_list) 1826 return -ENOMEM; 1827 1828 do { 1829 ret = idr_get_new_above(ps, bind_list, snum, &port); 1830 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); 1831 1832 if (ret) 1833 goto err1; 1834 1835 if (port != snum) { 1836 ret = -EADDRNOTAVAIL; 1837 goto err2; 1838 } 1839 1840 bind_list->ps = ps; 1841 bind_list->port = (unsigned short) port; 1842 cma_bind_port(bind_list, id_priv); 1843 return 0; 1844 err2: 1845 idr_remove(ps, port); 1846 err1: 1847 kfree(bind_list); 1848 return ret; 1849 } 1850 1851 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv) 1852 { 1853 struct rdma_bind_list *bind_list; 1854 int port, ret; 1855 1856 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 1857 if (!bind_list) 1858 return -ENOMEM; 1859 1860 retry: 1861 do { 1862 ret = idr_get_new_above(ps, bind_list, next_port, &port); 1863 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); 1864 1865 if (ret) 1866 goto err1; 1867 1868 if (port > sysctl_local_port_range[1]) { 1869 if (next_port != sysctl_local_port_range[0]) { 1870 idr_remove(ps, port); 1871 next_port = sysctl_local_port_range[0]; 1872 goto retry; 1873 } 1874 ret = -EADDRNOTAVAIL; 1875 goto err2; 1876 } 1877 1878 if (port == sysctl_local_port_range[1]) 1879 next_port = sysctl_local_port_range[0]; 1880 else 1881 next_port = port + 1; 1882 1883 bind_list->ps = ps; 1884 bind_list->port = (unsigned short) port; 1885 cma_bind_port(bind_list, id_priv); 1886 return 0; 1887 err2: 1888 idr_remove(ps, port); 1889 err1: 1890 kfree(bind_list); 1891 return ret; 1892 } 1893 1894 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv) 1895 { 1896 struct rdma_id_private *cur_id; 1897 struct sockaddr_in *sin, *cur_sin; 1898 struct rdma_bind_list *bind_list; 1899 struct hlist_node *node; 1900 unsigned short snum; 1901 1902 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1903 snum = ntohs(sin->sin_port); 1904 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) 1905 return -EACCES; 1906 1907 bind_list = idr_find(ps, snum); 1908 if (!bind_list) 1909 return cma_alloc_port(ps, id_priv, snum); 1910 1911 /* 1912 * We don't support binding to any address if anyone is bound to 1913 * a specific address on the same port. 1914 */ 1915 if (cma_any_addr(&id_priv->id.route.addr.src_addr)) 1916 return -EADDRNOTAVAIL; 1917 1918 hlist_for_each_entry(cur_id, node, &bind_list->owners, node) { 1919 if (cma_any_addr(&cur_id->id.route.addr.src_addr)) 1920 return -EADDRNOTAVAIL; 1921 1922 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr; 1923 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr) 1924 return -EADDRINUSE; 1925 } 1926 1927 cma_bind_port(bind_list, id_priv); 1928 return 0; 1929 } 1930 1931 static int cma_get_port(struct rdma_id_private *id_priv) 1932 { 1933 struct idr *ps; 1934 int ret; 1935 1936 switch (id_priv->id.ps) { 1937 case RDMA_PS_SDP: 1938 ps = &sdp_ps; 1939 break; 1940 case RDMA_PS_TCP: 1941 ps = &tcp_ps; 1942 break; 1943 case RDMA_PS_UDP: 1944 ps = &udp_ps; 1945 break; 1946 case RDMA_PS_IPOIB: 1947 ps = &ipoib_ps; 1948 break; 1949 default: 1950 return -EPROTONOSUPPORT; 1951 } 1952 1953 mutex_lock(&lock); 1954 if (cma_any_port(&id_priv->id.route.addr.src_addr)) 1955 ret = cma_alloc_any_port(ps, id_priv); 1956 else 1957 ret = cma_use_port(ps, id_priv); 1958 mutex_unlock(&lock); 1959 1960 return ret; 1961 } 1962 1963 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 1964 { 1965 struct rdma_id_private *id_priv; 1966 int ret; 1967 1968 if (addr->sa_family != AF_INET) 1969 return -EAFNOSUPPORT; 1970 1971 id_priv = container_of(id, struct rdma_id_private, id); 1972 if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND)) 1973 return -EINVAL; 1974 1975 if (!cma_any_addr(addr)) { 1976 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr); 1977 if (ret) 1978 goto err1; 1979 1980 mutex_lock(&lock); 1981 ret = cma_acquire_dev(id_priv); 1982 mutex_unlock(&lock); 1983 if (ret) 1984 goto err1; 1985 } 1986 1987 memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr)); 1988 ret = cma_get_port(id_priv); 1989 if (ret) 1990 goto err2; 1991 1992 return 0; 1993 err2: 1994 if (!cma_any_addr(addr)) { 1995 mutex_lock(&lock); 1996 cma_detach_from_dev(id_priv); 1997 mutex_unlock(&lock); 1998 } 1999 err1: 2000 cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE); 2001 return ret; 2002 } 2003 EXPORT_SYMBOL(rdma_bind_addr); 2004 2005 static int cma_format_hdr(void *hdr, enum rdma_port_space ps, 2006 struct rdma_route *route) 2007 { 2008 struct sockaddr_in *src4, *dst4; 2009 struct cma_hdr *cma_hdr; 2010 struct sdp_hh *sdp_hdr; 2011 2012 src4 = (struct sockaddr_in *) &route->addr.src_addr; 2013 dst4 = (struct sockaddr_in *) &route->addr.dst_addr; 2014 2015 switch (ps) { 2016 case RDMA_PS_SDP: 2017 sdp_hdr = hdr; 2018 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) 2019 return -EINVAL; 2020 sdp_set_ip_ver(sdp_hdr, 4); 2021 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2022 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2023 sdp_hdr->port = src4->sin_port; 2024 break; 2025 default: 2026 cma_hdr = hdr; 2027 cma_hdr->cma_version = CMA_VERSION; 2028 cma_set_ip_ver(cma_hdr, 4); 2029 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2030 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2031 cma_hdr->port = src4->sin_port; 2032 break; 2033 } 2034 return 0; 2035 } 2036 2037 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, 2038 struct ib_cm_event *ib_event) 2039 { 2040 struct rdma_id_private *id_priv = cm_id->context; 2041 struct rdma_cm_event event; 2042 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; 2043 int ret = 0; 2044 2045 memset(&event, 0, sizeof event); 2046 atomic_inc(&id_priv->dev_remove); 2047 if (!cma_comp(id_priv, CMA_CONNECT)) 2048 goto out; 2049 2050 switch (ib_event->event) { 2051 case IB_CM_SIDR_REQ_ERROR: 2052 event.event = RDMA_CM_EVENT_UNREACHABLE; 2053 event.status = -ETIMEDOUT; 2054 break; 2055 case IB_CM_SIDR_REP_RECEIVED: 2056 event.param.ud.private_data = ib_event->private_data; 2057 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; 2058 if (rep->status != IB_SIDR_SUCCESS) { 2059 event.event = RDMA_CM_EVENT_UNREACHABLE; 2060 event.status = ib_event->param.sidr_rep_rcvd.status; 2061 break; 2062 } 2063 if (id_priv->qkey != rep->qkey) { 2064 event.event = RDMA_CM_EVENT_UNREACHABLE; 2065 event.status = -EINVAL; 2066 break; 2067 } 2068 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, 2069 id_priv->id.route.path_rec, 2070 &event.param.ud.ah_attr); 2071 event.param.ud.qp_num = rep->qpn; 2072 event.param.ud.qkey = rep->qkey; 2073 event.event = RDMA_CM_EVENT_ESTABLISHED; 2074 event.status = 0; 2075 break; 2076 default: 2077 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d", 2078 ib_event->event); 2079 goto out; 2080 } 2081 2082 ret = id_priv->id.event_handler(&id_priv->id, &event); 2083 if (ret) { 2084 /* Destroy the CM ID by returning a non-zero value. */ 2085 id_priv->cm_id.ib = NULL; 2086 cma_exch(id_priv, CMA_DESTROYING); 2087 cma_release_remove(id_priv); 2088 rdma_destroy_id(&id_priv->id); 2089 return ret; 2090 } 2091 out: 2092 cma_release_remove(id_priv); 2093 return ret; 2094 } 2095 2096 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, 2097 struct rdma_conn_param *conn_param) 2098 { 2099 struct ib_cm_sidr_req_param req; 2100 struct rdma_route *route; 2101 int ret; 2102 2103 req.private_data_len = sizeof(struct cma_hdr) + 2104 conn_param->private_data_len; 2105 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2106 if (!req.private_data) 2107 return -ENOMEM; 2108 2109 if (conn_param->private_data && conn_param->private_data_len) 2110 memcpy((void *) req.private_data + sizeof(struct cma_hdr), 2111 conn_param->private_data, conn_param->private_data_len); 2112 2113 route = &id_priv->id.route; 2114 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route); 2115 if (ret) 2116 goto out; 2117 2118 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, 2119 cma_sidr_rep_handler, id_priv); 2120 if (IS_ERR(id_priv->cm_id.ib)) { 2121 ret = PTR_ERR(id_priv->cm_id.ib); 2122 goto out; 2123 } 2124 2125 req.path = route->path_rec; 2126 req.service_id = cma_get_service_id(id_priv->id.ps, 2127 &route->addr.dst_addr); 2128 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); 2129 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2130 2131 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); 2132 if (ret) { 2133 ib_destroy_cm_id(id_priv->cm_id.ib); 2134 id_priv->cm_id.ib = NULL; 2135 } 2136 out: 2137 kfree(req.private_data); 2138 return ret; 2139 } 2140 2141 static int cma_connect_ib(struct rdma_id_private *id_priv, 2142 struct rdma_conn_param *conn_param) 2143 { 2144 struct ib_cm_req_param req; 2145 struct rdma_route *route; 2146 void *private_data; 2147 int offset, ret; 2148 2149 memset(&req, 0, sizeof req); 2150 offset = cma_user_data_offset(id_priv->id.ps); 2151 req.private_data_len = offset + conn_param->private_data_len; 2152 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2153 if (!private_data) 2154 return -ENOMEM; 2155 2156 if (conn_param->private_data && conn_param->private_data_len) 2157 memcpy(private_data + offset, conn_param->private_data, 2158 conn_param->private_data_len); 2159 2160 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler, 2161 id_priv); 2162 if (IS_ERR(id_priv->cm_id.ib)) { 2163 ret = PTR_ERR(id_priv->cm_id.ib); 2164 goto out; 2165 } 2166 2167 route = &id_priv->id.route; 2168 ret = cma_format_hdr(private_data, id_priv->id.ps, route); 2169 if (ret) 2170 goto out; 2171 req.private_data = private_data; 2172 2173 req.primary_path = &route->path_rec[0]; 2174 if (route->num_paths == 2) 2175 req.alternate_path = &route->path_rec[1]; 2176 2177 req.service_id = cma_get_service_id(id_priv->id.ps, 2178 &route->addr.dst_addr); 2179 req.qp_num = id_priv->qp_num; 2180 req.qp_type = IB_QPT_RC; 2181 req.starting_psn = id_priv->seq_num; 2182 req.responder_resources = conn_param->responder_resources; 2183 req.initiator_depth = conn_param->initiator_depth; 2184 req.flow_control = conn_param->flow_control; 2185 req.retry_count = conn_param->retry_count; 2186 req.rnr_retry_count = conn_param->rnr_retry_count; 2187 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 2188 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 2189 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2190 req.srq = id_priv->srq ? 1 : 0; 2191 2192 ret = ib_send_cm_req(id_priv->cm_id.ib, &req); 2193 out: 2194 if (ret && !IS_ERR(id_priv->cm_id.ib)) { 2195 ib_destroy_cm_id(id_priv->cm_id.ib); 2196 id_priv->cm_id.ib = NULL; 2197 } 2198 2199 kfree(private_data); 2200 return ret; 2201 } 2202 2203 static int cma_connect_iw(struct rdma_id_private *id_priv, 2204 struct rdma_conn_param *conn_param) 2205 { 2206 struct iw_cm_id *cm_id; 2207 struct sockaddr_in* sin; 2208 int ret; 2209 struct iw_cm_conn_param iw_param; 2210 2211 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); 2212 if (IS_ERR(cm_id)) { 2213 ret = PTR_ERR(cm_id); 2214 goto out; 2215 } 2216 2217 id_priv->cm_id.iw = cm_id; 2218 2219 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr; 2220 cm_id->local_addr = *sin; 2221 2222 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr; 2223 cm_id->remote_addr = *sin; 2224 2225 ret = cma_modify_qp_rtr(&id_priv->id); 2226 if (ret) 2227 goto out; 2228 2229 iw_param.ord = conn_param->initiator_depth; 2230 iw_param.ird = conn_param->responder_resources; 2231 iw_param.private_data = conn_param->private_data; 2232 iw_param.private_data_len = conn_param->private_data_len; 2233 if (id_priv->id.qp) 2234 iw_param.qpn = id_priv->qp_num; 2235 else 2236 iw_param.qpn = conn_param->qp_num; 2237 ret = iw_cm_connect(cm_id, &iw_param); 2238 out: 2239 if (ret && !IS_ERR(cm_id)) { 2240 iw_destroy_cm_id(cm_id); 2241 id_priv->cm_id.iw = NULL; 2242 } 2243 return ret; 2244 } 2245 2246 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2247 { 2248 struct rdma_id_private *id_priv; 2249 int ret; 2250 2251 id_priv = container_of(id, struct rdma_id_private, id); 2252 if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT)) 2253 return -EINVAL; 2254 2255 if (!id->qp) { 2256 id_priv->qp_num = conn_param->qp_num; 2257 id_priv->srq = conn_param->srq; 2258 } 2259 2260 switch (rdma_node_get_transport(id->device->node_type)) { 2261 case RDMA_TRANSPORT_IB: 2262 if (cma_is_ud_ps(id->ps)) 2263 ret = cma_resolve_ib_udp(id_priv, conn_param); 2264 else 2265 ret = cma_connect_ib(id_priv, conn_param); 2266 break; 2267 case RDMA_TRANSPORT_IWARP: 2268 ret = cma_connect_iw(id_priv, conn_param); 2269 break; 2270 default: 2271 ret = -ENOSYS; 2272 break; 2273 } 2274 if (ret) 2275 goto err; 2276 2277 return 0; 2278 err: 2279 cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED); 2280 return ret; 2281 } 2282 EXPORT_SYMBOL(rdma_connect); 2283 2284 static int cma_accept_ib(struct rdma_id_private *id_priv, 2285 struct rdma_conn_param *conn_param) 2286 { 2287 struct ib_cm_rep_param rep; 2288 struct ib_qp_attr qp_attr; 2289 int qp_attr_mask, ret; 2290 2291 if (id_priv->id.qp) { 2292 ret = cma_modify_qp_rtr(&id_priv->id); 2293 if (ret) 2294 goto out; 2295 2296 qp_attr.qp_state = IB_QPS_RTS; 2297 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, &qp_attr, 2298 &qp_attr_mask); 2299 if (ret) 2300 goto out; 2301 2302 qp_attr.max_rd_atomic = conn_param->initiator_depth; 2303 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 2304 if (ret) 2305 goto out; 2306 } 2307 2308 memset(&rep, 0, sizeof rep); 2309 rep.qp_num = id_priv->qp_num; 2310 rep.starting_psn = id_priv->seq_num; 2311 rep.private_data = conn_param->private_data; 2312 rep.private_data_len = conn_param->private_data_len; 2313 rep.responder_resources = conn_param->responder_resources; 2314 rep.initiator_depth = conn_param->initiator_depth; 2315 rep.target_ack_delay = CMA_CM_RESPONSE_TIMEOUT; 2316 rep.failover_accepted = 0; 2317 rep.flow_control = conn_param->flow_control; 2318 rep.rnr_retry_count = conn_param->rnr_retry_count; 2319 rep.srq = id_priv->srq ? 1 : 0; 2320 2321 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); 2322 out: 2323 return ret; 2324 } 2325 2326 static int cma_accept_iw(struct rdma_id_private *id_priv, 2327 struct rdma_conn_param *conn_param) 2328 { 2329 struct iw_cm_conn_param iw_param; 2330 int ret; 2331 2332 ret = cma_modify_qp_rtr(&id_priv->id); 2333 if (ret) 2334 return ret; 2335 2336 iw_param.ord = conn_param->initiator_depth; 2337 iw_param.ird = conn_param->responder_resources; 2338 iw_param.private_data = conn_param->private_data; 2339 iw_param.private_data_len = conn_param->private_data_len; 2340 if (id_priv->id.qp) { 2341 iw_param.qpn = id_priv->qp_num; 2342 } else 2343 iw_param.qpn = conn_param->qp_num; 2344 2345 return iw_cm_accept(id_priv->cm_id.iw, &iw_param); 2346 } 2347 2348 static int cma_send_sidr_rep(struct rdma_id_private *id_priv, 2349 enum ib_cm_sidr_status status, 2350 const void *private_data, int private_data_len) 2351 { 2352 struct ib_cm_sidr_rep_param rep; 2353 2354 memset(&rep, 0, sizeof rep); 2355 rep.status = status; 2356 if (status == IB_SIDR_SUCCESS) { 2357 rep.qp_num = id_priv->qp_num; 2358 rep.qkey = id_priv->qkey; 2359 } 2360 rep.private_data = private_data; 2361 rep.private_data_len = private_data_len; 2362 2363 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); 2364 } 2365 2366 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2367 { 2368 struct rdma_id_private *id_priv; 2369 int ret; 2370 2371 id_priv = container_of(id, struct rdma_id_private, id); 2372 if (!cma_comp(id_priv, CMA_CONNECT)) 2373 return -EINVAL; 2374 2375 if (!id->qp && conn_param) { 2376 id_priv->qp_num = conn_param->qp_num; 2377 id_priv->srq = conn_param->srq; 2378 } 2379 2380 switch (rdma_node_get_transport(id->device->node_type)) { 2381 case RDMA_TRANSPORT_IB: 2382 if (cma_is_ud_ps(id->ps)) 2383 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 2384 conn_param->private_data, 2385 conn_param->private_data_len); 2386 else if (conn_param) 2387 ret = cma_accept_ib(id_priv, conn_param); 2388 else 2389 ret = cma_rep_recv(id_priv); 2390 break; 2391 case RDMA_TRANSPORT_IWARP: 2392 ret = cma_accept_iw(id_priv, conn_param); 2393 break; 2394 default: 2395 ret = -ENOSYS; 2396 break; 2397 } 2398 2399 if (ret) 2400 goto reject; 2401 2402 return 0; 2403 reject: 2404 cma_modify_qp_err(id); 2405 rdma_reject(id, NULL, 0); 2406 return ret; 2407 } 2408 EXPORT_SYMBOL(rdma_accept); 2409 2410 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) 2411 { 2412 struct rdma_id_private *id_priv; 2413 int ret; 2414 2415 id_priv = container_of(id, struct rdma_id_private, id); 2416 if (!cma_comp(id_priv, CMA_CONNECT)) 2417 return -EINVAL; 2418 2419 switch (id->device->node_type) { 2420 case RDMA_NODE_IB_CA: 2421 ret = ib_cm_notify(id_priv->cm_id.ib, event); 2422 break; 2423 default: 2424 ret = 0; 2425 break; 2426 } 2427 return ret; 2428 } 2429 EXPORT_SYMBOL(rdma_notify); 2430 2431 int rdma_reject(struct rdma_cm_id *id, const void *private_data, 2432 u8 private_data_len) 2433 { 2434 struct rdma_id_private *id_priv; 2435 int ret; 2436 2437 id_priv = container_of(id, struct rdma_id_private, id); 2438 if (!cma_comp(id_priv, CMA_CONNECT)) 2439 return -EINVAL; 2440 2441 switch (rdma_node_get_transport(id->device->node_type)) { 2442 case RDMA_TRANSPORT_IB: 2443 if (cma_is_ud_ps(id->ps)) 2444 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 2445 private_data, private_data_len); 2446 else 2447 ret = ib_send_cm_rej(id_priv->cm_id.ib, 2448 IB_CM_REJ_CONSUMER_DEFINED, NULL, 2449 0, private_data, private_data_len); 2450 break; 2451 case RDMA_TRANSPORT_IWARP: 2452 ret = iw_cm_reject(id_priv->cm_id.iw, 2453 private_data, private_data_len); 2454 break; 2455 default: 2456 ret = -ENOSYS; 2457 break; 2458 } 2459 return ret; 2460 } 2461 EXPORT_SYMBOL(rdma_reject); 2462 2463 int rdma_disconnect(struct rdma_cm_id *id) 2464 { 2465 struct rdma_id_private *id_priv; 2466 int ret; 2467 2468 id_priv = container_of(id, struct rdma_id_private, id); 2469 if (!cma_comp(id_priv, CMA_CONNECT) && 2470 !cma_comp(id_priv, CMA_DISCONNECT)) 2471 return -EINVAL; 2472 2473 switch (rdma_node_get_transport(id->device->node_type)) { 2474 case RDMA_TRANSPORT_IB: 2475 ret = cma_modify_qp_err(id); 2476 if (ret) 2477 goto out; 2478 /* Initiate or respond to a disconnect. */ 2479 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) 2480 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); 2481 break; 2482 case RDMA_TRANSPORT_IWARP: 2483 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); 2484 break; 2485 default: 2486 ret = -EINVAL; 2487 break; 2488 } 2489 out: 2490 return ret; 2491 } 2492 EXPORT_SYMBOL(rdma_disconnect); 2493 2494 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) 2495 { 2496 struct rdma_id_private *id_priv; 2497 struct cma_multicast *mc = multicast->context; 2498 struct rdma_cm_event event; 2499 int ret; 2500 2501 id_priv = mc->id_priv; 2502 atomic_inc(&id_priv->dev_remove); 2503 if (!cma_comp(id_priv, CMA_ADDR_BOUND) && 2504 !cma_comp(id_priv, CMA_ADDR_RESOLVED)) 2505 goto out; 2506 2507 if (!status && id_priv->id.qp) 2508 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, 2509 multicast->rec.mlid); 2510 2511 memset(&event, 0, sizeof event); 2512 event.status = status; 2513 event.param.ud.private_data = mc->context; 2514 if (!status) { 2515 event.event = RDMA_CM_EVENT_MULTICAST_JOIN; 2516 ib_init_ah_from_mcmember(id_priv->id.device, 2517 id_priv->id.port_num, &multicast->rec, 2518 &event.param.ud.ah_attr); 2519 event.param.ud.qp_num = 0xFFFFFF; 2520 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); 2521 } else 2522 event.event = RDMA_CM_EVENT_MULTICAST_ERROR; 2523 2524 ret = id_priv->id.event_handler(&id_priv->id, &event); 2525 if (ret) { 2526 cma_exch(id_priv, CMA_DESTROYING); 2527 cma_release_remove(id_priv); 2528 rdma_destroy_id(&id_priv->id); 2529 return 0; 2530 } 2531 out: 2532 cma_release_remove(id_priv); 2533 return 0; 2534 } 2535 2536 static void cma_set_mgid(struct rdma_id_private *id_priv, 2537 struct sockaddr *addr, union ib_gid *mgid) 2538 { 2539 unsigned char mc_map[MAX_ADDR_LEN]; 2540 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2541 struct sockaddr_in *sin = (struct sockaddr_in *) addr; 2542 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; 2543 2544 if (cma_any_addr(addr)) { 2545 memset(mgid, 0, sizeof *mgid); 2546 } else if ((addr->sa_family == AF_INET6) && 2547 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) == 2548 0xFF10A01B)) { 2549 /* IPv6 address is an SA assigned MGID. */ 2550 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 2551 } else { 2552 ip_ib_mc_map(sin->sin_addr.s_addr, mc_map); 2553 if (id_priv->id.ps == RDMA_PS_UDP) 2554 mc_map[7] = 0x01; /* Use RDMA CM signature */ 2555 mc_map[8] = ib_addr_get_pkey(dev_addr) >> 8; 2556 mc_map[9] = (unsigned char) ib_addr_get_pkey(dev_addr); 2557 *mgid = *(union ib_gid *) (mc_map + 4); 2558 } 2559 } 2560 2561 static int cma_join_ib_multicast(struct rdma_id_private *id_priv, 2562 struct cma_multicast *mc) 2563 { 2564 struct ib_sa_mcmember_rec rec; 2565 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2566 ib_sa_comp_mask comp_mask; 2567 int ret; 2568 2569 ib_addr_get_mgid(dev_addr, &rec.mgid); 2570 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, 2571 &rec.mgid, &rec); 2572 if (ret) 2573 return ret; 2574 2575 cma_set_mgid(id_priv, &mc->addr, &rec.mgid); 2576 if (id_priv->id.ps == RDMA_PS_UDP) 2577 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 2578 ib_addr_get_sgid(dev_addr, &rec.port_gid); 2579 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 2580 rec.join_state = 1; 2581 2582 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 2583 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | 2584 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | 2585 IB_SA_MCMEMBER_REC_FLOW_LABEL | 2586 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; 2587 2588 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, 2589 id_priv->id.port_num, &rec, 2590 comp_mask, GFP_KERNEL, 2591 cma_ib_mc_handler, mc); 2592 if (IS_ERR(mc->multicast.ib)) 2593 return PTR_ERR(mc->multicast.ib); 2594 2595 return 0; 2596 } 2597 2598 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, 2599 void *context) 2600 { 2601 struct rdma_id_private *id_priv; 2602 struct cma_multicast *mc; 2603 int ret; 2604 2605 id_priv = container_of(id, struct rdma_id_private, id); 2606 if (!cma_comp(id_priv, CMA_ADDR_BOUND) && 2607 !cma_comp(id_priv, CMA_ADDR_RESOLVED)) 2608 return -EINVAL; 2609 2610 mc = kmalloc(sizeof *mc, GFP_KERNEL); 2611 if (!mc) 2612 return -ENOMEM; 2613 2614 memcpy(&mc->addr, addr, ip_addr_size(addr)); 2615 mc->context = context; 2616 mc->id_priv = id_priv; 2617 2618 spin_lock(&id_priv->lock); 2619 list_add(&mc->list, &id_priv->mc_list); 2620 spin_unlock(&id_priv->lock); 2621 2622 switch (rdma_node_get_transport(id->device->node_type)) { 2623 case RDMA_TRANSPORT_IB: 2624 ret = cma_join_ib_multicast(id_priv, mc); 2625 break; 2626 default: 2627 ret = -ENOSYS; 2628 break; 2629 } 2630 2631 if (ret) { 2632 spin_lock_irq(&id_priv->lock); 2633 list_del(&mc->list); 2634 spin_unlock_irq(&id_priv->lock); 2635 kfree(mc); 2636 } 2637 return ret; 2638 } 2639 EXPORT_SYMBOL(rdma_join_multicast); 2640 2641 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) 2642 { 2643 struct rdma_id_private *id_priv; 2644 struct cma_multicast *mc; 2645 2646 id_priv = container_of(id, struct rdma_id_private, id); 2647 spin_lock_irq(&id_priv->lock); 2648 list_for_each_entry(mc, &id_priv->mc_list, list) { 2649 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) { 2650 list_del(&mc->list); 2651 spin_unlock_irq(&id_priv->lock); 2652 2653 if (id->qp) 2654 ib_detach_mcast(id->qp, 2655 &mc->multicast.ib->rec.mgid, 2656 mc->multicast.ib->rec.mlid); 2657 ib_sa_free_multicast(mc->multicast.ib); 2658 kfree(mc); 2659 return; 2660 } 2661 } 2662 spin_unlock_irq(&id_priv->lock); 2663 } 2664 EXPORT_SYMBOL(rdma_leave_multicast); 2665 2666 static void cma_add_one(struct ib_device *device) 2667 { 2668 struct cma_device *cma_dev; 2669 struct rdma_id_private *id_priv; 2670 2671 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); 2672 if (!cma_dev) 2673 return; 2674 2675 cma_dev->device = device; 2676 2677 init_completion(&cma_dev->comp); 2678 atomic_set(&cma_dev->refcount, 1); 2679 INIT_LIST_HEAD(&cma_dev->id_list); 2680 ib_set_client_data(device, &cma_client, cma_dev); 2681 2682 mutex_lock(&lock); 2683 list_add_tail(&cma_dev->list, &dev_list); 2684 list_for_each_entry(id_priv, &listen_any_list, list) 2685 cma_listen_on_dev(id_priv, cma_dev); 2686 mutex_unlock(&lock); 2687 } 2688 2689 static int cma_remove_id_dev(struct rdma_id_private *id_priv) 2690 { 2691 struct rdma_cm_event event; 2692 enum cma_state state; 2693 2694 /* Record that we want to remove the device */ 2695 state = cma_exch(id_priv, CMA_DEVICE_REMOVAL); 2696 if (state == CMA_DESTROYING) 2697 return 0; 2698 2699 cma_cancel_operation(id_priv, state); 2700 wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove)); 2701 2702 /* Check for destruction from another callback. */ 2703 if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL)) 2704 return 0; 2705 2706 memset(&event, 0, sizeof event); 2707 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; 2708 return id_priv->id.event_handler(&id_priv->id, &event); 2709 } 2710 2711 static void cma_process_remove(struct cma_device *cma_dev) 2712 { 2713 struct rdma_id_private *id_priv; 2714 int ret; 2715 2716 mutex_lock(&lock); 2717 while (!list_empty(&cma_dev->id_list)) { 2718 id_priv = list_entry(cma_dev->id_list.next, 2719 struct rdma_id_private, list); 2720 2721 if (cma_internal_listen(id_priv)) { 2722 cma_destroy_listen(id_priv); 2723 continue; 2724 } 2725 2726 list_del_init(&id_priv->list); 2727 atomic_inc(&id_priv->refcount); 2728 mutex_unlock(&lock); 2729 2730 ret = cma_remove_id_dev(id_priv); 2731 cma_deref_id(id_priv); 2732 if (ret) 2733 rdma_destroy_id(&id_priv->id); 2734 2735 mutex_lock(&lock); 2736 } 2737 mutex_unlock(&lock); 2738 2739 cma_deref_dev(cma_dev); 2740 wait_for_completion(&cma_dev->comp); 2741 } 2742 2743 static void cma_remove_one(struct ib_device *device) 2744 { 2745 struct cma_device *cma_dev; 2746 2747 cma_dev = ib_get_client_data(device, &cma_client); 2748 if (!cma_dev) 2749 return; 2750 2751 mutex_lock(&lock); 2752 list_del(&cma_dev->list); 2753 mutex_unlock(&lock); 2754 2755 cma_process_remove(cma_dev); 2756 kfree(cma_dev); 2757 } 2758 2759 static int cma_init(void) 2760 { 2761 int ret; 2762 2763 get_random_bytes(&next_port, sizeof next_port); 2764 next_port = (next_port % (sysctl_local_port_range[1] - 2765 sysctl_local_port_range[0])) + 2766 sysctl_local_port_range[0]; 2767 cma_wq = create_singlethread_workqueue("rdma_cm"); 2768 if (!cma_wq) 2769 return -ENOMEM; 2770 2771 ib_sa_register_client(&sa_client); 2772 rdma_addr_register_client(&addr_client); 2773 2774 ret = ib_register_client(&cma_client); 2775 if (ret) 2776 goto err; 2777 return 0; 2778 2779 err: 2780 rdma_addr_unregister_client(&addr_client); 2781 ib_sa_unregister_client(&sa_client); 2782 destroy_workqueue(cma_wq); 2783 return ret; 2784 } 2785 2786 static void cma_cleanup(void) 2787 { 2788 ib_unregister_client(&cma_client); 2789 rdma_addr_unregister_client(&addr_client); 2790 ib_sa_unregister_client(&sa_client); 2791 destroy_workqueue(cma_wq); 2792 idr_destroy(&sdp_ps); 2793 idr_destroy(&tcp_ps); 2794 idr_destroy(&udp_ps); 2795 idr_destroy(&ipoib_ps); 2796 } 2797 2798 module_init(cma_init); 2799 module_exit(cma_cleanup); 2800