1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2004-2007 Intel Corporation. All rights reserved. 4 * Copyright (c) 2004 Topspin Corporation. All rights reserved. 5 * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved. 6 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 7 * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. 8 */ 9 10 #include <linux/completion.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/device.h> 13 #include <linux/module.h> 14 #include <linux/err.h> 15 #include <linux/idr.h> 16 #include <linux/interrupt.h> 17 #include <linux/random.h> 18 #include <linux/rbtree.h> 19 #include <linux/spinlock.h> 20 #include <linux/slab.h> 21 #include <linux/sysfs.h> 22 #include <linux/workqueue.h> 23 #include <linux/kdev_t.h> 24 #include <linux/etherdevice.h> 25 26 #include <rdma/ib_cache.h> 27 #include <rdma/ib_cm.h> 28 #include "cm_msgs.h" 29 #include "core_priv.h" 30 31 MODULE_AUTHOR("Sean Hefty"); 32 MODULE_DESCRIPTION("InfiniBand CM"); 33 MODULE_LICENSE("Dual BSD/GPL"); 34 35 static const char * const ibcm_rej_reason_strs[] = { 36 [IB_CM_REJ_NO_QP] = "no QP", 37 [IB_CM_REJ_NO_EEC] = "no EEC", 38 [IB_CM_REJ_NO_RESOURCES] = "no resources", 39 [IB_CM_REJ_TIMEOUT] = "timeout", 40 [IB_CM_REJ_UNSUPPORTED] = "unsupported", 41 [IB_CM_REJ_INVALID_COMM_ID] = "invalid comm ID", 42 [IB_CM_REJ_INVALID_COMM_INSTANCE] = "invalid comm instance", 43 [IB_CM_REJ_INVALID_SERVICE_ID] = "invalid service ID", 44 [IB_CM_REJ_INVALID_TRANSPORT_TYPE] = "invalid transport type", 45 [IB_CM_REJ_STALE_CONN] = "stale conn", 46 [IB_CM_REJ_RDC_NOT_EXIST] = "RDC not exist", 47 [IB_CM_REJ_INVALID_GID] = "invalid GID", 48 [IB_CM_REJ_INVALID_LID] = "invalid LID", 49 [IB_CM_REJ_INVALID_SL] = "invalid SL", 50 [IB_CM_REJ_INVALID_TRAFFIC_CLASS] = "invalid traffic class", 51 [IB_CM_REJ_INVALID_HOP_LIMIT] = "invalid hop limit", 52 [IB_CM_REJ_INVALID_PACKET_RATE] = "invalid packet rate", 53 [IB_CM_REJ_INVALID_ALT_GID] = "invalid alt GID", 54 [IB_CM_REJ_INVALID_ALT_LID] = "invalid alt LID", 55 [IB_CM_REJ_INVALID_ALT_SL] = "invalid alt SL", 56 [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS] = "invalid alt traffic class", 57 [IB_CM_REJ_INVALID_ALT_HOP_LIMIT] = "invalid alt hop limit", 58 [IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate", 59 [IB_CM_REJ_PORT_CM_REDIRECT] = "port CM redirect", 60 [IB_CM_REJ_PORT_REDIRECT] = "port redirect", 61 [IB_CM_REJ_INVALID_MTU] = "invalid MTU", 62 [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources", 63 [IB_CM_REJ_CONSUMER_DEFINED] = "consumer defined", 64 [IB_CM_REJ_INVALID_RNR_RETRY] = "invalid RNR retry", 65 [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID", 66 [IB_CM_REJ_INVALID_CLASS_VERSION] = "invalid class version", 67 [IB_CM_REJ_INVALID_FLOW_LABEL] = "invalid flow label", 68 [IB_CM_REJ_INVALID_ALT_FLOW_LABEL] = "invalid alt flow label", 69 }; 70 71 const char *__attribute_const__ ibcm_reject_msg(int reason) 72 { 73 size_t index = reason; 74 75 if (index < ARRAY_SIZE(ibcm_rej_reason_strs) && 76 ibcm_rej_reason_strs[index]) 77 return ibcm_rej_reason_strs[index]; 78 else 79 return "unrecognized reason"; 80 } 81 EXPORT_SYMBOL(ibcm_reject_msg); 82 83 struct cm_id_private; 84 struct cm_work; 85 static int cm_add_one(struct ib_device *device); 86 static void cm_remove_one(struct ib_device *device, void *client_data); 87 static void cm_process_work(struct cm_id_private *cm_id_priv, 88 struct cm_work *work); 89 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, 90 struct ib_cm_sidr_rep_param *param); 91 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, 92 const void *private_data, u8 private_data_len); 93 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, 94 void *private_data, u8 private_data_len); 95 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv, 96 enum ib_cm_rej_reason reason, void *ari, 97 u8 ari_length, const void *private_data, 98 u8 private_data_len); 99 100 static struct ib_client cm_client = { 101 .name = "cm", 102 .add = cm_add_one, 103 .remove = cm_remove_one 104 }; 105 106 static struct ib_cm { 107 spinlock_t lock; 108 struct list_head device_list; 109 rwlock_t device_lock; 110 struct rb_root listen_service_table; 111 u64 listen_service_id; 112 /* struct rb_root peer_service_table; todo: fix peer to peer */ 113 struct rb_root remote_qp_table; 114 struct rb_root remote_id_table; 115 struct rb_root remote_sidr_table; 116 struct xarray local_id_table; 117 u32 local_id_next; 118 __be32 random_id_operand; 119 struct list_head timewait_list; 120 struct workqueue_struct *wq; 121 /* Sync on cm change port state */ 122 spinlock_t state_lock; 123 } cm; 124 125 /* Counter indexes ordered by attribute ID */ 126 enum { 127 CM_REQ_COUNTER, 128 CM_MRA_COUNTER, 129 CM_REJ_COUNTER, 130 CM_REP_COUNTER, 131 CM_RTU_COUNTER, 132 CM_DREQ_COUNTER, 133 CM_DREP_COUNTER, 134 CM_SIDR_REQ_COUNTER, 135 CM_SIDR_REP_COUNTER, 136 CM_LAP_COUNTER, 137 CM_APR_COUNTER, 138 CM_ATTR_COUNT, 139 CM_ATTR_ID_OFFSET = 0x0010, 140 }; 141 142 enum { 143 CM_XMIT, 144 CM_XMIT_RETRIES, 145 CM_RECV, 146 CM_RECV_DUPLICATES, 147 CM_COUNTER_GROUPS 148 }; 149 150 static char const counter_group_names[CM_COUNTER_GROUPS] 151 [sizeof("cm_rx_duplicates")] = { 152 "cm_tx_msgs", "cm_tx_retries", 153 "cm_rx_msgs", "cm_rx_duplicates" 154 }; 155 156 struct cm_counter_group { 157 struct kobject obj; 158 atomic_long_t counter[CM_ATTR_COUNT]; 159 }; 160 161 struct cm_counter_attribute { 162 struct attribute attr; 163 int index; 164 }; 165 166 #define CM_COUNTER_ATTR(_name, _index) \ 167 struct cm_counter_attribute cm_##_name##_counter_attr = { \ 168 .attr = { .name = __stringify(_name), .mode = 0444 }, \ 169 .index = _index \ 170 } 171 172 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER); 173 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER); 174 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER); 175 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER); 176 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER); 177 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER); 178 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER); 179 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER); 180 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER); 181 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER); 182 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER); 183 184 static struct attribute *cm_counter_default_attrs[] = { 185 &cm_req_counter_attr.attr, 186 &cm_mra_counter_attr.attr, 187 &cm_rej_counter_attr.attr, 188 &cm_rep_counter_attr.attr, 189 &cm_rtu_counter_attr.attr, 190 &cm_dreq_counter_attr.attr, 191 &cm_drep_counter_attr.attr, 192 &cm_sidr_req_counter_attr.attr, 193 &cm_sidr_rep_counter_attr.attr, 194 &cm_lap_counter_attr.attr, 195 &cm_apr_counter_attr.attr, 196 NULL 197 }; 198 199 struct cm_port { 200 struct cm_device *cm_dev; 201 struct ib_mad_agent *mad_agent; 202 struct kobject port_obj; 203 u8 port_num; 204 struct list_head cm_priv_prim_list; 205 struct list_head cm_priv_altr_list; 206 struct cm_counter_group counter_group[CM_COUNTER_GROUPS]; 207 }; 208 209 struct cm_device { 210 struct list_head list; 211 struct ib_device *ib_device; 212 u8 ack_delay; 213 int going_down; 214 struct cm_port *port[]; 215 }; 216 217 struct cm_av { 218 struct cm_port *port; 219 union ib_gid dgid; 220 struct rdma_ah_attr ah_attr; 221 u16 pkey_index; 222 u8 timeout; 223 }; 224 225 struct cm_work { 226 struct delayed_work work; 227 struct list_head list; 228 struct cm_port *port; 229 struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */ 230 __be32 local_id; /* Established / timewait */ 231 __be32 remote_id; 232 struct ib_cm_event cm_event; 233 struct sa_path_rec path[]; 234 }; 235 236 struct cm_timewait_info { 237 struct cm_work work; 238 struct list_head list; 239 struct rb_node remote_qp_node; 240 struct rb_node remote_id_node; 241 __be64 remote_ca_guid; 242 __be32 remote_qpn; 243 u8 inserted_remote_qp; 244 u8 inserted_remote_id; 245 }; 246 247 struct cm_id_private { 248 struct ib_cm_id id; 249 250 struct rb_node service_node; 251 struct rb_node sidr_id_node; 252 spinlock_t lock; /* Do not acquire inside cm.lock */ 253 struct completion comp; 254 refcount_t refcount; 255 /* Number of clients sharing this ib_cm_id. Only valid for listeners. 256 * Protected by the cm.lock spinlock. */ 257 int listen_sharecount; 258 struct rcu_head rcu; 259 260 struct ib_mad_send_buf *msg; 261 struct cm_timewait_info *timewait_info; 262 /* todo: use alternate port on send failure */ 263 struct cm_av av; 264 struct cm_av alt_av; 265 266 void *private_data; 267 __be64 tid; 268 __be32 local_qpn; 269 __be32 remote_qpn; 270 enum ib_qp_type qp_type; 271 __be32 sq_psn; 272 __be32 rq_psn; 273 int timeout_ms; 274 enum ib_mtu path_mtu; 275 __be16 pkey; 276 u8 private_data_len; 277 u8 max_cm_retries; 278 u8 responder_resources; 279 u8 initiator_depth; 280 u8 retry_count; 281 u8 rnr_retry_count; 282 u8 service_timeout; 283 u8 target_ack_delay; 284 285 struct list_head prim_list; 286 struct list_head altr_list; 287 /* Indicates that the send port mad is registered and av is set */ 288 int prim_send_port_not_ready; 289 int altr_send_port_not_ready; 290 291 struct list_head work_list; 292 atomic_t work_count; 293 }; 294 295 static void cm_work_handler(struct work_struct *work); 296 297 static inline void cm_deref_id(struct cm_id_private *cm_id_priv) 298 { 299 if (refcount_dec_and_test(&cm_id_priv->refcount)) 300 complete(&cm_id_priv->comp); 301 } 302 303 static int cm_alloc_msg(struct cm_id_private *cm_id_priv, 304 struct ib_mad_send_buf **msg) 305 { 306 struct ib_mad_agent *mad_agent; 307 struct ib_mad_send_buf *m; 308 struct ib_ah *ah; 309 struct cm_av *av; 310 unsigned long flags, flags2; 311 int ret = 0; 312 313 /* don't let the port to be released till the agent is down */ 314 spin_lock_irqsave(&cm.state_lock, flags2); 315 spin_lock_irqsave(&cm.lock, flags); 316 if (!cm_id_priv->prim_send_port_not_ready) 317 av = &cm_id_priv->av; 318 else if (!cm_id_priv->altr_send_port_not_ready && 319 (cm_id_priv->alt_av.port)) 320 av = &cm_id_priv->alt_av; 321 else { 322 pr_info("%s: not valid CM id\n", __func__); 323 ret = -ENODEV; 324 spin_unlock_irqrestore(&cm.lock, flags); 325 goto out; 326 } 327 spin_unlock_irqrestore(&cm.lock, flags); 328 /* Make sure the port haven't released the mad yet */ 329 mad_agent = cm_id_priv->av.port->mad_agent; 330 if (!mad_agent) { 331 pr_info("%s: not a valid MAD agent\n", __func__); 332 ret = -ENODEV; 333 goto out; 334 } 335 ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr, 0); 336 if (IS_ERR(ah)) { 337 ret = PTR_ERR(ah); 338 goto out; 339 } 340 341 m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn, 342 av->pkey_index, 343 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, 344 GFP_ATOMIC, 345 IB_MGMT_BASE_VERSION); 346 if (IS_ERR(m)) { 347 rdma_destroy_ah(ah, 0); 348 ret = PTR_ERR(m); 349 goto out; 350 } 351 352 /* Timeout set by caller if response is expected. */ 353 m->ah = ah; 354 m->retries = cm_id_priv->max_cm_retries; 355 356 refcount_inc(&cm_id_priv->refcount); 357 m->context[0] = cm_id_priv; 358 *msg = m; 359 360 out: 361 spin_unlock_irqrestore(&cm.state_lock, flags2); 362 return ret; 363 } 364 365 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port, 366 struct ib_mad_recv_wc *mad_recv_wc) 367 { 368 return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index, 369 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, 370 GFP_ATOMIC, 371 IB_MGMT_BASE_VERSION); 372 } 373 374 static int cm_create_response_msg_ah(struct cm_port *port, 375 struct ib_mad_recv_wc *mad_recv_wc, 376 struct ib_mad_send_buf *msg) 377 { 378 struct ib_ah *ah; 379 380 ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc, 381 mad_recv_wc->recv_buf.grh, port->port_num); 382 if (IS_ERR(ah)) 383 return PTR_ERR(ah); 384 385 msg->ah = ah; 386 return 0; 387 } 388 389 static void cm_free_msg(struct ib_mad_send_buf *msg) 390 { 391 if (msg->ah) 392 rdma_destroy_ah(msg->ah, 0); 393 if (msg->context[0]) 394 cm_deref_id(msg->context[0]); 395 ib_free_send_mad(msg); 396 } 397 398 static int cm_alloc_response_msg(struct cm_port *port, 399 struct ib_mad_recv_wc *mad_recv_wc, 400 struct ib_mad_send_buf **msg) 401 { 402 struct ib_mad_send_buf *m; 403 int ret; 404 405 m = cm_alloc_response_msg_no_ah(port, mad_recv_wc); 406 if (IS_ERR(m)) 407 return PTR_ERR(m); 408 409 ret = cm_create_response_msg_ah(port, mad_recv_wc, m); 410 if (ret) { 411 cm_free_msg(m); 412 return ret; 413 } 414 415 *msg = m; 416 return 0; 417 } 418 419 static void * cm_copy_private_data(const void *private_data, 420 u8 private_data_len) 421 { 422 void *data; 423 424 if (!private_data || !private_data_len) 425 return NULL; 426 427 data = kmemdup(private_data, private_data_len, GFP_KERNEL); 428 if (!data) 429 return ERR_PTR(-ENOMEM); 430 431 return data; 432 } 433 434 static void cm_set_private_data(struct cm_id_private *cm_id_priv, 435 void *private_data, u8 private_data_len) 436 { 437 if (cm_id_priv->private_data && cm_id_priv->private_data_len) 438 kfree(cm_id_priv->private_data); 439 440 cm_id_priv->private_data = private_data; 441 cm_id_priv->private_data_len = private_data_len; 442 } 443 444 static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc, 445 struct ib_grh *grh, struct cm_av *av) 446 { 447 struct rdma_ah_attr new_ah_attr; 448 int ret; 449 450 av->port = port; 451 av->pkey_index = wc->pkey_index; 452 453 /* 454 * av->ah_attr might be initialized based on past wc during incoming 455 * connect request or while sending out connect request. So initialize 456 * a new ah_attr on stack. If initialization fails, old ah_attr is 457 * used for sending any responses. If initialization is successful, 458 * than new ah_attr is used by overwriting old one. 459 */ 460 ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device, 461 port->port_num, wc, 462 grh, &new_ah_attr); 463 if (ret) 464 return ret; 465 466 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr); 467 return 0; 468 } 469 470 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc, 471 struct ib_grh *grh, struct cm_av *av) 472 { 473 av->port = port; 474 av->pkey_index = wc->pkey_index; 475 return ib_init_ah_attr_from_wc(port->cm_dev->ib_device, 476 port->port_num, wc, 477 grh, &av->ah_attr); 478 } 479 480 static void add_cm_id_to_port_list(struct cm_id_private *cm_id_priv, 481 struct cm_av *av, struct cm_port *port) 482 { 483 unsigned long flags; 484 485 spin_lock_irqsave(&cm.lock, flags); 486 if (&cm_id_priv->av == av) 487 list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list); 488 else if (&cm_id_priv->alt_av == av) 489 list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list); 490 else 491 WARN_ON(true); 492 spin_unlock_irqrestore(&cm.lock, flags); 493 } 494 495 static struct cm_port * 496 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr) 497 { 498 struct cm_device *cm_dev; 499 struct cm_port *port = NULL; 500 unsigned long flags; 501 502 if (attr) { 503 read_lock_irqsave(&cm.device_lock, flags); 504 list_for_each_entry(cm_dev, &cm.device_list, list) { 505 if (cm_dev->ib_device == attr->device) { 506 port = cm_dev->port[attr->port_num - 1]; 507 break; 508 } 509 } 510 read_unlock_irqrestore(&cm.device_lock, flags); 511 } else { 512 /* SGID attribute can be NULL in following 513 * conditions. 514 * (a) Alternative path 515 * (b) IB link layer without GRH 516 * (c) LAP send messages 517 */ 518 read_lock_irqsave(&cm.device_lock, flags); 519 list_for_each_entry(cm_dev, &cm.device_list, list) { 520 attr = rdma_find_gid(cm_dev->ib_device, 521 &path->sgid, 522 sa_conv_pathrec_to_gid_type(path), 523 NULL); 524 if (!IS_ERR(attr)) { 525 port = cm_dev->port[attr->port_num - 1]; 526 break; 527 } 528 } 529 read_unlock_irqrestore(&cm.device_lock, flags); 530 if (port) 531 rdma_put_gid_attr(attr); 532 } 533 return port; 534 } 535 536 static int cm_init_av_by_path(struct sa_path_rec *path, 537 const struct ib_gid_attr *sgid_attr, 538 struct cm_av *av, 539 struct cm_id_private *cm_id_priv) 540 { 541 struct rdma_ah_attr new_ah_attr; 542 struct cm_device *cm_dev; 543 struct cm_port *port; 544 int ret; 545 546 port = get_cm_port_from_path(path, sgid_attr); 547 if (!port) 548 return -EINVAL; 549 cm_dev = port->cm_dev; 550 551 ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num, 552 be16_to_cpu(path->pkey), &av->pkey_index); 553 if (ret) 554 return ret; 555 556 av->port = port; 557 558 /* 559 * av->ah_attr might be initialized based on wc or during 560 * request processing time which might have reference to sgid_attr. 561 * So initialize a new ah_attr on stack. 562 * If initialization fails, old ah_attr is used for sending any 563 * responses. If initialization is successful, than new ah_attr 564 * is used by overwriting the old one. So that right ah_attr 565 * can be used to return an error response. 566 */ 567 ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path, 568 &new_ah_attr, sgid_attr); 569 if (ret) 570 return ret; 571 572 av->timeout = path->packet_life_time + 1; 573 add_cm_id_to_port_list(cm_id_priv, av, port); 574 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr); 575 return 0; 576 } 577 578 static u32 cm_local_id(__be32 local_id) 579 { 580 return (__force u32) (local_id ^ cm.random_id_operand); 581 } 582 583 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id) 584 { 585 struct cm_id_private *cm_id_priv; 586 587 rcu_read_lock(); 588 cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id)); 589 if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id || 590 !refcount_inc_not_zero(&cm_id_priv->refcount)) 591 cm_id_priv = NULL; 592 rcu_read_unlock(); 593 594 return cm_id_priv; 595 } 596 597 /* 598 * Trivial helpers to strip endian annotation and compare; the 599 * endianness doesn't actually matter since we just need a stable 600 * order for the RB tree. 601 */ 602 static int be32_lt(__be32 a, __be32 b) 603 { 604 return (__force u32) a < (__force u32) b; 605 } 606 607 static int be32_gt(__be32 a, __be32 b) 608 { 609 return (__force u32) a > (__force u32) b; 610 } 611 612 static int be64_lt(__be64 a, __be64 b) 613 { 614 return (__force u64) a < (__force u64) b; 615 } 616 617 static int be64_gt(__be64 a, __be64 b) 618 { 619 return (__force u64) a > (__force u64) b; 620 } 621 622 /* 623 * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv 624 * if the new ID was inserted, NULL if it could not be inserted due to a 625 * collision, or the existing cm_id_priv ready for shared usage. 626 */ 627 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv, 628 ib_cm_handler shared_handler) 629 { 630 struct rb_node **link = &cm.listen_service_table.rb_node; 631 struct rb_node *parent = NULL; 632 struct cm_id_private *cur_cm_id_priv; 633 __be64 service_id = cm_id_priv->id.service_id; 634 __be64 service_mask = cm_id_priv->id.service_mask; 635 unsigned long flags; 636 637 spin_lock_irqsave(&cm.lock, flags); 638 while (*link) { 639 parent = *link; 640 cur_cm_id_priv = rb_entry(parent, struct cm_id_private, 641 service_node); 642 if ((cur_cm_id_priv->id.service_mask & service_id) == 643 (service_mask & cur_cm_id_priv->id.service_id) && 644 (cm_id_priv->id.device == cur_cm_id_priv->id.device)) { 645 /* 646 * Sharing an ib_cm_id with different handlers is not 647 * supported 648 */ 649 if (cur_cm_id_priv->id.cm_handler != shared_handler || 650 cur_cm_id_priv->id.context || 651 WARN_ON(!cur_cm_id_priv->id.cm_handler)) { 652 spin_unlock_irqrestore(&cm.lock, flags); 653 return NULL; 654 } 655 refcount_inc(&cur_cm_id_priv->refcount); 656 cur_cm_id_priv->listen_sharecount++; 657 spin_unlock_irqrestore(&cm.lock, flags); 658 return cur_cm_id_priv; 659 } 660 661 if (cm_id_priv->id.device < cur_cm_id_priv->id.device) 662 link = &(*link)->rb_left; 663 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device) 664 link = &(*link)->rb_right; 665 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id)) 666 link = &(*link)->rb_left; 667 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id)) 668 link = &(*link)->rb_right; 669 else 670 link = &(*link)->rb_right; 671 } 672 cm_id_priv->listen_sharecount++; 673 rb_link_node(&cm_id_priv->service_node, parent, link); 674 rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table); 675 spin_unlock_irqrestore(&cm.lock, flags); 676 return cm_id_priv; 677 } 678 679 static struct cm_id_private * cm_find_listen(struct ib_device *device, 680 __be64 service_id) 681 { 682 struct rb_node *node = cm.listen_service_table.rb_node; 683 struct cm_id_private *cm_id_priv; 684 685 while (node) { 686 cm_id_priv = rb_entry(node, struct cm_id_private, service_node); 687 if ((cm_id_priv->id.service_mask & service_id) == 688 cm_id_priv->id.service_id && 689 (cm_id_priv->id.device == device)) { 690 refcount_inc(&cm_id_priv->refcount); 691 return cm_id_priv; 692 } 693 if (device < cm_id_priv->id.device) 694 node = node->rb_left; 695 else if (device > cm_id_priv->id.device) 696 node = node->rb_right; 697 else if (be64_lt(service_id, cm_id_priv->id.service_id)) 698 node = node->rb_left; 699 else if (be64_gt(service_id, cm_id_priv->id.service_id)) 700 node = node->rb_right; 701 else 702 node = node->rb_right; 703 } 704 return NULL; 705 } 706 707 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info 708 *timewait_info) 709 { 710 struct rb_node **link = &cm.remote_id_table.rb_node; 711 struct rb_node *parent = NULL; 712 struct cm_timewait_info *cur_timewait_info; 713 __be64 remote_ca_guid = timewait_info->remote_ca_guid; 714 __be32 remote_id = timewait_info->work.remote_id; 715 716 while (*link) { 717 parent = *link; 718 cur_timewait_info = rb_entry(parent, struct cm_timewait_info, 719 remote_id_node); 720 if (be32_lt(remote_id, cur_timewait_info->work.remote_id)) 721 link = &(*link)->rb_left; 722 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id)) 723 link = &(*link)->rb_right; 724 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 725 link = &(*link)->rb_left; 726 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 727 link = &(*link)->rb_right; 728 else 729 return cur_timewait_info; 730 } 731 timewait_info->inserted_remote_id = 1; 732 rb_link_node(&timewait_info->remote_id_node, parent, link); 733 rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table); 734 return NULL; 735 } 736 737 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid, 738 __be32 remote_id) 739 { 740 struct rb_node *node = cm.remote_id_table.rb_node; 741 struct cm_timewait_info *timewait_info; 742 struct cm_id_private *res = NULL; 743 744 spin_lock_irq(&cm.lock); 745 while (node) { 746 timewait_info = rb_entry(node, struct cm_timewait_info, 747 remote_id_node); 748 if (be32_lt(remote_id, timewait_info->work.remote_id)) 749 node = node->rb_left; 750 else if (be32_gt(remote_id, timewait_info->work.remote_id)) 751 node = node->rb_right; 752 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid)) 753 node = node->rb_left; 754 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid)) 755 node = node->rb_right; 756 else { 757 res = cm_acquire_id(timewait_info->work.local_id, 758 timewait_info->work.remote_id); 759 break; 760 } 761 } 762 spin_unlock_irq(&cm.lock); 763 return res; 764 } 765 766 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info 767 *timewait_info) 768 { 769 struct rb_node **link = &cm.remote_qp_table.rb_node; 770 struct rb_node *parent = NULL; 771 struct cm_timewait_info *cur_timewait_info; 772 __be64 remote_ca_guid = timewait_info->remote_ca_guid; 773 __be32 remote_qpn = timewait_info->remote_qpn; 774 775 while (*link) { 776 parent = *link; 777 cur_timewait_info = rb_entry(parent, struct cm_timewait_info, 778 remote_qp_node); 779 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn)) 780 link = &(*link)->rb_left; 781 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn)) 782 link = &(*link)->rb_right; 783 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 784 link = &(*link)->rb_left; 785 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid)) 786 link = &(*link)->rb_right; 787 else 788 return cur_timewait_info; 789 } 790 timewait_info->inserted_remote_qp = 1; 791 rb_link_node(&timewait_info->remote_qp_node, parent, link); 792 rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table); 793 return NULL; 794 } 795 796 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private 797 *cm_id_priv) 798 { 799 struct rb_node **link = &cm.remote_sidr_table.rb_node; 800 struct rb_node *parent = NULL; 801 struct cm_id_private *cur_cm_id_priv; 802 union ib_gid *port_gid = &cm_id_priv->av.dgid; 803 __be32 remote_id = cm_id_priv->id.remote_id; 804 805 while (*link) { 806 parent = *link; 807 cur_cm_id_priv = rb_entry(parent, struct cm_id_private, 808 sidr_id_node); 809 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id)) 810 link = &(*link)->rb_left; 811 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id)) 812 link = &(*link)->rb_right; 813 else { 814 int cmp; 815 cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid, 816 sizeof *port_gid); 817 if (cmp < 0) 818 link = &(*link)->rb_left; 819 else if (cmp > 0) 820 link = &(*link)->rb_right; 821 else 822 return cur_cm_id_priv; 823 } 824 } 825 rb_link_node(&cm_id_priv->sidr_id_node, parent, link); 826 rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); 827 return NULL; 828 } 829 830 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device, 831 ib_cm_handler cm_handler, 832 void *context) 833 { 834 struct cm_id_private *cm_id_priv; 835 u32 id; 836 int ret; 837 838 cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL); 839 if (!cm_id_priv) 840 return ERR_PTR(-ENOMEM); 841 842 cm_id_priv->id.state = IB_CM_IDLE; 843 cm_id_priv->id.device = device; 844 cm_id_priv->id.cm_handler = cm_handler; 845 cm_id_priv->id.context = context; 846 cm_id_priv->id.remote_cm_qpn = 1; 847 848 RB_CLEAR_NODE(&cm_id_priv->service_node); 849 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node); 850 spin_lock_init(&cm_id_priv->lock); 851 init_completion(&cm_id_priv->comp); 852 INIT_LIST_HEAD(&cm_id_priv->work_list); 853 INIT_LIST_HEAD(&cm_id_priv->prim_list); 854 INIT_LIST_HEAD(&cm_id_priv->altr_list); 855 atomic_set(&cm_id_priv->work_count, -1); 856 refcount_set(&cm_id_priv->refcount, 1); 857 858 ret = xa_alloc_cyclic_irq(&cm.local_id_table, &id, NULL, xa_limit_32b, 859 &cm.local_id_next, GFP_KERNEL); 860 if (ret) 861 goto error; 862 cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand; 863 864 return cm_id_priv; 865 866 error: 867 kfree(cm_id_priv); 868 return ERR_PTR(ret); 869 } 870 871 /* 872 * Make the ID visible to the MAD handlers and other threads that use the 873 * xarray. 874 */ 875 static void cm_finalize_id(struct cm_id_private *cm_id_priv) 876 { 877 xa_store_irq(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id), 878 cm_id_priv, GFP_KERNEL); 879 } 880 881 struct ib_cm_id *ib_create_cm_id(struct ib_device *device, 882 ib_cm_handler cm_handler, 883 void *context) 884 { 885 struct cm_id_private *cm_id_priv; 886 887 cm_id_priv = cm_alloc_id_priv(device, cm_handler, context); 888 if (IS_ERR(cm_id_priv)) 889 return ERR_CAST(cm_id_priv); 890 891 cm_finalize_id(cm_id_priv); 892 return &cm_id_priv->id; 893 } 894 EXPORT_SYMBOL(ib_create_cm_id); 895 896 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv) 897 { 898 struct cm_work *work; 899 900 if (list_empty(&cm_id_priv->work_list)) 901 return NULL; 902 903 work = list_entry(cm_id_priv->work_list.next, struct cm_work, list); 904 list_del(&work->list); 905 return work; 906 } 907 908 static void cm_free_work(struct cm_work *work) 909 { 910 if (work->mad_recv_wc) 911 ib_free_recv_mad(work->mad_recv_wc); 912 kfree(work); 913 } 914 915 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv, 916 struct cm_work *work) 917 { 918 bool immediate; 919 920 /* 921 * To deliver the event to the user callback we have the drop the 922 * spinlock, however, we need to ensure that the user callback is single 923 * threaded and receives events in the temporal order. If there are 924 * already events being processed then thread new events onto a list, 925 * the thread currently processing will pick them up. 926 */ 927 immediate = atomic_inc_and_test(&cm_id_priv->work_count); 928 if (!immediate) { 929 list_add_tail(&work->list, &cm_id_priv->work_list); 930 /* 931 * This routine always consumes incoming reference. Once queued 932 * to the work_list then a reference is held by the thread 933 * currently running cm_process_work() and this reference is not 934 * needed. 935 */ 936 cm_deref_id(cm_id_priv); 937 } 938 spin_unlock_irq(&cm_id_priv->lock); 939 940 if (immediate) 941 cm_process_work(cm_id_priv, work); 942 } 943 944 static inline int cm_convert_to_ms(int iba_time) 945 { 946 /* approximate conversion to ms from 4.096us x 2^iba_time */ 947 return 1 << max(iba_time - 8, 0); 948 } 949 950 /* 951 * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time 952 * Because of how ack_timeout is stored, adding one doubles the timeout. 953 * To avoid large timeouts, select the max(ack_delay, life_time + 1), and 954 * increment it (round up) only if the other is within 50%. 955 */ 956 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time) 957 { 958 int ack_timeout = packet_life_time + 1; 959 960 if (ack_timeout >= ca_ack_delay) 961 ack_timeout += (ca_ack_delay >= (ack_timeout - 1)); 962 else 963 ack_timeout = ca_ack_delay + 964 (ack_timeout >= (ca_ack_delay - 1)); 965 966 return min(31, ack_timeout); 967 } 968 969 static void cm_remove_remote(struct cm_id_private *cm_id_priv) 970 { 971 struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info; 972 973 if (timewait_info->inserted_remote_id) { 974 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table); 975 timewait_info->inserted_remote_id = 0; 976 } 977 978 if (timewait_info->inserted_remote_qp) { 979 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table); 980 timewait_info->inserted_remote_qp = 0; 981 } 982 } 983 984 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id) 985 { 986 struct cm_timewait_info *timewait_info; 987 988 timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL); 989 if (!timewait_info) 990 return ERR_PTR(-ENOMEM); 991 992 timewait_info->work.local_id = local_id; 993 INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler); 994 timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT; 995 return timewait_info; 996 } 997 998 static void cm_enter_timewait(struct cm_id_private *cm_id_priv) 999 { 1000 int wait_time; 1001 unsigned long flags; 1002 struct cm_device *cm_dev; 1003 1004 lockdep_assert_held(&cm_id_priv->lock); 1005 1006 cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client); 1007 if (!cm_dev) 1008 return; 1009 1010 spin_lock_irqsave(&cm.lock, flags); 1011 cm_remove_remote(cm_id_priv); 1012 list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list); 1013 spin_unlock_irqrestore(&cm.lock, flags); 1014 1015 /* 1016 * The cm_id could be destroyed by the user before we exit timewait. 1017 * To protect against this, we search for the cm_id after exiting 1018 * timewait before notifying the user that we've exited timewait. 1019 */ 1020 cm_id_priv->id.state = IB_CM_TIMEWAIT; 1021 wait_time = cm_convert_to_ms(cm_id_priv->av.timeout); 1022 1023 /* Check if the device started its remove_one */ 1024 spin_lock_irqsave(&cm.lock, flags); 1025 if (!cm_dev->going_down) 1026 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work, 1027 msecs_to_jiffies(wait_time)); 1028 spin_unlock_irqrestore(&cm.lock, flags); 1029 1030 /* 1031 * The timewait_info is converted into a work and gets freed during 1032 * cm_free_work() in cm_timewait_handler(). 1033 */ 1034 BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0); 1035 cm_id_priv->timewait_info = NULL; 1036 } 1037 1038 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv) 1039 { 1040 unsigned long flags; 1041 1042 lockdep_assert_held(&cm_id_priv->lock); 1043 1044 cm_id_priv->id.state = IB_CM_IDLE; 1045 if (cm_id_priv->timewait_info) { 1046 spin_lock_irqsave(&cm.lock, flags); 1047 cm_remove_remote(cm_id_priv); 1048 spin_unlock_irqrestore(&cm.lock, flags); 1049 kfree(cm_id_priv->timewait_info); 1050 cm_id_priv->timewait_info = NULL; 1051 } 1052 } 1053 1054 static void cm_destroy_id(struct ib_cm_id *cm_id, int err) 1055 { 1056 struct cm_id_private *cm_id_priv; 1057 struct cm_work *work; 1058 1059 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 1060 spin_lock_irq(&cm_id_priv->lock); 1061 retest: 1062 switch (cm_id->state) { 1063 case IB_CM_LISTEN: 1064 spin_lock(&cm.lock); 1065 if (--cm_id_priv->listen_sharecount > 0) { 1066 /* The id is still shared. */ 1067 WARN_ON(refcount_read(&cm_id_priv->refcount) == 1); 1068 spin_unlock(&cm.lock); 1069 spin_unlock_irq(&cm_id_priv->lock); 1070 cm_deref_id(cm_id_priv); 1071 return; 1072 } 1073 cm_id->state = IB_CM_IDLE; 1074 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table); 1075 RB_CLEAR_NODE(&cm_id_priv->service_node); 1076 spin_unlock(&cm.lock); 1077 break; 1078 case IB_CM_SIDR_REQ_SENT: 1079 cm_id->state = IB_CM_IDLE; 1080 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1081 break; 1082 case IB_CM_SIDR_REQ_RCVD: 1083 cm_send_sidr_rep_locked(cm_id_priv, 1084 &(struct ib_cm_sidr_rep_param){ 1085 .status = IB_SIDR_REJECT }); 1086 /* cm_send_sidr_rep_locked will not move to IDLE if it fails */ 1087 cm_id->state = IB_CM_IDLE; 1088 break; 1089 case IB_CM_REQ_SENT: 1090 case IB_CM_MRA_REQ_RCVD: 1091 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1092 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT, 1093 &cm_id_priv->id.device->node_guid, 1094 sizeof(cm_id_priv->id.device->node_guid), 1095 NULL, 0); 1096 break; 1097 case IB_CM_REQ_RCVD: 1098 if (err == -ENOMEM) { 1099 /* Do not reject to allow future retries. */ 1100 cm_reset_to_idle(cm_id_priv); 1101 } else { 1102 cm_send_rej_locked(cm_id_priv, 1103 IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, 1104 NULL, 0); 1105 } 1106 break; 1107 case IB_CM_REP_SENT: 1108 case IB_CM_MRA_REP_RCVD: 1109 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1110 /* Fall through */ 1111 case IB_CM_MRA_REQ_SENT: 1112 case IB_CM_REP_RCVD: 1113 case IB_CM_MRA_REP_SENT: 1114 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL, 1115 0, NULL, 0); 1116 break; 1117 case IB_CM_ESTABLISHED: 1118 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) { 1119 cm_id->state = IB_CM_IDLE; 1120 break; 1121 } 1122 cm_send_dreq_locked(cm_id_priv, NULL, 0); 1123 goto retest; 1124 case IB_CM_DREQ_SENT: 1125 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 1126 cm_enter_timewait(cm_id_priv); 1127 goto retest; 1128 case IB_CM_DREQ_RCVD: 1129 cm_send_drep_locked(cm_id_priv, NULL, 0); 1130 WARN_ON(cm_id->state != IB_CM_TIMEWAIT); 1131 goto retest; 1132 case IB_CM_TIMEWAIT: 1133 /* 1134 * The cm_acquire_id in cm_timewait_handler will stop working 1135 * once we do xa_erase below, so just move to idle here for 1136 * consistency. 1137 */ 1138 cm_id->state = IB_CM_IDLE; 1139 break; 1140 case IB_CM_IDLE: 1141 break; 1142 } 1143 WARN_ON(cm_id->state != IB_CM_IDLE); 1144 1145 spin_lock(&cm.lock); 1146 /* Required for cleanup paths related cm_req_handler() */ 1147 if (cm_id_priv->timewait_info) { 1148 cm_remove_remote(cm_id_priv); 1149 kfree(cm_id_priv->timewait_info); 1150 cm_id_priv->timewait_info = NULL; 1151 } 1152 if (!list_empty(&cm_id_priv->altr_list) && 1153 (!cm_id_priv->altr_send_port_not_ready)) 1154 list_del(&cm_id_priv->altr_list); 1155 if (!list_empty(&cm_id_priv->prim_list) && 1156 (!cm_id_priv->prim_send_port_not_ready)) 1157 list_del(&cm_id_priv->prim_list); 1158 WARN_ON(cm_id_priv->listen_sharecount); 1159 WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node)); 1160 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) 1161 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); 1162 spin_unlock(&cm.lock); 1163 spin_unlock_irq(&cm_id_priv->lock); 1164 1165 xa_erase_irq(&cm.local_id_table, cm_local_id(cm_id->local_id)); 1166 cm_deref_id(cm_id_priv); 1167 wait_for_completion(&cm_id_priv->comp); 1168 while ((work = cm_dequeue_work(cm_id_priv)) != NULL) 1169 cm_free_work(work); 1170 1171 rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr); 1172 rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr); 1173 kfree(cm_id_priv->private_data); 1174 kfree_rcu(cm_id_priv, rcu); 1175 } 1176 1177 void ib_destroy_cm_id(struct ib_cm_id *cm_id) 1178 { 1179 cm_destroy_id(cm_id, 0); 1180 } 1181 EXPORT_SYMBOL(ib_destroy_cm_id); 1182 1183 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id, 1184 __be64 service_mask) 1185 { 1186 service_mask = service_mask ? service_mask : ~cpu_to_be64(0); 1187 service_id &= service_mask; 1188 if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID && 1189 (service_id != IB_CM_ASSIGN_SERVICE_ID)) 1190 return -EINVAL; 1191 1192 if (service_id == IB_CM_ASSIGN_SERVICE_ID) { 1193 cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++); 1194 cm_id_priv->id.service_mask = ~cpu_to_be64(0); 1195 } else { 1196 cm_id_priv->id.service_id = service_id; 1197 cm_id_priv->id.service_mask = service_mask; 1198 } 1199 return 0; 1200 } 1201 1202 /** 1203 * ib_cm_listen - Initiates listening on the specified service ID for 1204 * connection and service ID resolution requests. 1205 * @cm_id: Connection identifier associated with the listen request. 1206 * @service_id: Service identifier matched against incoming connection 1207 * and service ID resolution requests. The service ID should be specified 1208 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will 1209 * assign a service ID to the caller. 1210 * @service_mask: Mask applied to service ID used to listen across a 1211 * range of service IDs. If set to 0, the service ID is matched 1212 * exactly. This parameter is ignored if %service_id is set to 1213 * IB_CM_ASSIGN_SERVICE_ID. 1214 */ 1215 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask) 1216 { 1217 struct cm_id_private *cm_id_priv = 1218 container_of(cm_id, struct cm_id_private, id); 1219 unsigned long flags; 1220 int ret; 1221 1222 spin_lock_irqsave(&cm_id_priv->lock, flags); 1223 if (cm_id_priv->id.state != IB_CM_IDLE) { 1224 ret = -EINVAL; 1225 goto out; 1226 } 1227 1228 ret = cm_init_listen(cm_id_priv, service_id, service_mask); 1229 if (ret) 1230 goto out; 1231 1232 if (!cm_insert_listen(cm_id_priv, NULL)) { 1233 ret = -EBUSY; 1234 goto out; 1235 } 1236 1237 cm_id_priv->id.state = IB_CM_LISTEN; 1238 ret = 0; 1239 1240 out: 1241 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1242 return ret; 1243 } 1244 EXPORT_SYMBOL(ib_cm_listen); 1245 1246 /** 1247 * Create a new listening ib_cm_id and listen on the given service ID. 1248 * 1249 * If there's an existing ID listening on that same device and service ID, 1250 * return it. 1251 * 1252 * @device: Device associated with the cm_id. All related communication will 1253 * be associated with the specified device. 1254 * @cm_handler: Callback invoked to notify the user of CM events. 1255 * @service_id: Service identifier matched against incoming connection 1256 * and service ID resolution requests. The service ID should be specified 1257 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will 1258 * assign a service ID to the caller. 1259 * 1260 * Callers should call ib_destroy_cm_id when done with the listener ID. 1261 */ 1262 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device, 1263 ib_cm_handler cm_handler, 1264 __be64 service_id) 1265 { 1266 struct cm_id_private *listen_id_priv; 1267 struct cm_id_private *cm_id_priv; 1268 int err = 0; 1269 1270 /* Create an ID in advance, since the creation may sleep */ 1271 cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL); 1272 if (IS_ERR(cm_id_priv)) 1273 return ERR_CAST(cm_id_priv); 1274 1275 err = cm_init_listen(cm_id_priv, service_id, 0); 1276 if (err) 1277 return ERR_PTR(err); 1278 1279 spin_lock_irq(&cm_id_priv->lock); 1280 listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler); 1281 if (listen_id_priv != cm_id_priv) { 1282 spin_unlock_irq(&cm_id_priv->lock); 1283 ib_destroy_cm_id(&cm_id_priv->id); 1284 if (!listen_id_priv) 1285 return ERR_PTR(-EINVAL); 1286 return &listen_id_priv->id; 1287 } 1288 cm_id_priv->id.state = IB_CM_LISTEN; 1289 spin_unlock_irq(&cm_id_priv->lock); 1290 1291 /* 1292 * A listen ID does not need to be in the xarray since it does not 1293 * receive mads, is not placed in the remote_id or remote_qpn rbtree, 1294 * and does not enter timewait. 1295 */ 1296 1297 return &cm_id_priv->id; 1298 } 1299 EXPORT_SYMBOL(ib_cm_insert_listen); 1300 1301 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv) 1302 { 1303 u64 hi_tid, low_tid; 1304 1305 hi_tid = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32; 1306 low_tid = (u64)cm_id_priv->id.local_id; 1307 return cpu_to_be64(hi_tid | low_tid); 1308 } 1309 1310 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr, 1311 __be16 attr_id, __be64 tid) 1312 { 1313 hdr->base_version = IB_MGMT_BASE_VERSION; 1314 hdr->mgmt_class = IB_MGMT_CLASS_CM; 1315 hdr->class_version = IB_CM_CLASS_VERSION; 1316 hdr->method = IB_MGMT_METHOD_SEND; 1317 hdr->attr_id = attr_id; 1318 hdr->tid = tid; 1319 } 1320 1321 static void cm_format_req(struct cm_req_msg *req_msg, 1322 struct cm_id_private *cm_id_priv, 1323 struct ib_cm_req_param *param) 1324 { 1325 struct sa_path_rec *pri_path = param->primary_path; 1326 struct sa_path_rec *alt_path = param->alternate_path; 1327 bool pri_ext = false; 1328 1329 if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA) 1330 pri_ext = opa_is_extended_lid(pri_path->opa.dlid, 1331 pri_path->opa.slid); 1332 1333 cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID, 1334 cm_form_tid(cm_id_priv)); 1335 1336 IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg, 1337 be32_to_cpu(cm_id_priv->id.local_id)); 1338 IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id)); 1339 IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg, 1340 be64_to_cpu(cm_id_priv->id.device->node_guid)); 1341 IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num); 1342 IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth); 1343 IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg, 1344 param->remote_cm_response_timeout); 1345 cm_req_set_qp_type(req_msg, param->qp_type); 1346 IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control); 1347 IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn); 1348 IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg, 1349 param->local_cm_response_timeout); 1350 IBA_SET(CM_REQ_PARTITION_KEY, req_msg, 1351 be16_to_cpu(param->primary_path->pkey)); 1352 IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg, 1353 param->primary_path->mtu); 1354 IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries); 1355 1356 if (param->qp_type != IB_QPT_XRC_INI) { 1357 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg, 1358 param->responder_resources); 1359 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count); 1360 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg, 1361 param->rnr_retry_count); 1362 IBA_SET(CM_REQ_SRQ, req_msg, param->srq); 1363 } 1364 1365 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) = 1366 pri_path->sgid; 1367 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) = 1368 pri_path->dgid; 1369 if (pri_ext) { 1370 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) 1371 ->global.interface_id = 1372 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid)); 1373 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) 1374 ->global.interface_id = 1375 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid)); 1376 } 1377 if (pri_path->hop_limit <= 1) { 1378 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, 1379 be16_to_cpu(pri_ext ? 0 : 1380 htons(ntohl(sa_path_get_slid( 1381 pri_path))))); 1382 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, 1383 be16_to_cpu(pri_ext ? 0 : 1384 htons(ntohl(sa_path_get_dlid( 1385 pri_path))))); 1386 } else { 1387 /* Work-around until there's a way to obtain remote LID info */ 1388 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, 1389 be16_to_cpu(IB_LID_PERMISSIVE)); 1390 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, 1391 be16_to_cpu(IB_LID_PERMISSIVE)); 1392 } 1393 IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg, 1394 be32_to_cpu(pri_path->flow_label)); 1395 IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate); 1396 IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class); 1397 IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit); 1398 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl); 1399 IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg, 1400 (pri_path->hop_limit <= 1)); 1401 IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg, 1402 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, 1403 pri_path->packet_life_time)); 1404 1405 if (alt_path) { 1406 bool alt_ext = false; 1407 1408 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA) 1409 alt_ext = opa_is_extended_lid(alt_path->opa.dlid, 1410 alt_path->opa.slid); 1411 1412 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) = 1413 alt_path->sgid; 1414 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) = 1415 alt_path->dgid; 1416 if (alt_ext) { 1417 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, 1418 req_msg) 1419 ->global.interface_id = 1420 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid)); 1421 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, 1422 req_msg) 1423 ->global.interface_id = 1424 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid)); 1425 } 1426 if (alt_path->hop_limit <= 1) { 1427 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, 1428 be16_to_cpu( 1429 alt_ext ? 0 : 1430 htons(ntohl(sa_path_get_slid( 1431 alt_path))))); 1432 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, 1433 be16_to_cpu( 1434 alt_ext ? 0 : 1435 htons(ntohl(sa_path_get_dlid( 1436 alt_path))))); 1437 } else { 1438 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, 1439 be16_to_cpu(IB_LID_PERMISSIVE)); 1440 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, 1441 be16_to_cpu(IB_LID_PERMISSIVE)); 1442 } 1443 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg, 1444 be32_to_cpu(alt_path->flow_label)); 1445 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate); 1446 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg, 1447 alt_path->traffic_class); 1448 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg, 1449 alt_path->hop_limit); 1450 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl); 1451 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg, 1452 (alt_path->hop_limit <= 1)); 1453 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg, 1454 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, 1455 alt_path->packet_life_time)); 1456 } 1457 1458 if (param->private_data && param->private_data_len) 1459 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data, 1460 param->private_data_len); 1461 } 1462 1463 static int cm_validate_req_param(struct ib_cm_req_param *param) 1464 { 1465 if (!param->primary_path) 1466 return -EINVAL; 1467 1468 if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC && 1469 param->qp_type != IB_QPT_XRC_INI) 1470 return -EINVAL; 1471 1472 if (param->private_data && 1473 param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE) 1474 return -EINVAL; 1475 1476 if (param->alternate_path && 1477 (param->alternate_path->pkey != param->primary_path->pkey || 1478 param->alternate_path->mtu != param->primary_path->mtu)) 1479 return -EINVAL; 1480 1481 return 0; 1482 } 1483 1484 int ib_send_cm_req(struct ib_cm_id *cm_id, 1485 struct ib_cm_req_param *param) 1486 { 1487 struct cm_id_private *cm_id_priv; 1488 struct cm_req_msg *req_msg; 1489 unsigned long flags; 1490 int ret; 1491 1492 ret = cm_validate_req_param(param); 1493 if (ret) 1494 return ret; 1495 1496 /* Verify that we're not in timewait. */ 1497 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 1498 spin_lock_irqsave(&cm_id_priv->lock, flags); 1499 if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) { 1500 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1501 ret = -EINVAL; 1502 goto out; 1503 } 1504 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1505 1506 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv-> 1507 id.local_id); 1508 if (IS_ERR(cm_id_priv->timewait_info)) { 1509 ret = PTR_ERR(cm_id_priv->timewait_info); 1510 goto out; 1511 } 1512 1513 ret = cm_init_av_by_path(param->primary_path, 1514 param->ppath_sgid_attr, &cm_id_priv->av, 1515 cm_id_priv); 1516 if (ret) 1517 goto out; 1518 if (param->alternate_path) { 1519 ret = cm_init_av_by_path(param->alternate_path, NULL, 1520 &cm_id_priv->alt_av, cm_id_priv); 1521 if (ret) 1522 goto out; 1523 } 1524 cm_id->service_id = param->service_id; 1525 cm_id->service_mask = ~cpu_to_be64(0); 1526 cm_id_priv->timeout_ms = cm_convert_to_ms( 1527 param->primary_path->packet_life_time) * 2 + 1528 cm_convert_to_ms( 1529 param->remote_cm_response_timeout); 1530 cm_id_priv->max_cm_retries = param->max_cm_retries; 1531 cm_id_priv->initiator_depth = param->initiator_depth; 1532 cm_id_priv->responder_resources = param->responder_resources; 1533 cm_id_priv->retry_count = param->retry_count; 1534 cm_id_priv->path_mtu = param->primary_path->mtu; 1535 cm_id_priv->pkey = param->primary_path->pkey; 1536 cm_id_priv->qp_type = param->qp_type; 1537 1538 ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg); 1539 if (ret) 1540 goto out; 1541 1542 req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad; 1543 cm_format_req(req_msg, cm_id_priv, param); 1544 cm_id_priv->tid = req_msg->hdr.tid; 1545 cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms; 1546 cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT; 1547 1548 cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); 1549 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); 1550 1551 spin_lock_irqsave(&cm_id_priv->lock, flags); 1552 ret = ib_post_send_mad(cm_id_priv->msg, NULL); 1553 if (ret) { 1554 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1555 goto error2; 1556 } 1557 BUG_ON(cm_id->state != IB_CM_IDLE); 1558 cm_id->state = IB_CM_REQ_SENT; 1559 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 1560 return 0; 1561 1562 error2: cm_free_msg(cm_id_priv->msg); 1563 out: return ret; 1564 } 1565 EXPORT_SYMBOL(ib_send_cm_req); 1566 1567 static int cm_issue_rej(struct cm_port *port, 1568 struct ib_mad_recv_wc *mad_recv_wc, 1569 enum ib_cm_rej_reason reason, 1570 enum cm_msg_response msg_rejected, 1571 void *ari, u8 ari_length) 1572 { 1573 struct ib_mad_send_buf *msg = NULL; 1574 struct cm_rej_msg *rej_msg, *rcv_msg; 1575 int ret; 1576 1577 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg); 1578 if (ret) 1579 return ret; 1580 1581 /* We just need common CM header information. Cast to any message. */ 1582 rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad; 1583 rej_msg = (struct cm_rej_msg *) msg->mad; 1584 1585 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid); 1586 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, 1587 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg)); 1588 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1589 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg)); 1590 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected); 1591 IBA_SET(CM_REJ_REASON, rej_msg, reason); 1592 1593 if (ari && ari_length) { 1594 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); 1595 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); 1596 } 1597 1598 ret = ib_post_send_mad(msg, NULL); 1599 if (ret) 1600 cm_free_msg(msg); 1601 1602 return ret; 1603 } 1604 1605 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg) 1606 { 1607 return ((cpu_to_be16( 1608 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) || 1609 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, 1610 req_msg)))); 1611 } 1612 1613 static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num, 1614 struct sa_path_rec *path, union ib_gid *gid) 1615 { 1616 if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num)) 1617 path->rec_type = SA_PATH_REC_TYPE_OPA; 1618 else 1619 path->rec_type = SA_PATH_REC_TYPE_IB; 1620 } 1621 1622 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, 1623 struct sa_path_rec *primary_path, 1624 struct sa_path_rec *alt_path) 1625 { 1626 u32 lid; 1627 1628 if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) { 1629 sa_path_set_dlid(primary_path, 1630 IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, 1631 req_msg)); 1632 sa_path_set_slid(primary_path, 1633 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, 1634 req_msg)); 1635 } else { 1636 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1637 CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)); 1638 sa_path_set_dlid(primary_path, lid); 1639 1640 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1641 CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)); 1642 sa_path_set_slid(primary_path, lid); 1643 } 1644 1645 if (!cm_req_has_alt_path(req_msg)) 1646 return; 1647 1648 if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) { 1649 sa_path_set_dlid(alt_path, 1650 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, 1651 req_msg)); 1652 sa_path_set_slid(alt_path, 1653 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, 1654 req_msg)); 1655 } else { 1656 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1657 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg)); 1658 sa_path_set_dlid(alt_path, lid); 1659 1660 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 1661 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg)); 1662 sa_path_set_slid(alt_path, lid); 1663 } 1664 } 1665 1666 static void cm_format_paths_from_req(struct cm_req_msg *req_msg, 1667 struct sa_path_rec *primary_path, 1668 struct sa_path_rec *alt_path) 1669 { 1670 primary_path->dgid = 1671 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg); 1672 primary_path->sgid = 1673 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg); 1674 primary_path->flow_label = 1675 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg)); 1676 primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg); 1677 primary_path->traffic_class = 1678 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg); 1679 primary_path->reversible = 1; 1680 primary_path->pkey = 1681 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); 1682 primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg); 1683 primary_path->mtu_selector = IB_SA_EQ; 1684 primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); 1685 primary_path->rate_selector = IB_SA_EQ; 1686 primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg); 1687 primary_path->packet_life_time_selector = IB_SA_EQ; 1688 primary_path->packet_life_time = 1689 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg); 1690 primary_path->packet_life_time -= (primary_path->packet_life_time > 0); 1691 primary_path->service_id = 1692 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); 1693 if (sa_path_is_roce(primary_path)) 1694 primary_path->roce.route_resolved = false; 1695 1696 if (cm_req_has_alt_path(req_msg)) { 1697 alt_path->dgid = *IBA_GET_MEM_PTR( 1698 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg); 1699 alt_path->sgid = *IBA_GET_MEM_PTR( 1700 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg); 1701 alt_path->flow_label = cpu_to_be32( 1702 IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg)); 1703 alt_path->hop_limit = 1704 IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg); 1705 alt_path->traffic_class = 1706 IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg); 1707 alt_path->reversible = 1; 1708 alt_path->pkey = 1709 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); 1710 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg); 1711 alt_path->mtu_selector = IB_SA_EQ; 1712 alt_path->mtu = 1713 IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); 1714 alt_path->rate_selector = IB_SA_EQ; 1715 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg); 1716 alt_path->packet_life_time_selector = IB_SA_EQ; 1717 alt_path->packet_life_time = 1718 IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg); 1719 alt_path->packet_life_time -= (alt_path->packet_life_time > 0); 1720 alt_path->service_id = 1721 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); 1722 1723 if (sa_path_is_roce(alt_path)) 1724 alt_path->roce.route_resolved = false; 1725 } 1726 cm_format_path_lid_from_req(req_msg, primary_path, alt_path); 1727 } 1728 1729 static u16 cm_get_bth_pkey(struct cm_work *work) 1730 { 1731 struct ib_device *ib_dev = work->port->cm_dev->ib_device; 1732 u8 port_num = work->port->port_num; 1733 u16 pkey_index = work->mad_recv_wc->wc->pkey_index; 1734 u16 pkey; 1735 int ret; 1736 1737 ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey); 1738 if (ret) { 1739 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n", 1740 port_num, pkey_index, ret); 1741 return 0; 1742 } 1743 1744 return pkey; 1745 } 1746 1747 /** 1748 * Convert OPA SGID to IB SGID 1749 * ULPs (such as IPoIB) do not understand OPA GIDs and will 1750 * reject them as the local_gid will not match the sgid. Therefore, 1751 * change the pathrec's SGID to an IB SGID. 1752 * 1753 * @work: Work completion 1754 * @path: Path record 1755 */ 1756 static void cm_opa_to_ib_sgid(struct cm_work *work, 1757 struct sa_path_rec *path) 1758 { 1759 struct ib_device *dev = work->port->cm_dev->ib_device; 1760 u8 port_num = work->port->port_num; 1761 1762 if (rdma_cap_opa_ah(dev, port_num) && 1763 (ib_is_opa_gid(&path->sgid))) { 1764 union ib_gid sgid; 1765 1766 if (rdma_query_gid(dev, port_num, 0, &sgid)) { 1767 dev_warn(&dev->dev, 1768 "Error updating sgid in CM request\n"); 1769 return; 1770 } 1771 1772 path->sgid = sgid; 1773 } 1774 } 1775 1776 static void cm_format_req_event(struct cm_work *work, 1777 struct cm_id_private *cm_id_priv, 1778 struct ib_cm_id *listen_id) 1779 { 1780 struct cm_req_msg *req_msg; 1781 struct ib_cm_req_event_param *param; 1782 1783 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; 1784 param = &work->cm_event.param.req_rcvd; 1785 param->listen_id = listen_id; 1786 param->bth_pkey = cm_get_bth_pkey(work); 1787 param->port = cm_id_priv->av.port->port_num; 1788 param->primary_path = &work->path[0]; 1789 cm_opa_to_ib_sgid(work, param->primary_path); 1790 if (cm_req_has_alt_path(req_msg)) { 1791 param->alternate_path = &work->path[1]; 1792 cm_opa_to_ib_sgid(work, param->alternate_path); 1793 } else { 1794 param->alternate_path = NULL; 1795 } 1796 param->remote_ca_guid = 1797 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); 1798 param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg); 1799 param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg); 1800 param->qp_type = cm_req_get_qp_type(req_msg); 1801 param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg); 1802 param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); 1803 param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); 1804 param->local_cm_response_timeout = 1805 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg); 1806 param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg); 1807 param->remote_cm_response_timeout = 1808 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg); 1809 param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); 1810 param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); 1811 param->srq = IBA_GET(CM_REQ_SRQ, req_msg); 1812 param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; 1813 work->cm_event.private_data = 1814 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg); 1815 } 1816 1817 static void cm_process_work(struct cm_id_private *cm_id_priv, 1818 struct cm_work *work) 1819 { 1820 int ret; 1821 1822 /* We will typically only have the current event to report. */ 1823 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event); 1824 cm_free_work(work); 1825 1826 while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) { 1827 spin_lock_irq(&cm_id_priv->lock); 1828 work = cm_dequeue_work(cm_id_priv); 1829 spin_unlock_irq(&cm_id_priv->lock); 1830 if (!work) 1831 return; 1832 1833 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, 1834 &work->cm_event); 1835 cm_free_work(work); 1836 } 1837 cm_deref_id(cm_id_priv); 1838 if (ret) 1839 cm_destroy_id(&cm_id_priv->id, ret); 1840 } 1841 1842 static void cm_format_mra(struct cm_mra_msg *mra_msg, 1843 struct cm_id_private *cm_id_priv, 1844 enum cm_msg_response msg_mraed, u8 service_timeout, 1845 const void *private_data, u8 private_data_len) 1846 { 1847 cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid); 1848 IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed); 1849 IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg, 1850 be32_to_cpu(cm_id_priv->id.local_id)); 1851 IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg, 1852 be32_to_cpu(cm_id_priv->id.remote_id)); 1853 IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout); 1854 1855 if (private_data && private_data_len) 1856 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data, 1857 private_data_len); 1858 } 1859 1860 static void cm_format_rej(struct cm_rej_msg *rej_msg, 1861 struct cm_id_private *cm_id_priv, 1862 enum ib_cm_rej_reason reason, 1863 void *ari, 1864 u8 ari_length, 1865 const void *private_data, 1866 u8 private_data_len) 1867 { 1868 lockdep_assert_held(&cm_id_priv->lock); 1869 1870 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid); 1871 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, 1872 be32_to_cpu(cm_id_priv->id.remote_id)); 1873 1874 switch(cm_id_priv->id.state) { 1875 case IB_CM_REQ_RCVD: 1876 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0)); 1877 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); 1878 break; 1879 case IB_CM_MRA_REQ_SENT: 1880 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1881 be32_to_cpu(cm_id_priv->id.local_id)); 1882 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); 1883 break; 1884 case IB_CM_REP_RCVD: 1885 case IB_CM_MRA_REP_SENT: 1886 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1887 be32_to_cpu(cm_id_priv->id.local_id)); 1888 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP); 1889 break; 1890 default: 1891 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, 1892 be32_to_cpu(cm_id_priv->id.local_id)); 1893 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, 1894 CM_MSG_RESPONSE_OTHER); 1895 break; 1896 } 1897 1898 IBA_SET(CM_REJ_REASON, rej_msg, reason); 1899 if (ari && ari_length) { 1900 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); 1901 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); 1902 } 1903 1904 if (private_data && private_data_len) 1905 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data, 1906 private_data_len); 1907 } 1908 1909 static void cm_dup_req_handler(struct cm_work *work, 1910 struct cm_id_private *cm_id_priv) 1911 { 1912 struct ib_mad_send_buf *msg = NULL; 1913 int ret; 1914 1915 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 1916 counter[CM_REQ_COUNTER]); 1917 1918 /* Quick state check to discard duplicate REQs. */ 1919 spin_lock_irq(&cm_id_priv->lock); 1920 if (cm_id_priv->id.state == IB_CM_REQ_RCVD) { 1921 spin_unlock_irq(&cm_id_priv->lock); 1922 return; 1923 } 1924 spin_unlock_irq(&cm_id_priv->lock); 1925 1926 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg); 1927 if (ret) 1928 return; 1929 1930 spin_lock_irq(&cm_id_priv->lock); 1931 switch (cm_id_priv->id.state) { 1932 case IB_CM_MRA_REQ_SENT: 1933 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 1934 CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout, 1935 cm_id_priv->private_data, 1936 cm_id_priv->private_data_len); 1937 break; 1938 case IB_CM_TIMEWAIT: 1939 cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv, 1940 IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0); 1941 break; 1942 default: 1943 goto unlock; 1944 } 1945 spin_unlock_irq(&cm_id_priv->lock); 1946 1947 ret = ib_post_send_mad(msg, NULL); 1948 if (ret) 1949 goto free; 1950 return; 1951 1952 unlock: spin_unlock_irq(&cm_id_priv->lock); 1953 free: cm_free_msg(msg); 1954 } 1955 1956 static struct cm_id_private * cm_match_req(struct cm_work *work, 1957 struct cm_id_private *cm_id_priv) 1958 { 1959 struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv; 1960 struct cm_timewait_info *timewait_info; 1961 struct cm_req_msg *req_msg; 1962 1963 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; 1964 1965 /* Check for possible duplicate REQ. */ 1966 spin_lock_irq(&cm.lock); 1967 timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info); 1968 if (timewait_info) { 1969 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 1970 timewait_info->work.remote_id); 1971 spin_unlock_irq(&cm.lock); 1972 if (cur_cm_id_priv) { 1973 cm_dup_req_handler(work, cur_cm_id_priv); 1974 cm_deref_id(cur_cm_id_priv); 1975 } 1976 return NULL; 1977 } 1978 1979 /* Check for stale connections. */ 1980 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info); 1981 if (timewait_info) { 1982 cm_remove_remote(cm_id_priv); 1983 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 1984 timewait_info->work.remote_id); 1985 1986 spin_unlock_irq(&cm.lock); 1987 cm_issue_rej(work->port, work->mad_recv_wc, 1988 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ, 1989 NULL, 0); 1990 if (cur_cm_id_priv) { 1991 ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0); 1992 cm_deref_id(cur_cm_id_priv); 1993 } 1994 return NULL; 1995 } 1996 1997 /* Find matching listen request. */ 1998 listen_cm_id_priv = cm_find_listen( 1999 cm_id_priv->id.device, 2000 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg))); 2001 if (!listen_cm_id_priv) { 2002 cm_remove_remote(cm_id_priv); 2003 spin_unlock_irq(&cm.lock); 2004 cm_issue_rej(work->port, work->mad_recv_wc, 2005 IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ, 2006 NULL, 0); 2007 return NULL; 2008 } 2009 spin_unlock_irq(&cm.lock); 2010 return listen_cm_id_priv; 2011 } 2012 2013 /* 2014 * Work-around for inter-subnet connections. If the LIDs are permissive, 2015 * we need to override the LID/SL data in the REQ with the LID information 2016 * in the work completion. 2017 */ 2018 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc) 2019 { 2020 if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) { 2021 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, 2022 req_msg)) == IB_LID_PERMISSIVE) { 2023 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, 2024 be16_to_cpu(ib_lid_be16(wc->slid))); 2025 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl); 2026 } 2027 2028 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, 2029 req_msg)) == IB_LID_PERMISSIVE) 2030 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, 2031 wc->dlid_path_bits); 2032 } 2033 2034 if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) { 2035 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, 2036 req_msg)) == IB_LID_PERMISSIVE) { 2037 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, 2038 be16_to_cpu(ib_lid_be16(wc->slid))); 2039 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl); 2040 } 2041 2042 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, 2043 req_msg)) == IB_LID_PERMISSIVE) 2044 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, 2045 wc->dlid_path_bits); 2046 } 2047 } 2048 2049 static int cm_req_handler(struct cm_work *work) 2050 { 2051 struct cm_id_private *cm_id_priv, *listen_cm_id_priv; 2052 struct cm_req_msg *req_msg; 2053 const struct ib_global_route *grh; 2054 const struct ib_gid_attr *gid_attr; 2055 int ret; 2056 2057 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad; 2058 2059 cm_id_priv = 2060 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL); 2061 if (IS_ERR(cm_id_priv)) 2062 return PTR_ERR(cm_id_priv); 2063 2064 cm_id_priv->id.remote_id = 2065 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg)); 2066 cm_id_priv->id.service_id = 2067 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); 2068 cm_id_priv->id.service_mask = ~cpu_to_be64(0); 2069 cm_id_priv->tid = req_msg->hdr.tid; 2070 cm_id_priv->timeout_ms = cm_convert_to_ms( 2071 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg)); 2072 cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg); 2073 cm_id_priv->remote_qpn = 2074 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); 2075 cm_id_priv->initiator_depth = 2076 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); 2077 cm_id_priv->responder_resources = 2078 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); 2079 cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); 2080 cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); 2081 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); 2082 cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); 2083 cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); 2084 cm_id_priv->qp_type = cm_req_get_qp_type(req_msg); 2085 2086 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, 2087 work->mad_recv_wc->recv_buf.grh, 2088 &cm_id_priv->av); 2089 if (ret) 2090 goto destroy; 2091 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv-> 2092 id.local_id); 2093 if (IS_ERR(cm_id_priv->timewait_info)) { 2094 ret = PTR_ERR(cm_id_priv->timewait_info); 2095 goto destroy; 2096 } 2097 cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id; 2098 cm_id_priv->timewait_info->remote_ca_guid = 2099 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); 2100 cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn; 2101 2102 /* 2103 * Note that the ID pointer is not in the xarray at this point, 2104 * so this set is only visible to the local thread. 2105 */ 2106 cm_id_priv->id.state = IB_CM_REQ_RCVD; 2107 2108 listen_cm_id_priv = cm_match_req(work, cm_id_priv); 2109 if (!listen_cm_id_priv) { 2110 pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__, 2111 be32_to_cpu(cm_id_priv->id.local_id)); 2112 cm_id_priv->id.state = IB_CM_IDLE; 2113 ret = -EINVAL; 2114 goto destroy; 2115 } 2116 2117 cm_process_routed_req(req_msg, work->mad_recv_wc->wc); 2118 2119 memset(&work->path[0], 0, sizeof(work->path[0])); 2120 if (cm_req_has_alt_path(req_msg)) 2121 memset(&work->path[1], 0, sizeof(work->path[1])); 2122 grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr); 2123 gid_attr = grh->sgid_attr; 2124 2125 if (gid_attr && 2126 rdma_protocol_roce(work->port->cm_dev->ib_device, 2127 work->port->port_num)) { 2128 work->path[0].rec_type = 2129 sa_conv_gid_to_pathrec_type(gid_attr->gid_type); 2130 } else { 2131 cm_path_set_rec_type( 2132 work->port->cm_dev->ib_device, work->port->port_num, 2133 &work->path[0], 2134 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, 2135 req_msg)); 2136 } 2137 if (cm_req_has_alt_path(req_msg)) 2138 work->path[1].rec_type = work->path[0].rec_type; 2139 cm_format_paths_from_req(req_msg, &work->path[0], 2140 &work->path[1]); 2141 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) 2142 sa_path_set_dmac(&work->path[0], 2143 cm_id_priv->av.ah_attr.roce.dmac); 2144 work->path[0].hop_limit = grh->hop_limit; 2145 ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av, 2146 cm_id_priv); 2147 if (ret) { 2148 int err; 2149 2150 err = rdma_query_gid(work->port->cm_dev->ib_device, 2151 work->port->port_num, 0, 2152 &work->path[0].sgid); 2153 if (err) 2154 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID, 2155 NULL, 0, NULL, 0); 2156 else 2157 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID, 2158 &work->path[0].sgid, 2159 sizeof(work->path[0].sgid), 2160 NULL, 0); 2161 goto rejected; 2162 } 2163 if (cm_req_has_alt_path(req_msg)) { 2164 ret = cm_init_av_by_path(&work->path[1], NULL, 2165 &cm_id_priv->alt_av, cm_id_priv); 2166 if (ret) { 2167 ib_send_cm_rej(&cm_id_priv->id, 2168 IB_CM_REJ_INVALID_ALT_GID, 2169 &work->path[0].sgid, 2170 sizeof(work->path[0].sgid), NULL, 0); 2171 goto rejected; 2172 } 2173 } 2174 2175 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; 2176 cm_id_priv->id.context = listen_cm_id_priv->id.context; 2177 cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id); 2178 2179 /* Now MAD handlers can see the new ID */ 2180 spin_lock_irq(&cm_id_priv->lock); 2181 cm_finalize_id(cm_id_priv); 2182 2183 /* Refcount belongs to the event, pairs with cm_process_work() */ 2184 refcount_inc(&cm_id_priv->refcount); 2185 cm_queue_work_unlock(cm_id_priv, work); 2186 /* 2187 * Since this ID was just created and was not made visible to other MAD 2188 * handlers until the cm_finalize_id() above we know that the 2189 * cm_process_work() will deliver the event and the listen_cm_id 2190 * embedded in the event can be derefed here. 2191 */ 2192 cm_deref_id(listen_cm_id_priv); 2193 return 0; 2194 2195 rejected: 2196 cm_deref_id(listen_cm_id_priv); 2197 destroy: 2198 ib_destroy_cm_id(&cm_id_priv->id); 2199 return ret; 2200 } 2201 2202 static void cm_format_rep(struct cm_rep_msg *rep_msg, 2203 struct cm_id_private *cm_id_priv, 2204 struct ib_cm_rep_param *param) 2205 { 2206 cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid); 2207 IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg, 2208 be32_to_cpu(cm_id_priv->id.local_id)); 2209 IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg, 2210 be32_to_cpu(cm_id_priv->id.remote_id)); 2211 IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn); 2212 IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg, 2213 param->responder_resources); 2214 IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg, 2215 cm_id_priv->av.port->cm_dev->ack_delay); 2216 IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted); 2217 IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count); 2218 IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg, 2219 be64_to_cpu(cm_id_priv->id.device->node_guid)); 2220 2221 if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) { 2222 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg, 2223 param->initiator_depth); 2224 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg, 2225 param->flow_control); 2226 IBA_SET(CM_REP_SRQ, rep_msg, param->srq); 2227 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num); 2228 } else { 2229 IBA_SET(CM_REP_SRQ, rep_msg, 1); 2230 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num); 2231 } 2232 2233 if (param->private_data && param->private_data_len) 2234 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data, 2235 param->private_data_len); 2236 } 2237 2238 int ib_send_cm_rep(struct ib_cm_id *cm_id, 2239 struct ib_cm_rep_param *param) 2240 { 2241 struct cm_id_private *cm_id_priv; 2242 struct ib_mad_send_buf *msg; 2243 struct cm_rep_msg *rep_msg; 2244 unsigned long flags; 2245 int ret; 2246 2247 if (param->private_data && 2248 param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE) 2249 return -EINVAL; 2250 2251 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 2252 spin_lock_irqsave(&cm_id_priv->lock, flags); 2253 if (cm_id->state != IB_CM_REQ_RCVD && 2254 cm_id->state != IB_CM_MRA_REQ_SENT) { 2255 pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__, 2256 be32_to_cpu(cm_id_priv->id.local_id), cm_id->state); 2257 ret = -EINVAL; 2258 goto out; 2259 } 2260 2261 ret = cm_alloc_msg(cm_id_priv, &msg); 2262 if (ret) 2263 goto out; 2264 2265 rep_msg = (struct cm_rep_msg *) msg->mad; 2266 cm_format_rep(rep_msg, cm_id_priv, param); 2267 msg->timeout_ms = cm_id_priv->timeout_ms; 2268 msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT; 2269 2270 ret = ib_post_send_mad(msg, NULL); 2271 if (ret) { 2272 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2273 cm_free_msg(msg); 2274 return ret; 2275 } 2276 2277 cm_id->state = IB_CM_REP_SENT; 2278 cm_id_priv->msg = msg; 2279 cm_id_priv->initiator_depth = param->initiator_depth; 2280 cm_id_priv->responder_resources = param->responder_resources; 2281 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); 2282 WARN_ONCE(param->qp_num & 0xFF000000, 2283 "IBTA declares QPN to be 24 bits, but it is 0x%X\n", 2284 param->qp_num); 2285 cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF); 2286 2287 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2288 return ret; 2289 } 2290 EXPORT_SYMBOL(ib_send_cm_rep); 2291 2292 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg, 2293 struct cm_id_private *cm_id_priv, 2294 const void *private_data, 2295 u8 private_data_len) 2296 { 2297 cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid); 2298 IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg, 2299 be32_to_cpu(cm_id_priv->id.local_id)); 2300 IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg, 2301 be32_to_cpu(cm_id_priv->id.remote_id)); 2302 2303 if (private_data && private_data_len) 2304 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data, 2305 private_data_len); 2306 } 2307 2308 int ib_send_cm_rtu(struct ib_cm_id *cm_id, 2309 const void *private_data, 2310 u8 private_data_len) 2311 { 2312 struct cm_id_private *cm_id_priv; 2313 struct ib_mad_send_buf *msg; 2314 unsigned long flags; 2315 void *data; 2316 int ret; 2317 2318 if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE) 2319 return -EINVAL; 2320 2321 data = cm_copy_private_data(private_data, private_data_len); 2322 if (IS_ERR(data)) 2323 return PTR_ERR(data); 2324 2325 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 2326 spin_lock_irqsave(&cm_id_priv->lock, flags); 2327 if (cm_id->state != IB_CM_REP_RCVD && 2328 cm_id->state != IB_CM_MRA_REP_SENT) { 2329 pr_debug("%s: local_id %d, cm_id->state %d\n", __func__, 2330 be32_to_cpu(cm_id->local_id), cm_id->state); 2331 ret = -EINVAL; 2332 goto error; 2333 } 2334 2335 ret = cm_alloc_msg(cm_id_priv, &msg); 2336 if (ret) 2337 goto error; 2338 2339 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv, 2340 private_data, private_data_len); 2341 2342 ret = ib_post_send_mad(msg, NULL); 2343 if (ret) { 2344 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2345 cm_free_msg(msg); 2346 kfree(data); 2347 return ret; 2348 } 2349 2350 cm_id->state = IB_CM_ESTABLISHED; 2351 cm_set_private_data(cm_id_priv, data, private_data_len); 2352 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2353 return 0; 2354 2355 error: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2356 kfree(data); 2357 return ret; 2358 } 2359 EXPORT_SYMBOL(ib_send_cm_rtu); 2360 2361 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) 2362 { 2363 struct cm_rep_msg *rep_msg; 2364 struct ib_cm_rep_event_param *param; 2365 2366 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; 2367 param = &work->cm_event.param.rep_rcvd; 2368 param->remote_ca_guid = 2369 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); 2370 param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg); 2371 param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type)); 2372 param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg); 2373 param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); 2374 param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); 2375 param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); 2376 param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg); 2377 param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg); 2378 param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); 2379 param->srq = IBA_GET(CM_REP_SRQ, rep_msg); 2380 work->cm_event.private_data = 2381 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg); 2382 } 2383 2384 static void cm_dup_rep_handler(struct cm_work *work) 2385 { 2386 struct cm_id_private *cm_id_priv; 2387 struct cm_rep_msg *rep_msg; 2388 struct ib_mad_send_buf *msg = NULL; 2389 int ret; 2390 2391 rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad; 2392 cm_id_priv = cm_acquire_id( 2393 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 2394 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg))); 2395 if (!cm_id_priv) 2396 return; 2397 2398 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2399 counter[CM_REP_COUNTER]); 2400 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg); 2401 if (ret) 2402 goto deref; 2403 2404 spin_lock_irq(&cm_id_priv->lock); 2405 if (cm_id_priv->id.state == IB_CM_ESTABLISHED) 2406 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv, 2407 cm_id_priv->private_data, 2408 cm_id_priv->private_data_len); 2409 else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT) 2410 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 2411 CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout, 2412 cm_id_priv->private_data, 2413 cm_id_priv->private_data_len); 2414 else 2415 goto unlock; 2416 spin_unlock_irq(&cm_id_priv->lock); 2417 2418 ret = ib_post_send_mad(msg, NULL); 2419 if (ret) 2420 goto free; 2421 goto deref; 2422 2423 unlock: spin_unlock_irq(&cm_id_priv->lock); 2424 free: cm_free_msg(msg); 2425 deref: cm_deref_id(cm_id_priv); 2426 } 2427 2428 static int cm_rep_handler(struct cm_work *work) 2429 { 2430 struct cm_id_private *cm_id_priv; 2431 struct cm_rep_msg *rep_msg; 2432 int ret; 2433 struct cm_id_private *cur_cm_id_priv; 2434 struct cm_timewait_info *timewait_info; 2435 2436 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; 2437 cm_id_priv = cm_acquire_id( 2438 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0); 2439 if (!cm_id_priv) { 2440 cm_dup_rep_handler(work); 2441 pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__, 2442 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2443 return -EINVAL; 2444 } 2445 2446 cm_format_rep_event(work, cm_id_priv->qp_type); 2447 2448 spin_lock_irq(&cm_id_priv->lock); 2449 switch (cm_id_priv->id.state) { 2450 case IB_CM_REQ_SENT: 2451 case IB_CM_MRA_REQ_RCVD: 2452 break; 2453 default: 2454 ret = -EINVAL; 2455 pr_debug( 2456 "%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", 2457 __func__, cm_id_priv->id.state, 2458 IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), 2459 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2460 spin_unlock_irq(&cm_id_priv->lock); 2461 goto error; 2462 } 2463 2464 cm_id_priv->timewait_info->work.remote_id = 2465 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); 2466 cm_id_priv->timewait_info->remote_ca_guid = 2467 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); 2468 cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); 2469 2470 spin_lock(&cm.lock); 2471 /* Check for duplicate REP. */ 2472 if (cm_insert_remote_id(cm_id_priv->timewait_info)) { 2473 spin_unlock(&cm.lock); 2474 spin_unlock_irq(&cm_id_priv->lock); 2475 ret = -EINVAL; 2476 pr_debug("%s: Failed to insert remote id %d\n", __func__, 2477 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2478 goto error; 2479 } 2480 /* Check for a stale connection. */ 2481 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info); 2482 if (timewait_info) { 2483 cm_remove_remote(cm_id_priv); 2484 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 2485 timewait_info->work.remote_id); 2486 2487 spin_unlock(&cm.lock); 2488 spin_unlock_irq(&cm_id_priv->lock); 2489 cm_issue_rej(work->port, work->mad_recv_wc, 2490 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP, 2491 NULL, 0); 2492 ret = -EINVAL; 2493 pr_debug( 2494 "%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", 2495 __func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), 2496 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); 2497 2498 if (cur_cm_id_priv) { 2499 ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0); 2500 cm_deref_id(cur_cm_id_priv); 2501 } 2502 2503 goto error; 2504 } 2505 spin_unlock(&cm.lock); 2506 2507 cm_id_priv->id.state = IB_CM_REP_RCVD; 2508 cm_id_priv->id.remote_id = 2509 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); 2510 cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); 2511 cm_id_priv->initiator_depth = 2512 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); 2513 cm_id_priv->responder_resources = 2514 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); 2515 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); 2516 cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); 2517 cm_id_priv->target_ack_delay = 2518 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); 2519 cm_id_priv->av.timeout = 2520 cm_ack_timeout(cm_id_priv->target_ack_delay, 2521 cm_id_priv->av.timeout - 1); 2522 cm_id_priv->alt_av.timeout = 2523 cm_ack_timeout(cm_id_priv->target_ack_delay, 2524 cm_id_priv->alt_av.timeout - 1); 2525 2526 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2527 cm_queue_work_unlock(cm_id_priv, work); 2528 return 0; 2529 2530 error: 2531 cm_deref_id(cm_id_priv); 2532 return ret; 2533 } 2534 2535 static int cm_establish_handler(struct cm_work *work) 2536 { 2537 struct cm_id_private *cm_id_priv; 2538 2539 /* See comment in cm_establish about lookup. */ 2540 cm_id_priv = cm_acquire_id(work->local_id, work->remote_id); 2541 if (!cm_id_priv) 2542 return -EINVAL; 2543 2544 spin_lock_irq(&cm_id_priv->lock); 2545 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) { 2546 spin_unlock_irq(&cm_id_priv->lock); 2547 goto out; 2548 } 2549 2550 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2551 cm_queue_work_unlock(cm_id_priv, work); 2552 return 0; 2553 out: 2554 cm_deref_id(cm_id_priv); 2555 return -EINVAL; 2556 } 2557 2558 static int cm_rtu_handler(struct cm_work *work) 2559 { 2560 struct cm_id_private *cm_id_priv; 2561 struct cm_rtu_msg *rtu_msg; 2562 2563 rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad; 2564 cm_id_priv = cm_acquire_id( 2565 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)), 2566 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg))); 2567 if (!cm_id_priv) 2568 return -EINVAL; 2569 2570 work->cm_event.private_data = 2571 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg); 2572 2573 spin_lock_irq(&cm_id_priv->lock); 2574 if (cm_id_priv->id.state != IB_CM_REP_SENT && 2575 cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) { 2576 spin_unlock_irq(&cm_id_priv->lock); 2577 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2578 counter[CM_RTU_COUNTER]); 2579 goto out; 2580 } 2581 cm_id_priv->id.state = IB_CM_ESTABLISHED; 2582 2583 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2584 cm_queue_work_unlock(cm_id_priv, work); 2585 return 0; 2586 out: 2587 cm_deref_id(cm_id_priv); 2588 return -EINVAL; 2589 } 2590 2591 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, 2592 struct cm_id_private *cm_id_priv, 2593 const void *private_data, 2594 u8 private_data_len) 2595 { 2596 cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID, 2597 cm_form_tid(cm_id_priv)); 2598 IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg, 2599 be32_to_cpu(cm_id_priv->id.local_id)); 2600 IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg, 2601 be32_to_cpu(cm_id_priv->id.remote_id)); 2602 IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg, 2603 be32_to_cpu(cm_id_priv->remote_qpn)); 2604 2605 if (private_data && private_data_len) 2606 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data, 2607 private_data_len); 2608 } 2609 2610 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv, 2611 const void *private_data, u8 private_data_len) 2612 { 2613 struct ib_mad_send_buf *msg; 2614 int ret; 2615 2616 lockdep_assert_held(&cm_id_priv->lock); 2617 2618 if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE) 2619 return -EINVAL; 2620 2621 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) { 2622 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, 2623 be32_to_cpu(cm_id_priv->id.local_id), 2624 cm_id_priv->id.state); 2625 return -EINVAL; 2626 } 2627 2628 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT || 2629 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD) 2630 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2631 2632 ret = cm_alloc_msg(cm_id_priv, &msg); 2633 if (ret) { 2634 cm_enter_timewait(cm_id_priv); 2635 return ret; 2636 } 2637 2638 cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv, 2639 private_data, private_data_len); 2640 msg->timeout_ms = cm_id_priv->timeout_ms; 2641 msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT; 2642 2643 ret = ib_post_send_mad(msg, NULL); 2644 if (ret) { 2645 cm_enter_timewait(cm_id_priv); 2646 cm_free_msg(msg); 2647 return ret; 2648 } 2649 2650 cm_id_priv->id.state = IB_CM_DREQ_SENT; 2651 cm_id_priv->msg = msg; 2652 return 0; 2653 } 2654 2655 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data, 2656 u8 private_data_len) 2657 { 2658 struct cm_id_private *cm_id_priv = 2659 container_of(cm_id, struct cm_id_private, id); 2660 unsigned long flags; 2661 int ret; 2662 2663 spin_lock_irqsave(&cm_id_priv->lock, flags); 2664 ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len); 2665 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2666 return ret; 2667 } 2668 EXPORT_SYMBOL(ib_send_cm_dreq); 2669 2670 static void cm_format_drep(struct cm_drep_msg *drep_msg, 2671 struct cm_id_private *cm_id_priv, 2672 const void *private_data, 2673 u8 private_data_len) 2674 { 2675 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid); 2676 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, 2677 be32_to_cpu(cm_id_priv->id.local_id)); 2678 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, 2679 be32_to_cpu(cm_id_priv->id.remote_id)); 2680 2681 if (private_data && private_data_len) 2682 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data, 2683 private_data_len); 2684 } 2685 2686 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv, 2687 void *private_data, u8 private_data_len) 2688 { 2689 struct ib_mad_send_buf *msg; 2690 int ret; 2691 2692 lockdep_assert_held(&cm_id_priv->lock); 2693 2694 if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE) 2695 return -EINVAL; 2696 2697 if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) { 2698 pr_debug( 2699 "%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n", 2700 __func__, be32_to_cpu(cm_id_priv->id.local_id), 2701 cm_id_priv->id.state); 2702 kfree(private_data); 2703 return -EINVAL; 2704 } 2705 2706 cm_set_private_data(cm_id_priv, private_data, private_data_len); 2707 cm_enter_timewait(cm_id_priv); 2708 2709 ret = cm_alloc_msg(cm_id_priv, &msg); 2710 if (ret) 2711 return ret; 2712 2713 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv, 2714 private_data, private_data_len); 2715 2716 ret = ib_post_send_mad(msg, NULL); 2717 if (ret) { 2718 cm_free_msg(msg); 2719 return ret; 2720 } 2721 return 0; 2722 } 2723 2724 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data, 2725 u8 private_data_len) 2726 { 2727 struct cm_id_private *cm_id_priv = 2728 container_of(cm_id, struct cm_id_private, id); 2729 unsigned long flags; 2730 void *data; 2731 int ret; 2732 2733 data = cm_copy_private_data(private_data, private_data_len); 2734 if (IS_ERR(data)) 2735 return PTR_ERR(data); 2736 2737 spin_lock_irqsave(&cm_id_priv->lock, flags); 2738 ret = cm_send_drep_locked(cm_id_priv, data, private_data_len); 2739 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2740 return ret; 2741 } 2742 EXPORT_SYMBOL(ib_send_cm_drep); 2743 2744 static int cm_issue_drep(struct cm_port *port, 2745 struct ib_mad_recv_wc *mad_recv_wc) 2746 { 2747 struct ib_mad_send_buf *msg = NULL; 2748 struct cm_dreq_msg *dreq_msg; 2749 struct cm_drep_msg *drep_msg; 2750 int ret; 2751 2752 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg); 2753 if (ret) 2754 return ret; 2755 2756 dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad; 2757 drep_msg = (struct cm_drep_msg *) msg->mad; 2758 2759 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid); 2760 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, 2761 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)); 2762 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, 2763 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); 2764 2765 ret = ib_post_send_mad(msg, NULL); 2766 if (ret) 2767 cm_free_msg(msg); 2768 2769 return ret; 2770 } 2771 2772 static int cm_dreq_handler(struct cm_work *work) 2773 { 2774 struct cm_id_private *cm_id_priv; 2775 struct cm_dreq_msg *dreq_msg; 2776 struct ib_mad_send_buf *msg = NULL; 2777 2778 dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad; 2779 cm_id_priv = cm_acquire_id( 2780 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)), 2781 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg))); 2782 if (!cm_id_priv) { 2783 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2784 counter[CM_DREQ_COUNTER]); 2785 cm_issue_drep(work->port, work->mad_recv_wc); 2786 pr_debug( 2787 "%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", 2788 __func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg), 2789 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); 2790 return -EINVAL; 2791 } 2792 2793 work->cm_event.private_data = 2794 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg); 2795 2796 spin_lock_irq(&cm_id_priv->lock); 2797 if (cm_id_priv->local_qpn != 2798 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg))) 2799 goto unlock; 2800 2801 switch (cm_id_priv->id.state) { 2802 case IB_CM_REP_SENT: 2803 case IB_CM_DREQ_SENT: 2804 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2805 break; 2806 case IB_CM_ESTABLISHED: 2807 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT || 2808 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD) 2809 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2810 break; 2811 case IB_CM_MRA_REP_RCVD: 2812 break; 2813 case IB_CM_TIMEWAIT: 2814 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2815 counter[CM_DREQ_COUNTER]); 2816 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc); 2817 if (IS_ERR(msg)) 2818 goto unlock; 2819 2820 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv, 2821 cm_id_priv->private_data, 2822 cm_id_priv->private_data_len); 2823 spin_unlock_irq(&cm_id_priv->lock); 2824 2825 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) || 2826 ib_post_send_mad(msg, NULL)) 2827 cm_free_msg(msg); 2828 goto deref; 2829 case IB_CM_DREQ_RCVD: 2830 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 2831 counter[CM_DREQ_COUNTER]); 2832 goto unlock; 2833 default: 2834 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 2835 __func__, be32_to_cpu(cm_id_priv->id.local_id), 2836 cm_id_priv->id.state); 2837 goto unlock; 2838 } 2839 cm_id_priv->id.state = IB_CM_DREQ_RCVD; 2840 cm_id_priv->tid = dreq_msg->hdr.tid; 2841 cm_queue_work_unlock(cm_id_priv, work); 2842 return 0; 2843 2844 unlock: spin_unlock_irq(&cm_id_priv->lock); 2845 deref: cm_deref_id(cm_id_priv); 2846 return -EINVAL; 2847 } 2848 2849 static int cm_drep_handler(struct cm_work *work) 2850 { 2851 struct cm_id_private *cm_id_priv; 2852 struct cm_drep_msg *drep_msg; 2853 2854 drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad; 2855 cm_id_priv = cm_acquire_id( 2856 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)), 2857 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg))); 2858 if (!cm_id_priv) 2859 return -EINVAL; 2860 2861 work->cm_event.private_data = 2862 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg); 2863 2864 spin_lock_irq(&cm_id_priv->lock); 2865 if (cm_id_priv->id.state != IB_CM_DREQ_SENT && 2866 cm_id_priv->id.state != IB_CM_DREQ_RCVD) { 2867 spin_unlock_irq(&cm_id_priv->lock); 2868 goto out; 2869 } 2870 cm_enter_timewait(cm_id_priv); 2871 2872 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 2873 cm_queue_work_unlock(cm_id_priv, work); 2874 return 0; 2875 out: 2876 cm_deref_id(cm_id_priv); 2877 return -EINVAL; 2878 } 2879 2880 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv, 2881 enum ib_cm_rej_reason reason, void *ari, 2882 u8 ari_length, const void *private_data, 2883 u8 private_data_len) 2884 { 2885 struct ib_mad_send_buf *msg; 2886 int ret; 2887 2888 lockdep_assert_held(&cm_id_priv->lock); 2889 2890 if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) || 2891 (ari && ari_length > IB_CM_REJ_ARI_LENGTH)) 2892 return -EINVAL; 2893 2894 switch (cm_id_priv->id.state) { 2895 case IB_CM_REQ_SENT: 2896 case IB_CM_MRA_REQ_RCVD: 2897 case IB_CM_REQ_RCVD: 2898 case IB_CM_MRA_REQ_SENT: 2899 case IB_CM_REP_RCVD: 2900 case IB_CM_MRA_REP_SENT: 2901 cm_reset_to_idle(cm_id_priv); 2902 ret = cm_alloc_msg(cm_id_priv, &msg); 2903 if (ret) 2904 return ret; 2905 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason, 2906 ari, ari_length, private_data, private_data_len); 2907 break; 2908 case IB_CM_REP_SENT: 2909 case IB_CM_MRA_REP_RCVD: 2910 cm_enter_timewait(cm_id_priv); 2911 ret = cm_alloc_msg(cm_id_priv, &msg); 2912 if (ret) 2913 return ret; 2914 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason, 2915 ari, ari_length, private_data, private_data_len); 2916 break; 2917 default: 2918 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, 2919 be32_to_cpu(cm_id_priv->id.local_id), 2920 cm_id_priv->id.state); 2921 return -EINVAL; 2922 } 2923 2924 ret = ib_post_send_mad(msg, NULL); 2925 if (ret) { 2926 cm_free_msg(msg); 2927 return ret; 2928 } 2929 2930 return 0; 2931 } 2932 2933 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason, 2934 void *ari, u8 ari_length, const void *private_data, 2935 u8 private_data_len) 2936 { 2937 struct cm_id_private *cm_id_priv = 2938 container_of(cm_id, struct cm_id_private, id); 2939 unsigned long flags; 2940 int ret; 2941 2942 spin_lock_irqsave(&cm_id_priv->lock, flags); 2943 ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length, 2944 private_data, private_data_len); 2945 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 2946 return ret; 2947 } 2948 EXPORT_SYMBOL(ib_send_cm_rej); 2949 2950 static void cm_format_rej_event(struct cm_work *work) 2951 { 2952 struct cm_rej_msg *rej_msg; 2953 struct ib_cm_rej_event_param *param; 2954 2955 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; 2956 param = &work->cm_event.param.rej_rcvd; 2957 param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg); 2958 param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg); 2959 param->reason = IBA_GET(CM_REJ_REASON, rej_msg); 2960 work->cm_event.private_data = 2961 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg); 2962 } 2963 2964 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) 2965 { 2966 struct cm_id_private *cm_id_priv; 2967 __be32 remote_id; 2968 2969 remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg)); 2970 2971 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) { 2972 cm_id_priv = cm_find_remote_id( 2973 *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)), 2974 remote_id); 2975 } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) == 2976 CM_MSG_RESPONSE_REQ) 2977 cm_id_priv = cm_acquire_id( 2978 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), 2979 0); 2980 else 2981 cm_id_priv = cm_acquire_id( 2982 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), 2983 remote_id); 2984 2985 return cm_id_priv; 2986 } 2987 2988 static int cm_rej_handler(struct cm_work *work) 2989 { 2990 struct cm_id_private *cm_id_priv; 2991 struct cm_rej_msg *rej_msg; 2992 2993 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; 2994 cm_id_priv = cm_acquire_rejected_id(rej_msg); 2995 if (!cm_id_priv) 2996 return -EINVAL; 2997 2998 cm_format_rej_event(work); 2999 3000 spin_lock_irq(&cm_id_priv->lock); 3001 switch (cm_id_priv->id.state) { 3002 case IB_CM_REQ_SENT: 3003 case IB_CM_MRA_REQ_RCVD: 3004 case IB_CM_REP_SENT: 3005 case IB_CM_MRA_REP_RCVD: 3006 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3007 /* fall through */ 3008 case IB_CM_REQ_RCVD: 3009 case IB_CM_MRA_REQ_SENT: 3010 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN) 3011 cm_enter_timewait(cm_id_priv); 3012 else 3013 cm_reset_to_idle(cm_id_priv); 3014 break; 3015 case IB_CM_DREQ_SENT: 3016 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3017 /* fall through */ 3018 case IB_CM_REP_RCVD: 3019 case IB_CM_MRA_REP_SENT: 3020 cm_enter_timewait(cm_id_priv); 3021 break; 3022 case IB_CM_ESTABLISHED: 3023 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT || 3024 cm_id_priv->id.lap_state == IB_CM_LAP_SENT) { 3025 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT) 3026 ib_cancel_mad(cm_id_priv->av.port->mad_agent, 3027 cm_id_priv->msg); 3028 cm_enter_timewait(cm_id_priv); 3029 break; 3030 } 3031 /* fall through */ 3032 default: 3033 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 3034 __func__, be32_to_cpu(cm_id_priv->id.local_id), 3035 cm_id_priv->id.state); 3036 spin_unlock_irq(&cm_id_priv->lock); 3037 goto out; 3038 } 3039 3040 cm_queue_work_unlock(cm_id_priv, work); 3041 return 0; 3042 out: 3043 cm_deref_id(cm_id_priv); 3044 return -EINVAL; 3045 } 3046 3047 int ib_send_cm_mra(struct ib_cm_id *cm_id, 3048 u8 service_timeout, 3049 const void *private_data, 3050 u8 private_data_len) 3051 { 3052 struct cm_id_private *cm_id_priv; 3053 struct ib_mad_send_buf *msg; 3054 enum ib_cm_state cm_state; 3055 enum ib_cm_lap_state lap_state; 3056 enum cm_msg_response msg_response; 3057 void *data; 3058 unsigned long flags; 3059 int ret; 3060 3061 if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE) 3062 return -EINVAL; 3063 3064 data = cm_copy_private_data(private_data, private_data_len); 3065 if (IS_ERR(data)) 3066 return PTR_ERR(data); 3067 3068 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3069 3070 spin_lock_irqsave(&cm_id_priv->lock, flags); 3071 switch(cm_id_priv->id.state) { 3072 case IB_CM_REQ_RCVD: 3073 cm_state = IB_CM_MRA_REQ_SENT; 3074 lap_state = cm_id->lap_state; 3075 msg_response = CM_MSG_RESPONSE_REQ; 3076 break; 3077 case IB_CM_REP_RCVD: 3078 cm_state = IB_CM_MRA_REP_SENT; 3079 lap_state = cm_id->lap_state; 3080 msg_response = CM_MSG_RESPONSE_REP; 3081 break; 3082 case IB_CM_ESTABLISHED: 3083 if (cm_id->lap_state == IB_CM_LAP_RCVD) { 3084 cm_state = cm_id->state; 3085 lap_state = IB_CM_MRA_LAP_SENT; 3086 msg_response = CM_MSG_RESPONSE_OTHER; 3087 break; 3088 } 3089 /* fall through */ 3090 default: 3091 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 3092 __func__, be32_to_cpu(cm_id_priv->id.local_id), 3093 cm_id_priv->id.state); 3094 ret = -EINVAL; 3095 goto error1; 3096 } 3097 3098 if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) { 3099 ret = cm_alloc_msg(cm_id_priv, &msg); 3100 if (ret) 3101 goto error1; 3102 3103 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 3104 msg_response, service_timeout, 3105 private_data, private_data_len); 3106 ret = ib_post_send_mad(msg, NULL); 3107 if (ret) 3108 goto error2; 3109 } 3110 3111 cm_id->state = cm_state; 3112 cm_id->lap_state = lap_state; 3113 cm_id_priv->service_timeout = service_timeout; 3114 cm_set_private_data(cm_id_priv, data, private_data_len); 3115 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3116 return 0; 3117 3118 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3119 kfree(data); 3120 return ret; 3121 3122 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3123 kfree(data); 3124 cm_free_msg(msg); 3125 return ret; 3126 } 3127 EXPORT_SYMBOL(ib_send_cm_mra); 3128 3129 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) 3130 { 3131 switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) { 3132 case CM_MSG_RESPONSE_REQ: 3133 return cm_acquire_id( 3134 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), 3135 0); 3136 case CM_MSG_RESPONSE_REP: 3137 case CM_MSG_RESPONSE_OTHER: 3138 return cm_acquire_id( 3139 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), 3140 cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg))); 3141 default: 3142 return NULL; 3143 } 3144 } 3145 3146 static int cm_mra_handler(struct cm_work *work) 3147 { 3148 struct cm_id_private *cm_id_priv; 3149 struct cm_mra_msg *mra_msg; 3150 int timeout; 3151 3152 mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad; 3153 cm_id_priv = cm_acquire_mraed_id(mra_msg); 3154 if (!cm_id_priv) 3155 return -EINVAL; 3156 3157 work->cm_event.private_data = 3158 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg); 3159 work->cm_event.param.mra_rcvd.service_timeout = 3160 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg); 3161 timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) + 3162 cm_convert_to_ms(cm_id_priv->av.timeout); 3163 3164 spin_lock_irq(&cm_id_priv->lock); 3165 switch (cm_id_priv->id.state) { 3166 case IB_CM_REQ_SENT: 3167 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != 3168 CM_MSG_RESPONSE_REQ || 3169 ib_modify_mad(cm_id_priv->av.port->mad_agent, 3170 cm_id_priv->msg, timeout)) 3171 goto out; 3172 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD; 3173 break; 3174 case IB_CM_REP_SENT: 3175 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != 3176 CM_MSG_RESPONSE_REP || 3177 ib_modify_mad(cm_id_priv->av.port->mad_agent, 3178 cm_id_priv->msg, timeout)) 3179 goto out; 3180 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD; 3181 break; 3182 case IB_CM_ESTABLISHED: 3183 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != 3184 CM_MSG_RESPONSE_OTHER || 3185 cm_id_priv->id.lap_state != IB_CM_LAP_SENT || 3186 ib_modify_mad(cm_id_priv->av.port->mad_agent, 3187 cm_id_priv->msg, timeout)) { 3188 if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD) 3189 atomic_long_inc(&work->port-> 3190 counter_group[CM_RECV_DUPLICATES]. 3191 counter[CM_MRA_COUNTER]); 3192 goto out; 3193 } 3194 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD; 3195 break; 3196 case IB_CM_MRA_REQ_RCVD: 3197 case IB_CM_MRA_REP_RCVD: 3198 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3199 counter[CM_MRA_COUNTER]); 3200 /* fall through */ 3201 default: 3202 pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n", 3203 __func__, be32_to_cpu(cm_id_priv->id.local_id), 3204 cm_id_priv->id.state); 3205 goto out; 3206 } 3207 3208 cm_id_priv->msg->context[1] = (void *) (unsigned long) 3209 cm_id_priv->id.state; 3210 cm_queue_work_unlock(cm_id_priv, work); 3211 return 0; 3212 out: 3213 spin_unlock_irq(&cm_id_priv->lock); 3214 cm_deref_id(cm_id_priv); 3215 return -EINVAL; 3216 } 3217 3218 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg, 3219 struct sa_path_rec *path) 3220 { 3221 u32 lid; 3222 3223 if (path->rec_type != SA_PATH_REC_TYPE_OPA) { 3224 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID, 3225 lap_msg)); 3226 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID, 3227 lap_msg)); 3228 } else { 3229 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 3230 CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg)); 3231 sa_path_set_dlid(path, lid); 3232 3233 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( 3234 CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg)); 3235 sa_path_set_slid(path, lid); 3236 } 3237 } 3238 3239 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, 3240 struct sa_path_rec *path, 3241 struct cm_lap_msg *lap_msg) 3242 { 3243 path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg); 3244 path->sgid = 3245 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg); 3246 path->flow_label = 3247 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg)); 3248 path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg); 3249 path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg); 3250 path->reversible = 1; 3251 path->pkey = cm_id_priv->pkey; 3252 path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg); 3253 path->mtu_selector = IB_SA_EQ; 3254 path->mtu = cm_id_priv->path_mtu; 3255 path->rate_selector = IB_SA_EQ; 3256 path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg); 3257 path->packet_life_time_selector = IB_SA_EQ; 3258 path->packet_life_time = 3259 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg); 3260 path->packet_life_time -= (path->packet_life_time > 0); 3261 cm_format_path_lid_from_lap(lap_msg, path); 3262 } 3263 3264 static int cm_lap_handler(struct cm_work *work) 3265 { 3266 struct cm_id_private *cm_id_priv; 3267 struct cm_lap_msg *lap_msg; 3268 struct ib_cm_lap_event_param *param; 3269 struct ib_mad_send_buf *msg = NULL; 3270 int ret; 3271 3272 /* Currently Alternate path messages are not supported for 3273 * RoCE link layer. 3274 */ 3275 if (rdma_protocol_roce(work->port->cm_dev->ib_device, 3276 work->port->port_num)) 3277 return -EINVAL; 3278 3279 /* todo: verify LAP request and send reject APR if invalid. */ 3280 lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad; 3281 cm_id_priv = cm_acquire_id( 3282 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)), 3283 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg))); 3284 if (!cm_id_priv) 3285 return -EINVAL; 3286 3287 param = &work->cm_event.param.lap_rcvd; 3288 memset(&work->path[0], 0, sizeof(work->path[1])); 3289 cm_path_set_rec_type(work->port->cm_dev->ib_device, 3290 work->port->port_num, &work->path[0], 3291 IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, 3292 lap_msg)); 3293 param->alternate_path = &work->path[0]; 3294 cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg); 3295 work->cm_event.private_data = 3296 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg); 3297 3298 spin_lock_irq(&cm_id_priv->lock); 3299 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) 3300 goto unlock; 3301 3302 switch (cm_id_priv->id.lap_state) { 3303 case IB_CM_LAP_UNINIT: 3304 case IB_CM_LAP_IDLE: 3305 break; 3306 case IB_CM_MRA_LAP_SENT: 3307 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3308 counter[CM_LAP_COUNTER]); 3309 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc); 3310 if (IS_ERR(msg)) 3311 goto unlock; 3312 3313 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv, 3314 CM_MSG_RESPONSE_OTHER, 3315 cm_id_priv->service_timeout, 3316 cm_id_priv->private_data, 3317 cm_id_priv->private_data_len); 3318 spin_unlock_irq(&cm_id_priv->lock); 3319 3320 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) || 3321 ib_post_send_mad(msg, NULL)) 3322 cm_free_msg(msg); 3323 goto deref; 3324 case IB_CM_LAP_RCVD: 3325 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3326 counter[CM_LAP_COUNTER]); 3327 goto unlock; 3328 default: 3329 goto unlock; 3330 } 3331 3332 ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc, 3333 work->mad_recv_wc->recv_buf.grh, 3334 &cm_id_priv->av); 3335 if (ret) 3336 goto unlock; 3337 3338 ret = cm_init_av_by_path(param->alternate_path, NULL, 3339 &cm_id_priv->alt_av, cm_id_priv); 3340 if (ret) 3341 goto unlock; 3342 3343 cm_id_priv->id.lap_state = IB_CM_LAP_RCVD; 3344 cm_id_priv->tid = lap_msg->hdr.tid; 3345 cm_queue_work_unlock(cm_id_priv, work); 3346 return 0; 3347 3348 unlock: spin_unlock_irq(&cm_id_priv->lock); 3349 deref: cm_deref_id(cm_id_priv); 3350 return -EINVAL; 3351 } 3352 3353 static int cm_apr_handler(struct cm_work *work) 3354 { 3355 struct cm_id_private *cm_id_priv; 3356 struct cm_apr_msg *apr_msg; 3357 3358 /* Currently Alternate path messages are not supported for 3359 * RoCE link layer. 3360 */ 3361 if (rdma_protocol_roce(work->port->cm_dev->ib_device, 3362 work->port->port_num)) 3363 return -EINVAL; 3364 3365 apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad; 3366 cm_id_priv = cm_acquire_id( 3367 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)), 3368 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg))); 3369 if (!cm_id_priv) 3370 return -EINVAL; /* Unmatched reply. */ 3371 3372 work->cm_event.param.apr_rcvd.ap_status = 3373 IBA_GET(CM_APR_AR_STATUS, apr_msg); 3374 work->cm_event.param.apr_rcvd.apr_info = 3375 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg); 3376 work->cm_event.param.apr_rcvd.info_len = 3377 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg); 3378 work->cm_event.private_data = 3379 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg); 3380 3381 spin_lock_irq(&cm_id_priv->lock); 3382 if (cm_id_priv->id.state != IB_CM_ESTABLISHED || 3383 (cm_id_priv->id.lap_state != IB_CM_LAP_SENT && 3384 cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) { 3385 spin_unlock_irq(&cm_id_priv->lock); 3386 goto out; 3387 } 3388 cm_id_priv->id.lap_state = IB_CM_LAP_IDLE; 3389 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3390 cm_id_priv->msg = NULL; 3391 cm_queue_work_unlock(cm_id_priv, work); 3392 return 0; 3393 out: 3394 cm_deref_id(cm_id_priv); 3395 return -EINVAL; 3396 } 3397 3398 static int cm_timewait_handler(struct cm_work *work) 3399 { 3400 struct cm_timewait_info *timewait_info; 3401 struct cm_id_private *cm_id_priv; 3402 3403 timewait_info = container_of(work, struct cm_timewait_info, work); 3404 spin_lock_irq(&cm.lock); 3405 list_del(&timewait_info->list); 3406 spin_unlock_irq(&cm.lock); 3407 3408 cm_id_priv = cm_acquire_id(timewait_info->work.local_id, 3409 timewait_info->work.remote_id); 3410 if (!cm_id_priv) 3411 return -EINVAL; 3412 3413 spin_lock_irq(&cm_id_priv->lock); 3414 if (cm_id_priv->id.state != IB_CM_TIMEWAIT || 3415 cm_id_priv->remote_qpn != timewait_info->remote_qpn) { 3416 spin_unlock_irq(&cm_id_priv->lock); 3417 goto out; 3418 } 3419 cm_id_priv->id.state = IB_CM_IDLE; 3420 cm_queue_work_unlock(cm_id_priv, work); 3421 return 0; 3422 out: 3423 cm_deref_id(cm_id_priv); 3424 return -EINVAL; 3425 } 3426 3427 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg, 3428 struct cm_id_private *cm_id_priv, 3429 struct ib_cm_sidr_req_param *param) 3430 { 3431 cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID, 3432 cm_form_tid(cm_id_priv)); 3433 IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg, 3434 be32_to_cpu(cm_id_priv->id.local_id)); 3435 IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg, 3436 be16_to_cpu(param->path->pkey)); 3437 IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg, 3438 be64_to_cpu(param->service_id)); 3439 3440 if (param->private_data && param->private_data_len) 3441 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg, 3442 param->private_data, param->private_data_len); 3443 } 3444 3445 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, 3446 struct ib_cm_sidr_req_param *param) 3447 { 3448 struct cm_id_private *cm_id_priv; 3449 struct ib_mad_send_buf *msg; 3450 unsigned long flags; 3451 int ret; 3452 3453 if (!param->path || (param->private_data && 3454 param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE)) 3455 return -EINVAL; 3456 3457 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3458 ret = cm_init_av_by_path(param->path, param->sgid_attr, 3459 &cm_id_priv->av, 3460 cm_id_priv); 3461 if (ret) 3462 goto out; 3463 3464 cm_id->service_id = param->service_id; 3465 cm_id->service_mask = ~cpu_to_be64(0); 3466 cm_id_priv->timeout_ms = param->timeout_ms; 3467 cm_id_priv->max_cm_retries = param->max_cm_retries; 3468 ret = cm_alloc_msg(cm_id_priv, &msg); 3469 if (ret) 3470 goto out; 3471 3472 cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv, 3473 param); 3474 msg->timeout_ms = cm_id_priv->timeout_ms; 3475 msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT; 3476 3477 spin_lock_irqsave(&cm_id_priv->lock, flags); 3478 if (cm_id->state == IB_CM_IDLE) 3479 ret = ib_post_send_mad(msg, NULL); 3480 else 3481 ret = -EINVAL; 3482 3483 if (ret) { 3484 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3485 cm_free_msg(msg); 3486 goto out; 3487 } 3488 cm_id->state = IB_CM_SIDR_REQ_SENT; 3489 cm_id_priv->msg = msg; 3490 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3491 out: 3492 return ret; 3493 } 3494 EXPORT_SYMBOL(ib_send_cm_sidr_req); 3495 3496 static void cm_format_sidr_req_event(struct cm_work *work, 3497 const struct cm_id_private *rx_cm_id, 3498 struct ib_cm_id *listen_id) 3499 { 3500 struct cm_sidr_req_msg *sidr_req_msg; 3501 struct ib_cm_sidr_req_event_param *param; 3502 3503 sidr_req_msg = (struct cm_sidr_req_msg *) 3504 work->mad_recv_wc->recv_buf.mad; 3505 param = &work->cm_event.param.sidr_req_rcvd; 3506 param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg); 3507 param->listen_id = listen_id; 3508 param->service_id = 3509 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); 3510 param->bth_pkey = cm_get_bth_pkey(work); 3511 param->port = work->port->port_num; 3512 param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr; 3513 work->cm_event.private_data = 3514 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg); 3515 } 3516 3517 static int cm_sidr_req_handler(struct cm_work *work) 3518 { 3519 struct cm_id_private *cm_id_priv, *listen_cm_id_priv; 3520 struct cm_sidr_req_msg *sidr_req_msg; 3521 struct ib_wc *wc; 3522 int ret; 3523 3524 cm_id_priv = 3525 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL); 3526 if (IS_ERR(cm_id_priv)) 3527 return PTR_ERR(cm_id_priv); 3528 3529 /* Record SGID/SLID and request ID for lookup. */ 3530 sidr_req_msg = (struct cm_sidr_req_msg *) 3531 work->mad_recv_wc->recv_buf.mad; 3532 3533 cm_id_priv->id.remote_id = 3534 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg)); 3535 cm_id_priv->id.service_id = 3536 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); 3537 cm_id_priv->id.service_mask = ~cpu_to_be64(0); 3538 cm_id_priv->tid = sidr_req_msg->hdr.tid; 3539 3540 wc = work->mad_recv_wc->wc; 3541 cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid); 3542 cm_id_priv->av.dgid.global.interface_id = 0; 3543 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, 3544 work->mad_recv_wc->recv_buf.grh, 3545 &cm_id_priv->av); 3546 if (ret) 3547 goto out; 3548 3549 spin_lock_irq(&cm.lock); 3550 listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv); 3551 if (listen_cm_id_priv) { 3552 spin_unlock_irq(&cm.lock); 3553 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. 3554 counter[CM_SIDR_REQ_COUNTER]); 3555 goto out; /* Duplicate message. */ 3556 } 3557 cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD; 3558 listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device, 3559 cm_id_priv->id.service_id); 3560 if (!listen_cm_id_priv) { 3561 spin_unlock_irq(&cm.lock); 3562 ib_send_cm_sidr_rep(&cm_id_priv->id, 3563 &(struct ib_cm_sidr_rep_param){ 3564 .status = IB_SIDR_UNSUPPORTED }); 3565 goto out; /* No match. */ 3566 } 3567 spin_unlock_irq(&cm.lock); 3568 3569 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; 3570 cm_id_priv->id.context = listen_cm_id_priv->id.context; 3571 3572 /* 3573 * A SIDR ID does not need to be in the xarray since it does not receive 3574 * mads, is not placed in the remote_id or remote_qpn rbtree, and does 3575 * not enter timewait. 3576 */ 3577 3578 cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id); 3579 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event); 3580 cm_free_work(work); 3581 /* 3582 * A pointer to the listen_cm_id is held in the event, so this deref 3583 * must be after the event is delivered above. 3584 */ 3585 cm_deref_id(listen_cm_id_priv); 3586 if (ret) 3587 cm_destroy_id(&cm_id_priv->id, ret); 3588 return 0; 3589 out: 3590 ib_destroy_cm_id(&cm_id_priv->id); 3591 return -EINVAL; 3592 } 3593 3594 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, 3595 struct cm_id_private *cm_id_priv, 3596 struct ib_cm_sidr_rep_param *param) 3597 { 3598 cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID, 3599 cm_id_priv->tid); 3600 IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg, 3601 be32_to_cpu(cm_id_priv->id.remote_id)); 3602 IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status); 3603 IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num); 3604 IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg, 3605 be64_to_cpu(cm_id_priv->id.service_id)); 3606 IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey); 3607 3608 if (param->info && param->info_length) 3609 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg, 3610 param->info, param->info_length); 3611 3612 if (param->private_data && param->private_data_len) 3613 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg, 3614 param->private_data, param->private_data_len); 3615 } 3616 3617 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv, 3618 struct ib_cm_sidr_rep_param *param) 3619 { 3620 struct ib_mad_send_buf *msg; 3621 int ret; 3622 3623 lockdep_assert_held(&cm_id_priv->lock); 3624 3625 if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) || 3626 (param->private_data && 3627 param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE)) 3628 return -EINVAL; 3629 3630 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD) 3631 return -EINVAL; 3632 3633 ret = cm_alloc_msg(cm_id_priv, &msg); 3634 if (ret) 3635 return ret; 3636 3637 cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv, 3638 param); 3639 ret = ib_post_send_mad(msg, NULL); 3640 if (ret) { 3641 cm_free_msg(msg); 3642 return ret; 3643 } 3644 cm_id_priv->id.state = IB_CM_IDLE; 3645 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) { 3646 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table); 3647 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node); 3648 } 3649 return 0; 3650 } 3651 3652 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id, 3653 struct ib_cm_sidr_rep_param *param) 3654 { 3655 struct cm_id_private *cm_id_priv = 3656 container_of(cm_id, struct cm_id_private, id); 3657 unsigned long flags; 3658 int ret; 3659 3660 spin_lock_irqsave(&cm_id_priv->lock, flags); 3661 ret = cm_send_sidr_rep_locked(cm_id_priv, param); 3662 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3663 return ret; 3664 } 3665 EXPORT_SYMBOL(ib_send_cm_sidr_rep); 3666 3667 static void cm_format_sidr_rep_event(struct cm_work *work, 3668 const struct cm_id_private *cm_id_priv) 3669 { 3670 struct cm_sidr_rep_msg *sidr_rep_msg; 3671 struct ib_cm_sidr_rep_event_param *param; 3672 3673 sidr_rep_msg = (struct cm_sidr_rep_msg *) 3674 work->mad_recv_wc->recv_buf.mad; 3675 param = &work->cm_event.param.sidr_rep_rcvd; 3676 param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg); 3677 param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg); 3678 param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg); 3679 param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION, 3680 sidr_rep_msg); 3681 param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH, 3682 sidr_rep_msg); 3683 param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; 3684 work->cm_event.private_data = 3685 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg); 3686 } 3687 3688 static int cm_sidr_rep_handler(struct cm_work *work) 3689 { 3690 struct cm_sidr_rep_msg *sidr_rep_msg; 3691 struct cm_id_private *cm_id_priv; 3692 3693 sidr_rep_msg = (struct cm_sidr_rep_msg *) 3694 work->mad_recv_wc->recv_buf.mad; 3695 cm_id_priv = cm_acquire_id( 3696 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0); 3697 if (!cm_id_priv) 3698 return -EINVAL; /* Unmatched reply. */ 3699 3700 spin_lock_irq(&cm_id_priv->lock); 3701 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) { 3702 spin_unlock_irq(&cm_id_priv->lock); 3703 goto out; 3704 } 3705 cm_id_priv->id.state = IB_CM_IDLE; 3706 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg); 3707 spin_unlock_irq(&cm_id_priv->lock); 3708 3709 cm_format_sidr_rep_event(work, cm_id_priv); 3710 cm_process_work(cm_id_priv, work); 3711 return 0; 3712 out: 3713 cm_deref_id(cm_id_priv); 3714 return -EINVAL; 3715 } 3716 3717 static void cm_process_send_error(struct ib_mad_send_buf *msg, 3718 enum ib_wc_status wc_status) 3719 { 3720 struct cm_id_private *cm_id_priv; 3721 struct ib_cm_event cm_event; 3722 enum ib_cm_state state; 3723 int ret; 3724 3725 memset(&cm_event, 0, sizeof cm_event); 3726 cm_id_priv = msg->context[0]; 3727 3728 /* Discard old sends or ones without a response. */ 3729 spin_lock_irq(&cm_id_priv->lock); 3730 state = (enum ib_cm_state) (unsigned long) msg->context[1]; 3731 if (msg != cm_id_priv->msg || state != cm_id_priv->id.state) 3732 goto discard; 3733 3734 pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n", 3735 state, ib_wc_status_msg(wc_status)); 3736 switch (state) { 3737 case IB_CM_REQ_SENT: 3738 case IB_CM_MRA_REQ_RCVD: 3739 cm_reset_to_idle(cm_id_priv); 3740 cm_event.event = IB_CM_REQ_ERROR; 3741 break; 3742 case IB_CM_REP_SENT: 3743 case IB_CM_MRA_REP_RCVD: 3744 cm_reset_to_idle(cm_id_priv); 3745 cm_event.event = IB_CM_REP_ERROR; 3746 break; 3747 case IB_CM_DREQ_SENT: 3748 cm_enter_timewait(cm_id_priv); 3749 cm_event.event = IB_CM_DREQ_ERROR; 3750 break; 3751 case IB_CM_SIDR_REQ_SENT: 3752 cm_id_priv->id.state = IB_CM_IDLE; 3753 cm_event.event = IB_CM_SIDR_REQ_ERROR; 3754 break; 3755 default: 3756 goto discard; 3757 } 3758 spin_unlock_irq(&cm_id_priv->lock); 3759 cm_event.param.send_status = wc_status; 3760 3761 /* No other events can occur on the cm_id at this point. */ 3762 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event); 3763 cm_free_msg(msg); 3764 if (ret) 3765 ib_destroy_cm_id(&cm_id_priv->id); 3766 return; 3767 discard: 3768 spin_unlock_irq(&cm_id_priv->lock); 3769 cm_free_msg(msg); 3770 } 3771 3772 static void cm_send_handler(struct ib_mad_agent *mad_agent, 3773 struct ib_mad_send_wc *mad_send_wc) 3774 { 3775 struct ib_mad_send_buf *msg = mad_send_wc->send_buf; 3776 struct cm_port *port; 3777 u16 attr_index; 3778 3779 port = mad_agent->context; 3780 attr_index = be16_to_cpu(((struct ib_mad_hdr *) 3781 msg->mad)->attr_id) - CM_ATTR_ID_OFFSET; 3782 3783 /* 3784 * If the send was in response to a received message (context[0] is not 3785 * set to a cm_id), and is not a REJ, then it is a send that was 3786 * manually retried. 3787 */ 3788 if (!msg->context[0] && (attr_index != CM_REJ_COUNTER)) 3789 msg->retries = 1; 3790 3791 atomic_long_add(1 + msg->retries, 3792 &port->counter_group[CM_XMIT].counter[attr_index]); 3793 if (msg->retries) 3794 atomic_long_add(msg->retries, 3795 &port->counter_group[CM_XMIT_RETRIES]. 3796 counter[attr_index]); 3797 3798 switch (mad_send_wc->status) { 3799 case IB_WC_SUCCESS: 3800 case IB_WC_WR_FLUSH_ERR: 3801 cm_free_msg(msg); 3802 break; 3803 default: 3804 if (msg->context[0] && msg->context[1]) 3805 cm_process_send_error(msg, mad_send_wc->status); 3806 else 3807 cm_free_msg(msg); 3808 break; 3809 } 3810 } 3811 3812 static void cm_work_handler(struct work_struct *_work) 3813 { 3814 struct cm_work *work = container_of(_work, struct cm_work, work.work); 3815 int ret; 3816 3817 switch (work->cm_event.event) { 3818 case IB_CM_REQ_RECEIVED: 3819 ret = cm_req_handler(work); 3820 break; 3821 case IB_CM_MRA_RECEIVED: 3822 ret = cm_mra_handler(work); 3823 break; 3824 case IB_CM_REJ_RECEIVED: 3825 ret = cm_rej_handler(work); 3826 break; 3827 case IB_CM_REP_RECEIVED: 3828 ret = cm_rep_handler(work); 3829 break; 3830 case IB_CM_RTU_RECEIVED: 3831 ret = cm_rtu_handler(work); 3832 break; 3833 case IB_CM_USER_ESTABLISHED: 3834 ret = cm_establish_handler(work); 3835 break; 3836 case IB_CM_DREQ_RECEIVED: 3837 ret = cm_dreq_handler(work); 3838 break; 3839 case IB_CM_DREP_RECEIVED: 3840 ret = cm_drep_handler(work); 3841 break; 3842 case IB_CM_SIDR_REQ_RECEIVED: 3843 ret = cm_sidr_req_handler(work); 3844 break; 3845 case IB_CM_SIDR_REP_RECEIVED: 3846 ret = cm_sidr_rep_handler(work); 3847 break; 3848 case IB_CM_LAP_RECEIVED: 3849 ret = cm_lap_handler(work); 3850 break; 3851 case IB_CM_APR_RECEIVED: 3852 ret = cm_apr_handler(work); 3853 break; 3854 case IB_CM_TIMEWAIT_EXIT: 3855 ret = cm_timewait_handler(work); 3856 break; 3857 default: 3858 pr_debug("cm_event.event: 0x%x\n", work->cm_event.event); 3859 ret = -EINVAL; 3860 break; 3861 } 3862 if (ret) 3863 cm_free_work(work); 3864 } 3865 3866 static int cm_establish(struct ib_cm_id *cm_id) 3867 { 3868 struct cm_id_private *cm_id_priv; 3869 struct cm_work *work; 3870 unsigned long flags; 3871 int ret = 0; 3872 struct cm_device *cm_dev; 3873 3874 cm_dev = ib_get_client_data(cm_id->device, &cm_client); 3875 if (!cm_dev) 3876 return -ENODEV; 3877 3878 work = kmalloc(sizeof *work, GFP_ATOMIC); 3879 if (!work) 3880 return -ENOMEM; 3881 3882 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3883 spin_lock_irqsave(&cm_id_priv->lock, flags); 3884 switch (cm_id->state) 3885 { 3886 case IB_CM_REP_SENT: 3887 case IB_CM_MRA_REP_RCVD: 3888 cm_id->state = IB_CM_ESTABLISHED; 3889 break; 3890 case IB_CM_ESTABLISHED: 3891 ret = -EISCONN; 3892 break; 3893 default: 3894 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__, 3895 be32_to_cpu(cm_id->local_id), cm_id->state); 3896 ret = -EINVAL; 3897 break; 3898 } 3899 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3900 3901 if (ret) { 3902 kfree(work); 3903 goto out; 3904 } 3905 3906 /* 3907 * The CM worker thread may try to destroy the cm_id before it 3908 * can execute this work item. To prevent potential deadlock, 3909 * we need to find the cm_id once we're in the context of the 3910 * worker thread, rather than holding a reference on it. 3911 */ 3912 INIT_DELAYED_WORK(&work->work, cm_work_handler); 3913 work->local_id = cm_id->local_id; 3914 work->remote_id = cm_id->remote_id; 3915 work->mad_recv_wc = NULL; 3916 work->cm_event.event = IB_CM_USER_ESTABLISHED; 3917 3918 /* Check if the device started its remove_one */ 3919 spin_lock_irqsave(&cm.lock, flags); 3920 if (!cm_dev->going_down) { 3921 queue_delayed_work(cm.wq, &work->work, 0); 3922 } else { 3923 kfree(work); 3924 ret = -ENODEV; 3925 } 3926 spin_unlock_irqrestore(&cm.lock, flags); 3927 3928 out: 3929 return ret; 3930 } 3931 3932 static int cm_migrate(struct ib_cm_id *cm_id) 3933 { 3934 struct cm_id_private *cm_id_priv; 3935 struct cm_av tmp_av; 3936 unsigned long flags; 3937 int tmp_send_port_not_ready; 3938 int ret = 0; 3939 3940 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 3941 spin_lock_irqsave(&cm_id_priv->lock, flags); 3942 if (cm_id->state == IB_CM_ESTABLISHED && 3943 (cm_id->lap_state == IB_CM_LAP_UNINIT || 3944 cm_id->lap_state == IB_CM_LAP_IDLE)) { 3945 cm_id->lap_state = IB_CM_LAP_IDLE; 3946 /* Swap address vector */ 3947 tmp_av = cm_id_priv->av; 3948 cm_id_priv->av = cm_id_priv->alt_av; 3949 cm_id_priv->alt_av = tmp_av; 3950 /* Swap port send ready state */ 3951 tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready; 3952 cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready; 3953 cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready; 3954 } else 3955 ret = -EINVAL; 3956 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 3957 3958 return ret; 3959 } 3960 3961 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event) 3962 { 3963 int ret; 3964 3965 switch (event) { 3966 case IB_EVENT_COMM_EST: 3967 ret = cm_establish(cm_id); 3968 break; 3969 case IB_EVENT_PATH_MIG: 3970 ret = cm_migrate(cm_id); 3971 break; 3972 default: 3973 ret = -EINVAL; 3974 } 3975 return ret; 3976 } 3977 EXPORT_SYMBOL(ib_cm_notify); 3978 3979 static void cm_recv_handler(struct ib_mad_agent *mad_agent, 3980 struct ib_mad_send_buf *send_buf, 3981 struct ib_mad_recv_wc *mad_recv_wc) 3982 { 3983 struct cm_port *port = mad_agent->context; 3984 struct cm_work *work; 3985 enum ib_cm_event_type event; 3986 bool alt_path = false; 3987 u16 attr_id; 3988 int paths = 0; 3989 int going_down = 0; 3990 3991 switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) { 3992 case CM_REQ_ATTR_ID: 3993 alt_path = cm_req_has_alt_path((struct cm_req_msg *) 3994 mad_recv_wc->recv_buf.mad); 3995 paths = 1 + (alt_path != 0); 3996 event = IB_CM_REQ_RECEIVED; 3997 break; 3998 case CM_MRA_ATTR_ID: 3999 event = IB_CM_MRA_RECEIVED; 4000 break; 4001 case CM_REJ_ATTR_ID: 4002 event = IB_CM_REJ_RECEIVED; 4003 break; 4004 case CM_REP_ATTR_ID: 4005 event = IB_CM_REP_RECEIVED; 4006 break; 4007 case CM_RTU_ATTR_ID: 4008 event = IB_CM_RTU_RECEIVED; 4009 break; 4010 case CM_DREQ_ATTR_ID: 4011 event = IB_CM_DREQ_RECEIVED; 4012 break; 4013 case CM_DREP_ATTR_ID: 4014 event = IB_CM_DREP_RECEIVED; 4015 break; 4016 case CM_SIDR_REQ_ATTR_ID: 4017 event = IB_CM_SIDR_REQ_RECEIVED; 4018 break; 4019 case CM_SIDR_REP_ATTR_ID: 4020 event = IB_CM_SIDR_REP_RECEIVED; 4021 break; 4022 case CM_LAP_ATTR_ID: 4023 paths = 1; 4024 event = IB_CM_LAP_RECEIVED; 4025 break; 4026 case CM_APR_ATTR_ID: 4027 event = IB_CM_APR_RECEIVED; 4028 break; 4029 default: 4030 ib_free_recv_mad(mad_recv_wc); 4031 return; 4032 } 4033 4034 attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id); 4035 atomic_long_inc(&port->counter_group[CM_RECV]. 4036 counter[attr_id - CM_ATTR_ID_OFFSET]); 4037 4038 work = kmalloc(struct_size(work, path, paths), GFP_KERNEL); 4039 if (!work) { 4040 ib_free_recv_mad(mad_recv_wc); 4041 return; 4042 } 4043 4044 INIT_DELAYED_WORK(&work->work, cm_work_handler); 4045 work->cm_event.event = event; 4046 work->mad_recv_wc = mad_recv_wc; 4047 work->port = port; 4048 4049 /* Check if the device started its remove_one */ 4050 spin_lock_irq(&cm.lock); 4051 if (!port->cm_dev->going_down) 4052 queue_delayed_work(cm.wq, &work->work, 0); 4053 else 4054 going_down = 1; 4055 spin_unlock_irq(&cm.lock); 4056 4057 if (going_down) { 4058 kfree(work); 4059 ib_free_recv_mad(mad_recv_wc); 4060 } 4061 } 4062 4063 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv, 4064 struct ib_qp_attr *qp_attr, 4065 int *qp_attr_mask) 4066 { 4067 unsigned long flags; 4068 int ret; 4069 4070 spin_lock_irqsave(&cm_id_priv->lock, flags); 4071 switch (cm_id_priv->id.state) { 4072 case IB_CM_REQ_SENT: 4073 case IB_CM_MRA_REQ_RCVD: 4074 case IB_CM_REQ_RCVD: 4075 case IB_CM_MRA_REQ_SENT: 4076 case IB_CM_REP_RCVD: 4077 case IB_CM_MRA_REP_SENT: 4078 case IB_CM_REP_SENT: 4079 case IB_CM_MRA_REP_RCVD: 4080 case IB_CM_ESTABLISHED: 4081 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS | 4082 IB_QP_PKEY_INDEX | IB_QP_PORT; 4083 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE; 4084 if (cm_id_priv->responder_resources) 4085 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ | 4086 IB_ACCESS_REMOTE_ATOMIC; 4087 qp_attr->pkey_index = cm_id_priv->av.pkey_index; 4088 qp_attr->port_num = cm_id_priv->av.port->port_num; 4089 ret = 0; 4090 break; 4091 default: 4092 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 4093 __func__, be32_to_cpu(cm_id_priv->id.local_id), 4094 cm_id_priv->id.state); 4095 ret = -EINVAL; 4096 break; 4097 } 4098 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 4099 return ret; 4100 } 4101 4102 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv, 4103 struct ib_qp_attr *qp_attr, 4104 int *qp_attr_mask) 4105 { 4106 unsigned long flags; 4107 int ret; 4108 4109 spin_lock_irqsave(&cm_id_priv->lock, flags); 4110 switch (cm_id_priv->id.state) { 4111 case IB_CM_REQ_RCVD: 4112 case IB_CM_MRA_REQ_SENT: 4113 case IB_CM_REP_RCVD: 4114 case IB_CM_MRA_REP_SENT: 4115 case IB_CM_REP_SENT: 4116 case IB_CM_MRA_REP_RCVD: 4117 case IB_CM_ESTABLISHED: 4118 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | 4119 IB_QP_DEST_QPN | IB_QP_RQ_PSN; 4120 qp_attr->ah_attr = cm_id_priv->av.ah_attr; 4121 qp_attr->path_mtu = cm_id_priv->path_mtu; 4122 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn); 4123 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn); 4124 if (cm_id_priv->qp_type == IB_QPT_RC || 4125 cm_id_priv->qp_type == IB_QPT_XRC_TGT) { 4126 *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC | 4127 IB_QP_MIN_RNR_TIMER; 4128 qp_attr->max_dest_rd_atomic = 4129 cm_id_priv->responder_resources; 4130 qp_attr->min_rnr_timer = 0; 4131 } 4132 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) { 4133 *qp_attr_mask |= IB_QP_ALT_PATH; 4134 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num; 4135 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index; 4136 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout; 4137 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr; 4138 } 4139 ret = 0; 4140 break; 4141 default: 4142 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 4143 __func__, be32_to_cpu(cm_id_priv->id.local_id), 4144 cm_id_priv->id.state); 4145 ret = -EINVAL; 4146 break; 4147 } 4148 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 4149 return ret; 4150 } 4151 4152 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv, 4153 struct ib_qp_attr *qp_attr, 4154 int *qp_attr_mask) 4155 { 4156 unsigned long flags; 4157 int ret; 4158 4159 spin_lock_irqsave(&cm_id_priv->lock, flags); 4160 switch (cm_id_priv->id.state) { 4161 /* Allow transition to RTS before sending REP */ 4162 case IB_CM_REQ_RCVD: 4163 case IB_CM_MRA_REQ_SENT: 4164 4165 case IB_CM_REP_RCVD: 4166 case IB_CM_MRA_REP_SENT: 4167 case IB_CM_REP_SENT: 4168 case IB_CM_MRA_REP_RCVD: 4169 case IB_CM_ESTABLISHED: 4170 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) { 4171 *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN; 4172 qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn); 4173 switch (cm_id_priv->qp_type) { 4174 case IB_QPT_RC: 4175 case IB_QPT_XRC_INI: 4176 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY | 4177 IB_QP_MAX_QP_RD_ATOMIC; 4178 qp_attr->retry_cnt = cm_id_priv->retry_count; 4179 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count; 4180 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth; 4181 /* fall through */ 4182 case IB_QPT_XRC_TGT: 4183 *qp_attr_mask |= IB_QP_TIMEOUT; 4184 qp_attr->timeout = cm_id_priv->av.timeout; 4185 break; 4186 default: 4187 break; 4188 } 4189 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) { 4190 *qp_attr_mask |= IB_QP_PATH_MIG_STATE; 4191 qp_attr->path_mig_state = IB_MIG_REARM; 4192 } 4193 } else { 4194 *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE; 4195 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num; 4196 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index; 4197 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout; 4198 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr; 4199 qp_attr->path_mig_state = IB_MIG_REARM; 4200 } 4201 ret = 0; 4202 break; 4203 default: 4204 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n", 4205 __func__, be32_to_cpu(cm_id_priv->id.local_id), 4206 cm_id_priv->id.state); 4207 ret = -EINVAL; 4208 break; 4209 } 4210 spin_unlock_irqrestore(&cm_id_priv->lock, flags); 4211 return ret; 4212 } 4213 4214 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id, 4215 struct ib_qp_attr *qp_attr, 4216 int *qp_attr_mask) 4217 { 4218 struct cm_id_private *cm_id_priv; 4219 int ret; 4220 4221 cm_id_priv = container_of(cm_id, struct cm_id_private, id); 4222 switch (qp_attr->qp_state) { 4223 case IB_QPS_INIT: 4224 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask); 4225 break; 4226 case IB_QPS_RTR: 4227 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask); 4228 break; 4229 case IB_QPS_RTS: 4230 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask); 4231 break; 4232 default: 4233 ret = -EINVAL; 4234 break; 4235 } 4236 return ret; 4237 } 4238 EXPORT_SYMBOL(ib_cm_init_qp_attr); 4239 4240 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr, 4241 char *buf) 4242 { 4243 struct cm_counter_group *group; 4244 struct cm_counter_attribute *cm_attr; 4245 4246 group = container_of(obj, struct cm_counter_group, obj); 4247 cm_attr = container_of(attr, struct cm_counter_attribute, attr); 4248 4249 return sprintf(buf, "%ld\n", 4250 atomic_long_read(&group->counter[cm_attr->index])); 4251 } 4252 4253 static const struct sysfs_ops cm_counter_ops = { 4254 .show = cm_show_counter 4255 }; 4256 4257 static struct kobj_type cm_counter_obj_type = { 4258 .sysfs_ops = &cm_counter_ops, 4259 .default_attrs = cm_counter_default_attrs 4260 }; 4261 4262 static char *cm_devnode(struct device *dev, umode_t *mode) 4263 { 4264 if (mode) 4265 *mode = 0666; 4266 return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev)); 4267 } 4268 4269 struct class cm_class = { 4270 .owner = THIS_MODULE, 4271 .name = "infiniband_cm", 4272 .devnode = cm_devnode, 4273 }; 4274 EXPORT_SYMBOL(cm_class); 4275 4276 static int cm_create_port_fs(struct cm_port *port) 4277 { 4278 int i, ret; 4279 4280 for (i = 0; i < CM_COUNTER_GROUPS; i++) { 4281 ret = ib_port_register_module_stat(port->cm_dev->ib_device, 4282 port->port_num, 4283 &port->counter_group[i].obj, 4284 &cm_counter_obj_type, 4285 counter_group_names[i]); 4286 if (ret) 4287 goto error; 4288 } 4289 4290 return 0; 4291 4292 error: 4293 while (i--) 4294 ib_port_unregister_module_stat(&port->counter_group[i].obj); 4295 return ret; 4296 4297 } 4298 4299 static void cm_remove_port_fs(struct cm_port *port) 4300 { 4301 int i; 4302 4303 for (i = 0; i < CM_COUNTER_GROUPS; i++) 4304 ib_port_unregister_module_stat(&port->counter_group[i].obj); 4305 4306 } 4307 4308 static int cm_add_one(struct ib_device *ib_device) 4309 { 4310 struct cm_device *cm_dev; 4311 struct cm_port *port; 4312 struct ib_mad_reg_req reg_req = { 4313 .mgmt_class = IB_MGMT_CLASS_CM, 4314 .mgmt_class_version = IB_CM_CLASS_VERSION, 4315 }; 4316 struct ib_port_modify port_modify = { 4317 .set_port_cap_mask = IB_PORT_CM_SUP 4318 }; 4319 unsigned long flags; 4320 int ret; 4321 int count = 0; 4322 u8 i; 4323 4324 cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt), 4325 GFP_KERNEL); 4326 if (!cm_dev) 4327 return -ENOMEM; 4328 4329 cm_dev->ib_device = ib_device; 4330 cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay; 4331 cm_dev->going_down = 0; 4332 4333 set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask); 4334 for (i = 1; i <= ib_device->phys_port_cnt; i++) { 4335 if (!rdma_cap_ib_cm(ib_device, i)) 4336 continue; 4337 4338 port = kzalloc(sizeof *port, GFP_KERNEL); 4339 if (!port) { 4340 ret = -ENOMEM; 4341 goto error1; 4342 } 4343 4344 cm_dev->port[i-1] = port; 4345 port->cm_dev = cm_dev; 4346 port->port_num = i; 4347 4348 INIT_LIST_HEAD(&port->cm_priv_prim_list); 4349 INIT_LIST_HEAD(&port->cm_priv_altr_list); 4350 4351 ret = cm_create_port_fs(port); 4352 if (ret) 4353 goto error1; 4354 4355 port->mad_agent = ib_register_mad_agent(ib_device, i, 4356 IB_QPT_GSI, 4357 ®_req, 4358 0, 4359 cm_send_handler, 4360 cm_recv_handler, 4361 port, 4362 0); 4363 if (IS_ERR(port->mad_agent)) { 4364 ret = PTR_ERR(port->mad_agent); 4365 goto error2; 4366 } 4367 4368 ret = ib_modify_port(ib_device, i, 0, &port_modify); 4369 if (ret) 4370 goto error3; 4371 4372 count++; 4373 } 4374 4375 if (!count) { 4376 ret = -EOPNOTSUPP; 4377 goto free; 4378 } 4379 4380 ib_set_client_data(ib_device, &cm_client, cm_dev); 4381 4382 write_lock_irqsave(&cm.device_lock, flags); 4383 list_add_tail(&cm_dev->list, &cm.device_list); 4384 write_unlock_irqrestore(&cm.device_lock, flags); 4385 return 0; 4386 4387 error3: 4388 ib_unregister_mad_agent(port->mad_agent); 4389 error2: 4390 cm_remove_port_fs(port); 4391 error1: 4392 port_modify.set_port_cap_mask = 0; 4393 port_modify.clr_port_cap_mask = IB_PORT_CM_SUP; 4394 kfree(port); 4395 while (--i) { 4396 if (!rdma_cap_ib_cm(ib_device, i)) 4397 continue; 4398 4399 port = cm_dev->port[i-1]; 4400 ib_modify_port(ib_device, port->port_num, 0, &port_modify); 4401 ib_unregister_mad_agent(port->mad_agent); 4402 cm_remove_port_fs(port); 4403 kfree(port); 4404 } 4405 free: 4406 kfree(cm_dev); 4407 return ret; 4408 } 4409 4410 static void cm_remove_one(struct ib_device *ib_device, void *client_data) 4411 { 4412 struct cm_device *cm_dev = client_data; 4413 struct cm_port *port; 4414 struct cm_id_private *cm_id_priv; 4415 struct ib_mad_agent *cur_mad_agent; 4416 struct ib_port_modify port_modify = { 4417 .clr_port_cap_mask = IB_PORT_CM_SUP 4418 }; 4419 unsigned long flags; 4420 int i; 4421 4422 write_lock_irqsave(&cm.device_lock, flags); 4423 list_del(&cm_dev->list); 4424 write_unlock_irqrestore(&cm.device_lock, flags); 4425 4426 spin_lock_irq(&cm.lock); 4427 cm_dev->going_down = 1; 4428 spin_unlock_irq(&cm.lock); 4429 4430 for (i = 1; i <= ib_device->phys_port_cnt; i++) { 4431 if (!rdma_cap_ib_cm(ib_device, i)) 4432 continue; 4433 4434 port = cm_dev->port[i-1]; 4435 ib_modify_port(ib_device, port->port_num, 0, &port_modify); 4436 /* Mark all the cm_id's as not valid */ 4437 spin_lock_irq(&cm.lock); 4438 list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list) 4439 cm_id_priv->altr_send_port_not_ready = 1; 4440 list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list) 4441 cm_id_priv->prim_send_port_not_ready = 1; 4442 spin_unlock_irq(&cm.lock); 4443 /* 4444 * We flush the queue here after the going_down set, this 4445 * verify that no new works will be queued in the recv handler, 4446 * after that we can call the unregister_mad_agent 4447 */ 4448 flush_workqueue(cm.wq); 4449 spin_lock_irq(&cm.state_lock); 4450 cur_mad_agent = port->mad_agent; 4451 port->mad_agent = NULL; 4452 spin_unlock_irq(&cm.state_lock); 4453 ib_unregister_mad_agent(cur_mad_agent); 4454 cm_remove_port_fs(port); 4455 kfree(port); 4456 } 4457 4458 kfree(cm_dev); 4459 } 4460 4461 static int __init ib_cm_init(void) 4462 { 4463 int ret; 4464 4465 INIT_LIST_HEAD(&cm.device_list); 4466 rwlock_init(&cm.device_lock); 4467 spin_lock_init(&cm.lock); 4468 spin_lock_init(&cm.state_lock); 4469 cm.listen_service_table = RB_ROOT; 4470 cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID); 4471 cm.remote_id_table = RB_ROOT; 4472 cm.remote_qp_table = RB_ROOT; 4473 cm.remote_sidr_table = RB_ROOT; 4474 xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); 4475 get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand); 4476 INIT_LIST_HEAD(&cm.timewait_list); 4477 4478 ret = class_register(&cm_class); 4479 if (ret) { 4480 ret = -ENOMEM; 4481 goto error1; 4482 } 4483 4484 cm.wq = alloc_workqueue("ib_cm", 0, 1); 4485 if (!cm.wq) { 4486 ret = -ENOMEM; 4487 goto error2; 4488 } 4489 4490 ret = ib_register_client(&cm_client); 4491 if (ret) 4492 goto error3; 4493 4494 return 0; 4495 error3: 4496 destroy_workqueue(cm.wq); 4497 error2: 4498 class_unregister(&cm_class); 4499 error1: 4500 return ret; 4501 } 4502 4503 static void __exit ib_cm_cleanup(void) 4504 { 4505 struct cm_timewait_info *timewait_info, *tmp; 4506 4507 spin_lock_irq(&cm.lock); 4508 list_for_each_entry(timewait_info, &cm.timewait_list, list) 4509 cancel_delayed_work(&timewait_info->work.work); 4510 spin_unlock_irq(&cm.lock); 4511 4512 ib_unregister_client(&cm_client); 4513 destroy_workqueue(cm.wq); 4514 4515 list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) { 4516 list_del(&timewait_info->list); 4517 kfree(timewait_info); 4518 } 4519 4520 class_unregister(&cm_class); 4521 WARN_ON(!xa_empty(&cm.local_id_table)); 4522 } 4523 4524 module_init(ib_cm_init); 4525 module_exit(ib_cm_cleanup); 4526