1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3 *
4 * Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
5 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
6 * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8 *
9 * This software is available to you under a choice of one of two
10 * licenses. You may choose to be licensed under the terms of the GNU
11 * General Public License (GPL) Version 2, available from the file
12 * COPYING in the main directory of this source tree, or the
13 * OpenIB.org BSD license below:
14 *
15 * Redistribution and use in source and binary forms, with or
16 * without modification, are permitted provided that the following
17 * conditions are met:
18 *
19 * - Redistributions of source code must retain the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer.
22 *
23 * - Redistributions in binary form must reproduce the above
24 * copyright notice, this list of conditions and the following
25 * disclaimer in the documentation and/or other materials
26 * provided with the distribution.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
29 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
30 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
31 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
32 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
33 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
34 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 * SOFTWARE.
36 */
37
38 #include <sys/cdefs.h>
39 #include <linux/completion.h>
40 #include <linux/dma-mapping.h>
41 #include <linux/device.h>
42 #include <linux/module.h>
43 #include <linux/err.h>
44 #include <linux/idr.h>
45 #include <linux/interrupt.h>
46 #include <linux/random.h>
47 #include <linux/rbtree.h>
48 #include <linux/spinlock.h>
49 #include <linux/slab.h>
50 #include <linux/sysfs.h>
51 #include <linux/workqueue.h>
52 #include <linux/kdev_t.h>
53 #include <linux/etherdevice.h>
54
55 #include <asm/atomic-long.h>
56
57 #include <rdma/ib_addr.h>
58 #include <rdma/ib_cache.h>
59 #include <rdma/ib_cm.h>
60 #include "cm_msgs.h"
61 #include "core_priv.h"
62
63 MODULE_AUTHOR("Sean Hefty");
64 MODULE_DESCRIPTION("InfiniBand CM");
65 MODULE_LICENSE("Dual BSD/GPL");
66
67 static const char * const ibcm_rej_reason_strs[] = {
68 [IB_CM_REJ_NO_QP] = "no QP",
69 [IB_CM_REJ_NO_EEC] = "no EEC",
70 [IB_CM_REJ_NO_RESOURCES] = "no resources",
71 [IB_CM_REJ_TIMEOUT] = "timeout",
72 [IB_CM_REJ_UNSUPPORTED] = "unsupported",
73 [IB_CM_REJ_INVALID_COMM_ID] = "invalid comm ID",
74 [IB_CM_REJ_INVALID_COMM_INSTANCE] = "invalid comm instance",
75 [IB_CM_REJ_INVALID_SERVICE_ID] = "invalid service ID",
76 [IB_CM_REJ_INVALID_TRANSPORT_TYPE] = "invalid transport type",
77 [IB_CM_REJ_STALE_CONN] = "stale conn",
78 [IB_CM_REJ_RDC_NOT_EXIST] = "RDC not exist",
79 [IB_CM_REJ_INVALID_GID] = "invalid GID",
80 [IB_CM_REJ_INVALID_LID] = "invalid LID",
81 [IB_CM_REJ_INVALID_SL] = "invalid SL",
82 [IB_CM_REJ_INVALID_TRAFFIC_CLASS] = "invalid traffic class",
83 [IB_CM_REJ_INVALID_HOP_LIMIT] = "invalid hop limit",
84 [IB_CM_REJ_INVALID_PACKET_RATE] = "invalid packet rate",
85 [IB_CM_REJ_INVALID_ALT_GID] = "invalid alt GID",
86 [IB_CM_REJ_INVALID_ALT_LID] = "invalid alt LID",
87 [IB_CM_REJ_INVALID_ALT_SL] = "invalid alt SL",
88 [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS] = "invalid alt traffic class",
89 [IB_CM_REJ_INVALID_ALT_HOP_LIMIT] = "invalid alt hop limit",
90 [IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate",
91 [IB_CM_REJ_PORT_CM_REDIRECT] = "port CM redirect",
92 [IB_CM_REJ_PORT_REDIRECT] = "port redirect",
93 [IB_CM_REJ_INVALID_MTU] = "invalid MTU",
94 [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
95 [IB_CM_REJ_CONSUMER_DEFINED] = "consumer defined",
96 [IB_CM_REJ_INVALID_RNR_RETRY] = "invalid RNR retry",
97 [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID",
98 [IB_CM_REJ_INVALID_CLASS_VERSION] = "invalid class version",
99 [IB_CM_REJ_INVALID_FLOW_LABEL] = "invalid flow label",
100 [IB_CM_REJ_INVALID_ALT_FLOW_LABEL] = "invalid alt flow label",
101 };
102
ibcm_reject_msg(int reason)103 const char *__attribute_const__ ibcm_reject_msg(int reason)
104 {
105 size_t index = reason;
106
107 if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
108 ibcm_rej_reason_strs[index])
109 return ibcm_rej_reason_strs[index];
110 else
111 return "unrecognized reason";
112 }
113 EXPORT_SYMBOL(ibcm_reject_msg);
114
115 static void cm_add_one(struct ib_device *device);
116 static void cm_remove_one(struct ib_device *device, void *client_data);
117
118 static struct ib_client cm_client = {
119 .name = "cm",
120 .add = cm_add_one,
121 .remove = cm_remove_one
122 };
123
124 static struct ib_cm {
125 spinlock_t lock;
126 struct list_head device_list;
127 rwlock_t device_lock;
128 struct rb_root listen_service_table;
129 u64 listen_service_id;
130 /* struct rb_root peer_service_table; todo: fix peer to peer */
131 struct rb_root remote_qp_table;
132 struct rb_root remote_id_table;
133 struct rb_root remote_sidr_table;
134 struct idr local_id_table;
135 __be32 random_id_operand;
136 struct list_head timewait_list;
137 struct workqueue_struct *wq;
138 /* Sync on cm change port state */
139 spinlock_t state_lock;
140 } cm;
141
142 /* Counter indexes ordered by attribute ID */
143 enum {
144 CM_REQ_COUNTER,
145 CM_MRA_COUNTER,
146 CM_REJ_COUNTER,
147 CM_REP_COUNTER,
148 CM_RTU_COUNTER,
149 CM_DREQ_COUNTER,
150 CM_DREP_COUNTER,
151 CM_SIDR_REQ_COUNTER,
152 CM_SIDR_REP_COUNTER,
153 CM_LAP_COUNTER,
154 CM_APR_COUNTER,
155 CM_ATTR_COUNT,
156 CM_ATTR_ID_OFFSET = 0x0010,
157 };
158
159 enum {
160 CM_XMIT,
161 CM_XMIT_RETRIES,
162 CM_RECV,
163 CM_RECV_DUPLICATES,
164 CM_COUNTER_GROUPS
165 };
166
167 static char const counter_group_names[CM_COUNTER_GROUPS]
168 [sizeof("cm_rx_duplicates")] = {
169 "cm_tx_msgs", "cm_tx_retries",
170 "cm_rx_msgs", "cm_rx_duplicates"
171 };
172
173 struct cm_counter_group {
174 struct kobject obj;
175 atomic_long_t counter[CM_ATTR_COUNT];
176 };
177
178 struct cm_counter_attribute {
179 struct attribute attr;
180 int index;
181 };
182
183 #define CM_COUNTER_ATTR(_name, _index) \
184 struct cm_counter_attribute cm_##_name##_counter_attr = { \
185 .attr = { .name = __stringify(_name), .mode = 0444 }, \
186 .index = _index \
187 }
188
189 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
190 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
191 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
192 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
193 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
194 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
195 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
196 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
197 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
198 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
199 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
200
201 static struct attribute *cm_counter_default_attrs[] = {
202 &cm_req_counter_attr.attr,
203 &cm_mra_counter_attr.attr,
204 &cm_rej_counter_attr.attr,
205 &cm_rep_counter_attr.attr,
206 &cm_rtu_counter_attr.attr,
207 &cm_dreq_counter_attr.attr,
208 &cm_drep_counter_attr.attr,
209 &cm_sidr_req_counter_attr.attr,
210 &cm_sidr_rep_counter_attr.attr,
211 &cm_lap_counter_attr.attr,
212 &cm_apr_counter_attr.attr,
213 NULL
214 };
215
216 struct cm_port {
217 struct cm_device *cm_dev;
218 struct ib_mad_agent *mad_agent;
219 struct kobject port_obj;
220 u8 port_num;
221 struct list_head cm_priv_prim_list;
222 struct list_head cm_priv_altr_list;
223 struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
224 };
225
226 struct cm_device {
227 struct list_head list;
228 struct ib_device *ib_device;
229 u8 ack_delay;
230 int going_down;
231 struct cm_port *port[0];
232 };
233
234 struct cm_av {
235 struct cm_port *port;
236 union ib_gid dgid;
237 struct rdma_ah_attr ah_attr;
238 u16 pkey_index;
239 u8 timeout;
240 };
241
242 struct cm_work {
243 struct delayed_work work;
244 struct list_head list;
245 struct cm_port *port;
246 struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
247 __be32 local_id; /* Established / timewait */
248 __be32 remote_id;
249 struct ib_cm_event cm_event;
250 struct sa_path_rec path[0];
251 };
252
253 struct cm_timewait_info {
254 struct cm_work work; /* Must be first. */
255 struct list_head list;
256 struct rb_node remote_qp_node;
257 struct rb_node remote_id_node;
258 __be64 remote_ca_guid;
259 __be32 remote_qpn;
260 u8 inserted_remote_qp;
261 u8 inserted_remote_id;
262 };
263
264 struct cm_id_private {
265 struct ib_cm_id id;
266
267 struct rb_node service_node;
268 struct rb_node sidr_id_node;
269 spinlock_t lock; /* Do not acquire inside cm.lock */
270 struct completion comp;
271 atomic_t refcount;
272 /* Number of clients sharing this ib_cm_id. Only valid for listeners.
273 * Protected by the cm.lock spinlock. */
274 int listen_sharecount;
275
276 struct ib_mad_send_buf *msg;
277 struct cm_timewait_info *timewait_info;
278 /* todo: use alternate port on send failure */
279 struct cm_av av;
280 struct cm_av alt_av;
281
282 void *private_data;
283 __be64 tid;
284 __be32 local_qpn;
285 __be32 remote_qpn;
286 enum ib_qp_type qp_type;
287 __be32 sq_psn;
288 __be32 rq_psn;
289 int timeout_ms;
290 enum ib_mtu path_mtu;
291 __be16 pkey;
292 u8 private_data_len;
293 u8 max_cm_retries;
294 u8 peer_to_peer;
295 u8 responder_resources;
296 u8 initiator_depth;
297 u8 retry_count;
298 u8 rnr_retry_count;
299 u8 service_timeout;
300 u8 target_ack_delay;
301
302 struct list_head prim_list;
303 struct list_head altr_list;
304 /* Indicates that the send port mad is registered and av is set */
305 int prim_send_port_not_ready;
306 int altr_send_port_not_ready;
307
308 struct list_head work_list;
309 atomic_t work_count;
310 };
311
312 static void cm_work_handler(struct work_struct *work);
313
cm_deref_id(struct cm_id_private * cm_id_priv)314 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
315 {
316 if (atomic_dec_and_test(&cm_id_priv->refcount))
317 complete(&cm_id_priv->comp);
318 }
319
cm_alloc_msg(struct cm_id_private * cm_id_priv,struct ib_mad_send_buf ** msg)320 static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
321 struct ib_mad_send_buf **msg)
322 {
323 struct ib_mad_agent *mad_agent;
324 struct ib_mad_send_buf *m;
325 struct ib_ah *ah;
326 struct cm_av *av;
327 unsigned long flags, flags2;
328 int ret = 0;
329
330 /* don't let the port to be released till the agent is down */
331 spin_lock_irqsave(&cm.state_lock, flags2);
332 spin_lock_irqsave(&cm.lock, flags);
333 if (!cm_id_priv->prim_send_port_not_ready)
334 av = &cm_id_priv->av;
335 else if (!cm_id_priv->altr_send_port_not_ready &&
336 (cm_id_priv->alt_av.port))
337 av = &cm_id_priv->alt_av;
338 else {
339 pr_info("%s: not valid CM id\n", __func__);
340 ret = -ENODEV;
341 spin_unlock_irqrestore(&cm.lock, flags);
342 goto out;
343 }
344 spin_unlock_irqrestore(&cm.lock, flags);
345 /* Make sure the port haven't released the mad yet */
346 mad_agent = cm_id_priv->av.port->mad_agent;
347 if (!mad_agent) {
348 pr_info("%s: not a valid MAD agent\n", __func__);
349 ret = -ENODEV;
350 goto out;
351 }
352 ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr, 0);
353 if (IS_ERR(ah)) {
354 ret = PTR_ERR(ah);
355 goto out;
356 }
357
358 m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
359 av->pkey_index,
360 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
361 GFP_ATOMIC,
362 IB_MGMT_BASE_VERSION);
363 if (IS_ERR(m)) {
364 rdma_destroy_ah(ah, 0);
365 ret = PTR_ERR(m);
366 goto out;
367 }
368
369 /* Timeout set by caller if response is expected. */
370 m->ah = ah;
371 m->retries = cm_id_priv->max_cm_retries;
372
373 atomic_inc(&cm_id_priv->refcount);
374 m->context[0] = cm_id_priv;
375 *msg = m;
376
377 out:
378 spin_unlock_irqrestore(&cm.state_lock, flags2);
379 return ret;
380 }
381
cm_alloc_response_msg_no_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)382 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
383 struct ib_mad_recv_wc *mad_recv_wc)
384 {
385 return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
386 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
387 GFP_ATOMIC,
388 IB_MGMT_BASE_VERSION);
389 }
390
cm_create_response_msg_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf * msg)391 static int cm_create_response_msg_ah(struct cm_port *port,
392 struct ib_mad_recv_wc *mad_recv_wc,
393 struct ib_mad_send_buf *msg)
394 {
395 struct ib_ah *ah;
396
397 ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
398 mad_recv_wc->recv_buf.grh, port->port_num);
399 if (IS_ERR(ah))
400 return PTR_ERR(ah);
401
402 msg->ah = ah;
403 return 0;
404 }
405
cm_free_msg(struct ib_mad_send_buf * msg)406 static void cm_free_msg(struct ib_mad_send_buf *msg)
407 {
408 if (msg->ah)
409 rdma_destroy_ah(msg->ah, 0);
410 if (msg->context[0])
411 cm_deref_id(msg->context[0]);
412 ib_free_send_mad(msg);
413 }
414
cm_alloc_response_msg(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf ** msg)415 static int cm_alloc_response_msg(struct cm_port *port,
416 struct ib_mad_recv_wc *mad_recv_wc,
417 struct ib_mad_send_buf **msg)
418 {
419 struct ib_mad_send_buf *m;
420 int ret;
421
422 m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
423 if (IS_ERR(m))
424 return PTR_ERR(m);
425
426 ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
427 if (ret) {
428 cm_free_msg(m);
429 return ret;
430 }
431
432 *msg = m;
433 return 0;
434 }
435
cm_copy_private_data(const void * private_data,u8 private_data_len)436 static void * cm_copy_private_data(const void *private_data,
437 u8 private_data_len)
438 {
439 void *data;
440
441 if (!private_data || !private_data_len)
442 return NULL;
443
444 data = kmemdup(private_data, private_data_len, GFP_KERNEL);
445 if (!data)
446 return ERR_PTR(-ENOMEM);
447
448 return data;
449 }
450
cm_set_private_data(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)451 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
452 void *private_data, u8 private_data_len)
453 {
454 if (cm_id_priv->private_data && cm_id_priv->private_data_len)
455 kfree(cm_id_priv->private_data);
456
457 cm_id_priv->private_data = private_data;
458 cm_id_priv->private_data_len = private_data_len;
459 }
460
cm_init_av_for_lap(struct cm_port * port,struct ib_wc * wc,struct ib_grh * grh,struct cm_av * av)461 static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
462 struct ib_grh *grh, struct cm_av *av)
463 {
464 struct rdma_ah_attr new_ah_attr;
465 int ret;
466
467 av->port = port;
468 av->pkey_index = wc->pkey_index;
469
470 /*
471 * av->ah_attr might be initialized based on past wc during incoming
472 * connect request or while sending out connect request. So initialize
473 * a new ah_attr on stack. If initialization fails, old ah_attr is
474 * used for sending any responses. If initialization is successful,
475 * than new ah_attr is used by overwriting old one.
476 */
477 ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
478 port->port_num, wc,
479 grh, &new_ah_attr);
480 if (ret)
481 return ret;
482
483 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
484 return 0;
485 }
486
cm_init_av_for_response(struct cm_port * port,struct ib_wc * wc,struct ib_grh * grh,struct cm_av * av)487 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
488 struct ib_grh *grh, struct cm_av *av)
489 {
490 av->port = port;
491 av->pkey_index = wc->pkey_index;
492 return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
493 port->port_num, wc,
494 grh, &av->ah_attr);
495 }
496
add_cm_id_to_port_list(struct cm_id_private * cm_id_priv,struct cm_av * av,struct cm_port * port)497 static int add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,
498 struct cm_av *av,
499 struct cm_port *port)
500 {
501 unsigned long flags;
502 int ret = 0;
503
504 spin_lock_irqsave(&cm.lock, flags);
505
506 if (&cm_id_priv->av == av)
507 list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
508 else if (&cm_id_priv->alt_av == av)
509 list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
510 else
511 ret = -EINVAL;
512
513 spin_unlock_irqrestore(&cm.lock, flags);
514 return ret;
515 }
516
517 static struct cm_port *
get_cm_port_from_path(struct sa_path_rec * path,const struct ib_gid_attr * attr)518 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
519 {
520 struct cm_device *cm_dev;
521 struct cm_port *port = NULL;
522 unsigned long flags;
523
524 if (attr) {
525 read_lock_irqsave(&cm.device_lock, flags);
526 list_for_each_entry(cm_dev, &cm.device_list, list) {
527 if (cm_dev->ib_device == attr->device) {
528 port = cm_dev->port[attr->port_num - 1];
529 break;
530 }
531 }
532 read_unlock_irqrestore(&cm.device_lock, flags);
533 } else {
534 /* SGID attribute can be NULL in following
535 * conditions.
536 * (a) Alternative path
537 * (b) IB link layer without GRH
538 * (c) LAP send messages
539 */
540 read_lock_irqsave(&cm.device_lock, flags);
541 list_for_each_entry(cm_dev, &cm.device_list, list) {
542 attr = rdma_find_gid(cm_dev->ib_device,
543 &path->sgid,
544 sa_conv_pathrec_to_gid_type(path),
545 NULL);
546 if (!IS_ERR(attr)) {
547 port = cm_dev->port[attr->port_num - 1];
548 break;
549 }
550 }
551 read_unlock_irqrestore(&cm.device_lock, flags);
552 if (port)
553 rdma_put_gid_attr(attr);
554 }
555 return port;
556 }
557
cm_init_av_by_path(struct sa_path_rec * path,const struct ib_gid_attr * sgid_attr,struct cm_av * av,struct cm_id_private * cm_id_priv)558 static int cm_init_av_by_path(struct sa_path_rec *path,
559 const struct ib_gid_attr *sgid_attr,
560 struct cm_av *av,
561 struct cm_id_private *cm_id_priv)
562 {
563 struct rdma_ah_attr new_ah_attr;
564 struct cm_device *cm_dev;
565 struct cm_port *port;
566 int ret;
567
568 port = get_cm_port_from_path(path, sgid_attr);
569 if (!port)
570 return -EINVAL;
571 cm_dev = port->cm_dev;
572
573 ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
574 be16_to_cpu(path->pkey), &av->pkey_index);
575 if (ret)
576 return ret;
577
578 av->port = port;
579
580 /*
581 * av->ah_attr might be initialized based on wc or during
582 * request processing time which might have reference to sgid_attr.
583 * So initialize a new ah_attr on stack.
584 * If initialization fails, old ah_attr is used for sending any
585 * responses. If initialization is successful, than new ah_attr
586 * is used by overwriting the old one. So that right ah_attr
587 * can be used to return an error response.
588 */
589 ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
590 &new_ah_attr, sgid_attr);
591 if (ret)
592 return ret;
593
594 av->timeout = path->packet_life_time + 1;
595
596 ret = add_cm_id_to_port_list(cm_id_priv, av, port);
597 if (ret) {
598 rdma_destroy_ah_attr(&new_ah_attr);
599 return ret;
600 }
601 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
602 return 0;
603 }
604
cm_alloc_id(struct cm_id_private * cm_id_priv)605 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
606 {
607 unsigned long flags;
608 int id;
609
610 idr_preload(GFP_KERNEL);
611 spin_lock_irqsave(&cm.lock, flags);
612
613 id = idr_alloc_cyclic(&cm.local_id_table, cm_id_priv, 0, 0, GFP_NOWAIT);
614
615 spin_unlock_irqrestore(&cm.lock, flags);
616 idr_preload_end();
617
618 cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
619 return id < 0 ? id : 0;
620 }
621
cm_free_id(__be32 local_id)622 static void cm_free_id(__be32 local_id)
623 {
624 spin_lock_irq(&cm.lock);
625 idr_remove(&cm.local_id_table,
626 (__force int) (local_id ^ cm.random_id_operand));
627 spin_unlock_irq(&cm.lock);
628 }
629
cm_get_id(__be32 local_id,__be32 remote_id)630 static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
631 {
632 struct cm_id_private *cm_id_priv;
633
634 cm_id_priv = idr_find(&cm.local_id_table,
635 (__force int) (local_id ^ cm.random_id_operand));
636 if (cm_id_priv) {
637 if (cm_id_priv->id.remote_id == remote_id)
638 atomic_inc(&cm_id_priv->refcount);
639 else
640 cm_id_priv = NULL;
641 }
642
643 return cm_id_priv;
644 }
645
cm_acquire_id(__be32 local_id,__be32 remote_id)646 static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id)
647 {
648 struct cm_id_private *cm_id_priv;
649
650 spin_lock_irq(&cm.lock);
651 cm_id_priv = cm_get_id(local_id, remote_id);
652 spin_unlock_irq(&cm.lock);
653
654 return cm_id_priv;
655 }
656
657 /*
658 * Trivial helpers to strip endian annotation and compare; the
659 * endianness doesn't actually matter since we just need a stable
660 * order for the RB tree.
661 */
be32_lt(__be32 a,__be32 b)662 static int be32_lt(__be32 a, __be32 b)
663 {
664 return (__force u32) a < (__force u32) b;
665 }
666
be32_gt(__be32 a,__be32 b)667 static int be32_gt(__be32 a, __be32 b)
668 {
669 return (__force u32) a > (__force u32) b;
670 }
671
be64_lt(__be64 a,__be64 b)672 static int be64_lt(__be64 a, __be64 b)
673 {
674 return (__force u64) a < (__force u64) b;
675 }
676
be64_gt(__be64 a,__be64 b)677 static int be64_gt(__be64 a, __be64 b)
678 {
679 return (__force u64) a > (__force u64) b;
680 }
681
cm_insert_listen(struct cm_id_private * cm_id_priv)682 static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)
683 {
684 struct rb_node **link = &cm.listen_service_table.rb_node;
685 struct rb_node *parent = NULL;
686 struct cm_id_private *cur_cm_id_priv;
687 __be64 service_id = cm_id_priv->id.service_id;
688 __be64 service_mask = cm_id_priv->id.service_mask;
689
690 while (*link) {
691 parent = *link;
692 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
693 service_node);
694 if ((cur_cm_id_priv->id.service_mask & service_id) ==
695 (service_mask & cur_cm_id_priv->id.service_id) &&
696 (cm_id_priv->id.device == cur_cm_id_priv->id.device))
697 return cur_cm_id_priv;
698
699 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
700 link = &(*link)->rb_left;
701 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
702 link = &(*link)->rb_right;
703 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
704 link = &(*link)->rb_left;
705 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
706 link = &(*link)->rb_right;
707 else
708 link = &(*link)->rb_right;
709 }
710 rb_link_node(&cm_id_priv->service_node, parent, link);
711 rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
712 return NULL;
713 }
714
cm_find_listen(struct ib_device * device,__be64 service_id)715 static struct cm_id_private * cm_find_listen(struct ib_device *device,
716 __be64 service_id)
717 {
718 struct rb_node *node = cm.listen_service_table.rb_node;
719 struct cm_id_private *cm_id_priv;
720
721 while (node) {
722 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
723 if ((cm_id_priv->id.service_mask & service_id) ==
724 cm_id_priv->id.service_id &&
725 (cm_id_priv->id.device == device))
726 return cm_id_priv;
727
728 if (device < cm_id_priv->id.device)
729 node = node->rb_left;
730 else if (device > cm_id_priv->id.device)
731 node = node->rb_right;
732 else if (be64_lt(service_id, cm_id_priv->id.service_id))
733 node = node->rb_left;
734 else if (be64_gt(service_id, cm_id_priv->id.service_id))
735 node = node->rb_right;
736 else
737 node = node->rb_right;
738 }
739 return NULL;
740 }
741
cm_insert_remote_id(struct cm_timewait_info * timewait_info)742 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
743 *timewait_info)
744 {
745 struct rb_node **link = &cm.remote_id_table.rb_node;
746 struct rb_node *parent = NULL;
747 struct cm_timewait_info *cur_timewait_info;
748 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
749 __be32 remote_id = timewait_info->work.remote_id;
750
751 while (*link) {
752 parent = *link;
753 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
754 remote_id_node);
755 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
756 link = &(*link)->rb_left;
757 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
758 link = &(*link)->rb_right;
759 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
760 link = &(*link)->rb_left;
761 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
762 link = &(*link)->rb_right;
763 else
764 return cur_timewait_info;
765 }
766 timewait_info->inserted_remote_id = 1;
767 rb_link_node(&timewait_info->remote_id_node, parent, link);
768 rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
769 return NULL;
770 }
771
cm_find_remote_id(__be64 remote_ca_guid,__be32 remote_id)772 static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
773 __be32 remote_id)
774 {
775 struct rb_node *node = cm.remote_id_table.rb_node;
776 struct cm_timewait_info *timewait_info;
777
778 while (node) {
779 timewait_info = rb_entry(node, struct cm_timewait_info,
780 remote_id_node);
781 if (be32_lt(remote_id, timewait_info->work.remote_id))
782 node = node->rb_left;
783 else if (be32_gt(remote_id, timewait_info->work.remote_id))
784 node = node->rb_right;
785 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
786 node = node->rb_left;
787 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
788 node = node->rb_right;
789 else
790 return timewait_info;
791 }
792 return NULL;
793 }
794
cm_insert_remote_qpn(struct cm_timewait_info * timewait_info)795 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
796 *timewait_info)
797 {
798 struct rb_node **link = &cm.remote_qp_table.rb_node;
799 struct rb_node *parent = NULL;
800 struct cm_timewait_info *cur_timewait_info;
801 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
802 __be32 remote_qpn = timewait_info->remote_qpn;
803
804 while (*link) {
805 parent = *link;
806 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
807 remote_qp_node);
808 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
809 link = &(*link)->rb_left;
810 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
811 link = &(*link)->rb_right;
812 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
813 link = &(*link)->rb_left;
814 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
815 link = &(*link)->rb_right;
816 else
817 return cur_timewait_info;
818 }
819 timewait_info->inserted_remote_qp = 1;
820 rb_link_node(&timewait_info->remote_qp_node, parent, link);
821 rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
822 return NULL;
823 }
824
cm_insert_remote_sidr(struct cm_id_private * cm_id_priv)825 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
826 *cm_id_priv)
827 {
828 struct rb_node **link = &cm.remote_sidr_table.rb_node;
829 struct rb_node *parent = NULL;
830 struct cm_id_private *cur_cm_id_priv;
831 union ib_gid *port_gid = &cm_id_priv->av.dgid;
832 __be32 remote_id = cm_id_priv->id.remote_id;
833
834 while (*link) {
835 parent = *link;
836 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
837 sidr_id_node);
838 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
839 link = &(*link)->rb_left;
840 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
841 link = &(*link)->rb_right;
842 else {
843 int cmp;
844 cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
845 sizeof *port_gid);
846 if (cmp < 0)
847 link = &(*link)->rb_left;
848 else if (cmp > 0)
849 link = &(*link)->rb_right;
850 else
851 return cur_cm_id_priv;
852 }
853 }
854 rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
855 rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
856 return NULL;
857 }
858
cm_reject_sidr_req(struct cm_id_private * cm_id_priv,enum ib_cm_sidr_status status)859 static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,
860 enum ib_cm_sidr_status status)
861 {
862 struct ib_cm_sidr_rep_param param;
863
864 memset(¶m, 0, sizeof param);
865 param.status = status;
866 ib_send_cm_sidr_rep(&cm_id_priv->id, ¶m);
867 }
868
ib_create_cm_id(struct ib_device * device,ib_cm_handler cm_handler,void * context)869 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
870 ib_cm_handler cm_handler,
871 void *context)
872 {
873 struct cm_id_private *cm_id_priv;
874 int ret;
875
876 cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
877 if (!cm_id_priv)
878 return ERR_PTR(-ENOMEM);
879
880 cm_id_priv->id.state = IB_CM_IDLE;
881 cm_id_priv->id.device = device;
882 cm_id_priv->id.cm_handler = cm_handler;
883 cm_id_priv->id.context = context;
884 cm_id_priv->id.remote_cm_qpn = 1;
885 ret = cm_alloc_id(cm_id_priv);
886 if (ret)
887 goto error;
888
889 spin_lock_init(&cm_id_priv->lock);
890 init_completion(&cm_id_priv->comp);
891 INIT_LIST_HEAD(&cm_id_priv->work_list);
892 INIT_LIST_HEAD(&cm_id_priv->prim_list);
893 INIT_LIST_HEAD(&cm_id_priv->altr_list);
894 atomic_set(&cm_id_priv->work_count, -1);
895 atomic_set(&cm_id_priv->refcount, 1);
896 return &cm_id_priv->id;
897
898 error:
899 kfree(cm_id_priv);
900 return ERR_PTR(-ENOMEM);
901 }
902 EXPORT_SYMBOL(ib_create_cm_id);
903
cm_dequeue_work(struct cm_id_private * cm_id_priv)904 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
905 {
906 struct cm_work *work;
907
908 if (list_empty(&cm_id_priv->work_list))
909 return NULL;
910
911 work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
912 list_del(&work->list);
913 return work;
914 }
915
cm_free_work(struct cm_work * work)916 static void cm_free_work(struct cm_work *work)
917 {
918 if (work->mad_recv_wc)
919 ib_free_recv_mad(work->mad_recv_wc);
920 kfree(work);
921 }
922
cm_convert_to_ms(int iba_time)923 static inline int cm_convert_to_ms(int iba_time)
924 {
925 /* approximate conversion to ms from 4.096us x 2^iba_time */
926 return 1 << max(iba_time - 8, 0);
927 }
928
929 /*
930 * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
931 * Because of how ack_timeout is stored, adding one doubles the timeout.
932 * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
933 * increment it (round up) only if the other is within 50%.
934 */
cm_ack_timeout(u8 ca_ack_delay,u8 packet_life_time)935 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
936 {
937 int ack_timeout = packet_life_time + 1;
938
939 if (ack_timeout >= ca_ack_delay)
940 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
941 else
942 ack_timeout = ca_ack_delay +
943 (ack_timeout >= (ca_ack_delay - 1));
944
945 return min(31, ack_timeout);
946 }
947
cm_cleanup_timewait(struct cm_timewait_info * timewait_info)948 static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
949 {
950 if (timewait_info->inserted_remote_id) {
951 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
952 timewait_info->inserted_remote_id = 0;
953 }
954
955 if (timewait_info->inserted_remote_qp) {
956 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
957 timewait_info->inserted_remote_qp = 0;
958 }
959 }
960
cm_create_timewait_info(__be32 local_id)961 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
962 {
963 struct cm_timewait_info *timewait_info;
964
965 timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
966 if (!timewait_info)
967 return ERR_PTR(-ENOMEM);
968
969 timewait_info->work.local_id = local_id;
970 INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
971 timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
972 return timewait_info;
973 }
974
cm_enter_timewait(struct cm_id_private * cm_id_priv)975 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
976 {
977 int wait_time;
978 unsigned long flags;
979 struct cm_device *cm_dev;
980
981 cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
982 if (!cm_dev)
983 return;
984
985 spin_lock_irqsave(&cm.lock, flags);
986 cm_cleanup_timewait(cm_id_priv->timewait_info);
987 list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
988 spin_unlock_irqrestore(&cm.lock, flags);
989
990 /*
991 * The cm_id could be destroyed by the user before we exit timewait.
992 * To protect against this, we search for the cm_id after exiting
993 * timewait before notifying the user that we've exited timewait.
994 */
995 cm_id_priv->id.state = IB_CM_TIMEWAIT;
996 wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
997
998 /* Check if the device started its remove_one */
999 spin_lock_irqsave(&cm.lock, flags);
1000 if (!cm_dev->going_down)
1001 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1002 msecs_to_jiffies(wait_time));
1003 spin_unlock_irqrestore(&cm.lock, flags);
1004
1005 cm_id_priv->timewait_info = NULL;
1006 }
1007
cm_reset_to_idle(struct cm_id_private * cm_id_priv)1008 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1009 {
1010 unsigned long flags;
1011
1012 cm_id_priv->id.state = IB_CM_IDLE;
1013 if (cm_id_priv->timewait_info) {
1014 spin_lock_irqsave(&cm.lock, flags);
1015 cm_cleanup_timewait(cm_id_priv->timewait_info);
1016 spin_unlock_irqrestore(&cm.lock, flags);
1017 kfree(cm_id_priv->timewait_info);
1018 cm_id_priv->timewait_info = NULL;
1019 }
1020 }
1021
cm_destroy_id(struct ib_cm_id * cm_id,int err)1022 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1023 {
1024 struct cm_id_private *cm_id_priv;
1025 struct cm_work *work;
1026
1027 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1028 retest:
1029 spin_lock_irq(&cm_id_priv->lock);
1030 switch (cm_id->state) {
1031 case IB_CM_LISTEN:
1032 spin_unlock_irq(&cm_id_priv->lock);
1033
1034 spin_lock_irq(&cm.lock);
1035 if (--cm_id_priv->listen_sharecount > 0) {
1036 /* The id is still shared. */
1037 cm_deref_id(cm_id_priv);
1038 spin_unlock_irq(&cm.lock);
1039 return;
1040 }
1041 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1042 spin_unlock_irq(&cm.lock);
1043 break;
1044 case IB_CM_SIDR_REQ_SENT:
1045 cm_id->state = IB_CM_IDLE;
1046 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1047 spin_unlock_irq(&cm_id_priv->lock);
1048 break;
1049 case IB_CM_SIDR_REQ_RCVD:
1050 spin_unlock_irq(&cm_id_priv->lock);
1051 cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);
1052 spin_lock_irq(&cm.lock);
1053 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1054 rb_erase(&cm_id_priv->sidr_id_node,
1055 &cm.remote_sidr_table);
1056 spin_unlock_irq(&cm.lock);
1057 break;
1058 case IB_CM_REQ_SENT:
1059 case IB_CM_MRA_REQ_RCVD:
1060 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1061 spin_unlock_irq(&cm_id_priv->lock);
1062 ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
1063 &cm_id_priv->id.device->node_guid,
1064 sizeof cm_id_priv->id.device->node_guid,
1065 NULL, 0);
1066 break;
1067 case IB_CM_REQ_RCVD:
1068 if (err == -ENOMEM) {
1069 /* Do not reject to allow future retries. */
1070 cm_reset_to_idle(cm_id_priv);
1071 spin_unlock_irq(&cm_id_priv->lock);
1072 } else {
1073 spin_unlock_irq(&cm_id_priv->lock);
1074 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
1075 NULL, 0, NULL, 0);
1076 }
1077 break;
1078 case IB_CM_REP_SENT:
1079 case IB_CM_MRA_REP_RCVD:
1080 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1081 /* Fall through */
1082 case IB_CM_MRA_REQ_SENT:
1083 case IB_CM_REP_RCVD:
1084 case IB_CM_MRA_REP_SENT:
1085 spin_unlock_irq(&cm_id_priv->lock);
1086 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
1087 NULL, 0, NULL, 0);
1088 break;
1089 case IB_CM_ESTABLISHED:
1090 spin_unlock_irq(&cm_id_priv->lock);
1091 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT)
1092 break;
1093 ib_send_cm_dreq(cm_id, NULL, 0);
1094 goto retest;
1095 case IB_CM_DREQ_SENT:
1096 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1097 cm_enter_timewait(cm_id_priv);
1098 spin_unlock_irq(&cm_id_priv->lock);
1099 break;
1100 case IB_CM_DREQ_RCVD:
1101 spin_unlock_irq(&cm_id_priv->lock);
1102 ib_send_cm_drep(cm_id, NULL, 0);
1103 break;
1104 default:
1105 spin_unlock_irq(&cm_id_priv->lock);
1106 break;
1107 }
1108
1109 spin_lock_irq(&cm.lock);
1110 if (!list_empty(&cm_id_priv->altr_list) &&
1111 (!cm_id_priv->altr_send_port_not_ready))
1112 list_del(&cm_id_priv->altr_list);
1113 if (!list_empty(&cm_id_priv->prim_list) &&
1114 (!cm_id_priv->prim_send_port_not_ready))
1115 list_del(&cm_id_priv->prim_list);
1116 spin_unlock_irq(&cm.lock);
1117
1118 cm_free_id(cm_id->local_id);
1119 cm_deref_id(cm_id_priv);
1120 wait_for_completion(&cm_id_priv->comp);
1121 while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1122 cm_free_work(work);
1123
1124 rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr);
1125 rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr);
1126 kfree(cm_id_priv->private_data);
1127 kfree(cm_id_priv);
1128 }
1129
ib_destroy_cm_id(struct ib_cm_id * cm_id)1130 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1131 {
1132 cm_destroy_id(cm_id, 0);
1133 }
1134 EXPORT_SYMBOL(ib_destroy_cm_id);
1135
1136 /**
1137 * __ib_cm_listen - Initiates listening on the specified service ID for
1138 * connection and service ID resolution requests.
1139 * @cm_id: Connection identifier associated with the listen request.
1140 * @service_id: Service identifier matched against incoming connection
1141 * and service ID resolution requests. The service ID should be specified
1142 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1143 * assign a service ID to the caller.
1144 * @service_mask: Mask applied to service ID used to listen across a
1145 * range of service IDs. If set to 0, the service ID is matched
1146 * exactly. This parameter is ignored if %service_id is set to
1147 * IB_CM_ASSIGN_SERVICE_ID.
1148 */
__ib_cm_listen(struct ib_cm_id * cm_id,__be64 service_id,__be64 service_mask)1149 static int __ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id,
1150 __be64 service_mask)
1151 {
1152 struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
1153 int ret = 0;
1154
1155 service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
1156 service_id &= service_mask;
1157 if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1158 (service_id != IB_CM_ASSIGN_SERVICE_ID))
1159 return -EINVAL;
1160
1161 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1162 if (cm_id->state != IB_CM_IDLE)
1163 return -EINVAL;
1164
1165 cm_id->state = IB_CM_LISTEN;
1166 ++cm_id_priv->listen_sharecount;
1167
1168 if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
1169 cm_id->service_id = cpu_to_be64(cm.listen_service_id++);
1170 cm_id->service_mask = ~cpu_to_be64(0);
1171 } else {
1172 cm_id->service_id = service_id;
1173 cm_id->service_mask = service_mask;
1174 }
1175 cur_cm_id_priv = cm_insert_listen(cm_id_priv);
1176
1177 if (cur_cm_id_priv) {
1178 cm_id->state = IB_CM_IDLE;
1179 --cm_id_priv->listen_sharecount;
1180 ret = -EBUSY;
1181 }
1182 return ret;
1183 }
1184
ib_cm_listen(struct ib_cm_id * cm_id,__be64 service_id,__be64 service_mask)1185 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)
1186 {
1187 unsigned long flags;
1188 int ret;
1189
1190 spin_lock_irqsave(&cm.lock, flags);
1191 ret = __ib_cm_listen(cm_id, service_id, service_mask);
1192 spin_unlock_irqrestore(&cm.lock, flags);
1193
1194 return ret;
1195 }
1196 EXPORT_SYMBOL(ib_cm_listen);
1197
1198 /**
1199 * Create a new listening ib_cm_id and listen on the given service ID.
1200 *
1201 * If there's an existing ID listening on that same device and service ID,
1202 * return it.
1203 *
1204 * @device: Device associated with the cm_id. All related communication will
1205 * be associated with the specified device.
1206 * @cm_handler: Callback invoked to notify the user of CM events.
1207 * @service_id: Service identifier matched against incoming connection
1208 * and service ID resolution requests. The service ID should be specified
1209 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1210 * assign a service ID to the caller.
1211 *
1212 * Callers should call ib_destroy_cm_id when done with the listener ID.
1213 */
ib_cm_insert_listen(struct ib_device * device,ib_cm_handler cm_handler,__be64 service_id)1214 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1215 ib_cm_handler cm_handler,
1216 __be64 service_id)
1217 {
1218 struct cm_id_private *cm_id_priv;
1219 struct ib_cm_id *cm_id;
1220 unsigned long flags;
1221 int err = 0;
1222
1223 /* Create an ID in advance, since the creation may sleep */
1224 cm_id = ib_create_cm_id(device, cm_handler, NULL);
1225 if (IS_ERR(cm_id))
1226 return cm_id;
1227
1228 spin_lock_irqsave(&cm.lock, flags);
1229
1230 if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1231 goto new_id;
1232
1233 /* Find an existing ID */
1234 cm_id_priv = cm_find_listen(device, service_id);
1235 if (cm_id_priv) {
1236 if (cm_id->cm_handler != cm_handler || cm_id->context) {
1237 /* Sharing an ib_cm_id with different handlers is not
1238 * supported */
1239 spin_unlock_irqrestore(&cm.lock, flags);
1240 ib_destroy_cm_id(cm_id);
1241 return ERR_PTR(-EINVAL);
1242 }
1243 atomic_inc(&cm_id_priv->refcount);
1244 ++cm_id_priv->listen_sharecount;
1245 spin_unlock_irqrestore(&cm.lock, flags);
1246
1247 ib_destroy_cm_id(cm_id);
1248 cm_id = &cm_id_priv->id;
1249 return cm_id;
1250 }
1251
1252 new_id:
1253 /* Use newly created ID */
1254 err = __ib_cm_listen(cm_id, service_id, 0);
1255
1256 spin_unlock_irqrestore(&cm.lock, flags);
1257
1258 if (err) {
1259 ib_destroy_cm_id(cm_id);
1260 return ERR_PTR(err);
1261 }
1262 return cm_id;
1263 }
1264 EXPORT_SYMBOL(ib_cm_insert_listen);
1265
cm_form_tid(struct cm_id_private * cm_id_priv)1266 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1267 {
1268 u64 hi_tid, low_tid;
1269
1270 hi_tid = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1271 low_tid = (u64)cm_id_priv->id.local_id;
1272 return cpu_to_be64(hi_tid | low_tid);
1273 }
1274
cm_format_mad_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid)1275 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1276 __be16 attr_id, __be64 tid)
1277 {
1278 hdr->base_version = IB_MGMT_BASE_VERSION;
1279 hdr->mgmt_class = IB_MGMT_CLASS_CM;
1280 hdr->class_version = IB_CM_CLASS_VERSION;
1281 hdr->method = IB_MGMT_METHOD_SEND;
1282 hdr->attr_id = attr_id;
1283 hdr->tid = tid;
1284 }
1285
cm_format_req(struct cm_req_msg * req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_req_param * param)1286 static void cm_format_req(struct cm_req_msg *req_msg,
1287 struct cm_id_private *cm_id_priv,
1288 struct ib_cm_req_param *param)
1289 {
1290 struct sa_path_rec *pri_path = param->primary_path;
1291 struct sa_path_rec *alt_path = param->alternate_path;
1292 bool pri_ext = false;
1293
1294 if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1295 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1296 pri_path->opa.slid);
1297
1298 cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1299 cm_form_tid(cm_id_priv));
1300
1301 req_msg->local_comm_id = cm_id_priv->id.local_id;
1302 req_msg->service_id = param->service_id;
1303 req_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
1304 cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num));
1305 cm_req_set_init_depth(req_msg, param->initiator_depth);
1306 cm_req_set_remote_resp_timeout(req_msg,
1307 param->remote_cm_response_timeout);
1308 cm_req_set_qp_type(req_msg, param->qp_type);
1309 cm_req_set_flow_ctrl(req_msg, param->flow_control);
1310 cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn));
1311 cm_req_set_local_resp_timeout(req_msg,
1312 param->local_cm_response_timeout);
1313 req_msg->pkey = param->primary_path->pkey;
1314 cm_req_set_path_mtu(req_msg, param->primary_path->mtu);
1315 cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
1316
1317 if (param->qp_type != IB_QPT_XRC_INI) {
1318 cm_req_set_resp_res(req_msg, param->responder_resources);
1319 cm_req_set_retry_count(req_msg, param->retry_count);
1320 cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count);
1321 cm_req_set_srq(req_msg, param->srq);
1322 }
1323
1324 req_msg->primary_local_gid = pri_path->sgid;
1325 req_msg->primary_remote_gid = pri_path->dgid;
1326 if (pri_ext) {
1327 req_msg->primary_local_gid.global.interface_id
1328 = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1329 req_msg->primary_remote_gid.global.interface_id
1330 = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1331 }
1332 if (pri_path->hop_limit <= 1) {
1333 req_msg->primary_local_lid = pri_ext ? 0 :
1334 htons(ntohl(sa_path_get_slid(pri_path)));
1335 req_msg->primary_remote_lid = pri_ext ? 0 :
1336 htons(ntohl(sa_path_get_dlid(pri_path)));
1337 } else {
1338 /* Work-around until there's a way to obtain remote LID info */
1339 req_msg->primary_local_lid = IB_LID_PERMISSIVE;
1340 req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
1341 }
1342 cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
1343 cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
1344 req_msg->primary_traffic_class = pri_path->traffic_class;
1345 req_msg->primary_hop_limit = pri_path->hop_limit;
1346 cm_req_set_primary_sl(req_msg, pri_path->sl);
1347 cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1));
1348 cm_req_set_primary_local_ack_timeout(req_msg,
1349 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1350 pri_path->packet_life_time));
1351
1352 if (alt_path) {
1353 bool alt_ext = false;
1354
1355 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1356 alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1357 alt_path->opa.slid);
1358
1359 req_msg->alt_local_gid = alt_path->sgid;
1360 req_msg->alt_remote_gid = alt_path->dgid;
1361 if (alt_ext) {
1362 req_msg->alt_local_gid.global.interface_id
1363 = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1364 req_msg->alt_remote_gid.global.interface_id
1365 = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1366 }
1367 if (alt_path->hop_limit <= 1) {
1368 req_msg->alt_local_lid = alt_ext ? 0 :
1369 htons(ntohl(sa_path_get_slid(alt_path)));
1370 req_msg->alt_remote_lid = alt_ext ? 0 :
1371 htons(ntohl(sa_path_get_dlid(alt_path)));
1372 } else {
1373 req_msg->alt_local_lid = IB_LID_PERMISSIVE;
1374 req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
1375 }
1376 cm_req_set_alt_flow_label(req_msg,
1377 alt_path->flow_label);
1378 cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
1379 req_msg->alt_traffic_class = alt_path->traffic_class;
1380 req_msg->alt_hop_limit = alt_path->hop_limit;
1381 cm_req_set_alt_sl(req_msg, alt_path->sl);
1382 cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1));
1383 cm_req_set_alt_local_ack_timeout(req_msg,
1384 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1385 alt_path->packet_life_time));
1386 }
1387
1388 if (param->private_data && param->private_data_len)
1389 memcpy(req_msg->private_data, param->private_data,
1390 param->private_data_len);
1391 }
1392
cm_validate_req_param(struct ib_cm_req_param * param)1393 static int cm_validate_req_param(struct ib_cm_req_param *param)
1394 {
1395 /* peer-to-peer not supported */
1396 if (param->peer_to_peer)
1397 return -EINVAL;
1398
1399 if (!param->primary_path)
1400 return -EINVAL;
1401
1402 if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1403 param->qp_type != IB_QPT_XRC_INI)
1404 return -EINVAL;
1405
1406 if (param->private_data &&
1407 param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1408 return -EINVAL;
1409
1410 if (param->alternate_path &&
1411 (param->alternate_path->pkey != param->primary_path->pkey ||
1412 param->alternate_path->mtu != param->primary_path->mtu))
1413 return -EINVAL;
1414
1415 return 0;
1416 }
1417
ib_send_cm_req(struct ib_cm_id * cm_id,struct ib_cm_req_param * param)1418 int ib_send_cm_req(struct ib_cm_id *cm_id,
1419 struct ib_cm_req_param *param)
1420 {
1421 struct cm_id_private *cm_id_priv;
1422 struct cm_req_msg *req_msg;
1423 unsigned long flags;
1424 int ret;
1425
1426 ret = cm_validate_req_param(param);
1427 if (ret)
1428 return ret;
1429
1430 /* Verify that we're not in timewait. */
1431 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1432 spin_lock_irqsave(&cm_id_priv->lock, flags);
1433 if (cm_id->state != IB_CM_IDLE) {
1434 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1435 ret = -EINVAL;
1436 goto out;
1437 }
1438 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1439
1440 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1441 id.local_id);
1442 if (IS_ERR(cm_id_priv->timewait_info)) {
1443 ret = PTR_ERR(cm_id_priv->timewait_info);
1444 goto out;
1445 }
1446
1447 ret = cm_init_av_by_path(param->primary_path,
1448 param->ppath_sgid_attr, &cm_id_priv->av,
1449 cm_id_priv);
1450 if (ret)
1451 goto error1;
1452 if (param->alternate_path) {
1453 ret = cm_init_av_by_path(param->alternate_path, NULL,
1454 &cm_id_priv->alt_av, cm_id_priv);
1455 if (ret)
1456 goto error1;
1457 }
1458 cm_id->service_id = param->service_id;
1459 cm_id->service_mask = ~cpu_to_be64(0);
1460 cm_id_priv->timeout_ms = cm_convert_to_ms(
1461 param->primary_path->packet_life_time) * 2 +
1462 cm_convert_to_ms(
1463 param->remote_cm_response_timeout);
1464 cm_id_priv->max_cm_retries = param->max_cm_retries;
1465 cm_id_priv->initiator_depth = param->initiator_depth;
1466 cm_id_priv->responder_resources = param->responder_resources;
1467 cm_id_priv->retry_count = param->retry_count;
1468 cm_id_priv->path_mtu = param->primary_path->mtu;
1469 cm_id_priv->pkey = param->primary_path->pkey;
1470 cm_id_priv->qp_type = param->qp_type;
1471
1472 ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
1473 if (ret)
1474 goto error1;
1475
1476 req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
1477 cm_format_req(req_msg, cm_id_priv, param);
1478 cm_id_priv->tid = req_msg->hdr.tid;
1479 cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
1480 cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
1481
1482 cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg);
1483 cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg);
1484
1485 spin_lock_irqsave(&cm_id_priv->lock, flags);
1486 ret = ib_post_send_mad(cm_id_priv->msg, NULL);
1487 if (ret) {
1488 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1489 goto error2;
1490 }
1491 BUG_ON(cm_id->state != IB_CM_IDLE);
1492 cm_id->state = IB_CM_REQ_SENT;
1493 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1494 return 0;
1495
1496 error2: cm_free_msg(cm_id_priv->msg);
1497 error1: kfree(cm_id_priv->timewait_info);
1498 out: return ret;
1499 }
1500 EXPORT_SYMBOL(ib_send_cm_req);
1501
cm_issue_rej(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,enum ib_cm_rej_reason reason,enum cm_msg_response msg_rejected,void * ari,u8 ari_length)1502 static int cm_issue_rej(struct cm_port *port,
1503 struct ib_mad_recv_wc *mad_recv_wc,
1504 enum ib_cm_rej_reason reason,
1505 enum cm_msg_response msg_rejected,
1506 void *ari, u8 ari_length)
1507 {
1508 struct ib_mad_send_buf *msg = NULL;
1509 struct cm_rej_msg *rej_msg, *rcv_msg;
1510 int ret;
1511
1512 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1513 if (ret)
1514 return ret;
1515
1516 /* We just need common CM header information. Cast to any message. */
1517 rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1518 rej_msg = (struct cm_rej_msg *) msg->mad;
1519
1520 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1521 rej_msg->remote_comm_id = rcv_msg->local_comm_id;
1522 rej_msg->local_comm_id = rcv_msg->remote_comm_id;
1523 cm_rej_set_msg_rejected(rej_msg, msg_rejected);
1524 rej_msg->reason = cpu_to_be16(reason);
1525
1526 if (ari && ari_length) {
1527 cm_rej_set_reject_info_len(rej_msg, ari_length);
1528 memcpy(rej_msg->ari, ari, ari_length);
1529 }
1530
1531 ret = ib_post_send_mad(msg, NULL);
1532 if (ret)
1533 cm_free_msg(msg);
1534
1535 return ret;
1536 }
1537
cm_req_has_alt_path(struct cm_req_msg * req_msg)1538 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1539 {
1540 return ((req_msg->alt_local_lid) ||
1541 (ib_is_opa_gid(&req_msg->alt_local_gid)));
1542 }
1543
cm_path_set_rec_type(struct ib_device * ib_device,u8 port_num,struct sa_path_rec * path,union ib_gid * gid)1544 static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,
1545 struct sa_path_rec *path, union ib_gid *gid)
1546 {
1547 if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1548 path->rec_type = SA_PATH_REC_TYPE_OPA;
1549 else
1550 path->rec_type = SA_PATH_REC_TYPE_IB;
1551 }
1552
cm_format_path_lid_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path)1553 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1554 struct sa_path_rec *primary_path,
1555 struct sa_path_rec *alt_path)
1556 {
1557 u32 lid;
1558
1559 if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1560 sa_path_set_dlid(primary_path,
1561 htonl(ntohs(req_msg->primary_local_lid)));
1562 sa_path_set_slid(primary_path,
1563 htonl(ntohs(req_msg->primary_remote_lid)));
1564 } else {
1565 lid = opa_get_lid_from_gid(&req_msg->primary_local_gid);
1566 sa_path_set_dlid(primary_path, cpu_to_be32(lid));
1567
1568 lid = opa_get_lid_from_gid(&req_msg->primary_remote_gid);
1569 sa_path_set_slid(primary_path, cpu_to_be32(lid));
1570 }
1571
1572 if (!cm_req_has_alt_path(req_msg))
1573 return;
1574
1575 if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1576 sa_path_set_dlid(alt_path,
1577 htonl(ntohs(req_msg->alt_local_lid)));
1578 sa_path_set_slid(alt_path,
1579 htonl(ntohs(req_msg->alt_remote_lid)));
1580 } else {
1581 lid = opa_get_lid_from_gid(&req_msg->alt_local_gid);
1582 sa_path_set_dlid(alt_path, cpu_to_be32(lid));
1583
1584 lid = opa_get_lid_from_gid(&req_msg->alt_remote_gid);
1585 sa_path_set_slid(alt_path, cpu_to_be32(lid));
1586 }
1587 }
1588
cm_format_paths_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path)1589 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1590 struct sa_path_rec *primary_path,
1591 struct sa_path_rec *alt_path)
1592 {
1593 primary_path->dgid = req_msg->primary_local_gid;
1594 primary_path->sgid = req_msg->primary_remote_gid;
1595 primary_path->flow_label = cm_req_get_primary_flow_label(req_msg);
1596 primary_path->hop_limit = req_msg->primary_hop_limit;
1597 primary_path->traffic_class = req_msg->primary_traffic_class;
1598 primary_path->reversible = 1;
1599 primary_path->pkey = req_msg->pkey;
1600 primary_path->sl = cm_req_get_primary_sl(req_msg);
1601 primary_path->mtu_selector = IB_SA_EQ;
1602 primary_path->mtu = cm_req_get_path_mtu(req_msg);
1603 primary_path->rate_selector = IB_SA_EQ;
1604 primary_path->rate = cm_req_get_primary_packet_rate(req_msg);
1605 primary_path->packet_life_time_selector = IB_SA_EQ;
1606 primary_path->packet_life_time =
1607 cm_req_get_primary_local_ack_timeout(req_msg);
1608 primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1609 primary_path->service_id = req_msg->service_id;
1610 if (sa_path_is_roce(primary_path))
1611 primary_path->roce.route_resolved = false;
1612
1613 if (cm_req_has_alt_path(req_msg)) {
1614 alt_path->dgid = req_msg->alt_local_gid;
1615 alt_path->sgid = req_msg->alt_remote_gid;
1616 alt_path->flow_label = cm_req_get_alt_flow_label(req_msg);
1617 alt_path->hop_limit = req_msg->alt_hop_limit;
1618 alt_path->traffic_class = req_msg->alt_traffic_class;
1619 alt_path->reversible = 1;
1620 alt_path->pkey = req_msg->pkey;
1621 alt_path->sl = cm_req_get_alt_sl(req_msg);
1622 alt_path->mtu_selector = IB_SA_EQ;
1623 alt_path->mtu = cm_req_get_path_mtu(req_msg);
1624 alt_path->rate_selector = IB_SA_EQ;
1625 alt_path->rate = cm_req_get_alt_packet_rate(req_msg);
1626 alt_path->packet_life_time_selector = IB_SA_EQ;
1627 alt_path->packet_life_time =
1628 cm_req_get_alt_local_ack_timeout(req_msg);
1629 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1630 alt_path->service_id = req_msg->service_id;
1631
1632 if (sa_path_is_roce(alt_path))
1633 alt_path->roce.route_resolved = false;
1634 }
1635 cm_format_path_lid_from_req(req_msg, primary_path, alt_path);
1636 }
1637
cm_get_bth_pkey(struct cm_work * work)1638 static u16 cm_get_bth_pkey(struct cm_work *work)
1639 {
1640 struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1641 u8 port_num = work->port->port_num;
1642 u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1643 u16 pkey;
1644 int ret;
1645
1646 ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1647 if (ret) {
1648 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n",
1649 port_num, pkey_index, ret);
1650 return 0;
1651 }
1652
1653 return pkey;
1654 }
1655
1656 /**
1657 * Convert OPA SGID to IB SGID
1658 * ULPs (such as IPoIB) do not understand OPA GIDs and will
1659 * reject them as the local_gid will not match the sgid. Therefore,
1660 * change the pathrec's SGID to an IB SGID.
1661 *
1662 * @work: Work completion
1663 * @path: Path record
1664 */
cm_opa_to_ib_sgid(struct cm_work * work,struct sa_path_rec * path)1665 static void cm_opa_to_ib_sgid(struct cm_work *work,
1666 struct sa_path_rec *path)
1667 {
1668 struct ib_device *dev = work->port->cm_dev->ib_device;
1669 u8 port_num = work->port->port_num;
1670
1671 if (rdma_cap_opa_ah(dev, port_num) &&
1672 (ib_is_opa_gid(&path->sgid))) {
1673 union ib_gid sgid;
1674
1675 if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1676 dev_warn(&dev->dev,
1677 "Error updating sgid in CM request\n");
1678 return;
1679 }
1680
1681 path->sgid = sgid;
1682 }
1683 }
1684
cm_format_req_event(struct cm_work * work,struct cm_id_private * cm_id_priv,struct ib_cm_id * listen_id)1685 static void cm_format_req_event(struct cm_work *work,
1686 struct cm_id_private *cm_id_priv,
1687 struct ib_cm_id *listen_id)
1688 {
1689 struct cm_req_msg *req_msg;
1690 struct ib_cm_req_event_param *param;
1691
1692 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1693 param = &work->cm_event.param.req_rcvd;
1694 param->listen_id = listen_id;
1695 param->bth_pkey = cm_get_bth_pkey(work);
1696 param->port = cm_id_priv->av.port->port_num;
1697 param->primary_path = &work->path[0];
1698 cm_opa_to_ib_sgid(work, param->primary_path);
1699 if (cm_req_has_alt_path(req_msg)) {
1700 param->alternate_path = &work->path[1];
1701 cm_opa_to_ib_sgid(work, param->alternate_path);
1702 } else {
1703 param->alternate_path = NULL;
1704 }
1705 param->remote_ca_guid = req_msg->local_ca_guid;
1706 param->remote_qkey = be32_to_cpu(req_msg->local_qkey);
1707 param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));
1708 param->qp_type = cm_req_get_qp_type(req_msg);
1709 param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg));
1710 param->responder_resources = cm_req_get_init_depth(req_msg);
1711 param->initiator_depth = cm_req_get_resp_res(req_msg);
1712 param->local_cm_response_timeout =
1713 cm_req_get_remote_resp_timeout(req_msg);
1714 param->flow_control = cm_req_get_flow_ctrl(req_msg);
1715 param->remote_cm_response_timeout =
1716 cm_req_get_local_resp_timeout(req_msg);
1717 param->retry_count = cm_req_get_retry_count(req_msg);
1718 param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
1719 param->srq = cm_req_get_srq(req_msg);
1720 param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1721 work->cm_event.private_data = &req_msg->private_data;
1722 }
1723
cm_process_work(struct cm_id_private * cm_id_priv,struct cm_work * work)1724 static void cm_process_work(struct cm_id_private *cm_id_priv,
1725 struct cm_work *work)
1726 {
1727 int ret;
1728
1729 /* We will typically only have the current event to report. */
1730 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1731 cm_free_work(work);
1732
1733 while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1734 spin_lock_irq(&cm_id_priv->lock);
1735 work = cm_dequeue_work(cm_id_priv);
1736 spin_unlock_irq(&cm_id_priv->lock);
1737 BUG_ON(!work);
1738 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1739 &work->cm_event);
1740 cm_free_work(work);
1741 }
1742 cm_deref_id(cm_id_priv);
1743 if (ret)
1744 cm_destroy_id(&cm_id_priv->id, ret);
1745 }
1746
cm_format_mra(struct cm_mra_msg * mra_msg,struct cm_id_private * cm_id_priv,enum cm_msg_response msg_mraed,u8 service_timeout,const void * private_data,u8 private_data_len)1747 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1748 struct cm_id_private *cm_id_priv,
1749 enum cm_msg_response msg_mraed, u8 service_timeout,
1750 const void *private_data, u8 private_data_len)
1751 {
1752 cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1753 cm_mra_set_msg_mraed(mra_msg, msg_mraed);
1754 mra_msg->local_comm_id = cm_id_priv->id.local_id;
1755 mra_msg->remote_comm_id = cm_id_priv->id.remote_id;
1756 cm_mra_set_service_timeout(mra_msg, service_timeout);
1757
1758 if (private_data && private_data_len)
1759 memcpy(mra_msg->private_data, private_data, private_data_len);
1760 }
1761
cm_format_rej(struct cm_rej_msg * rej_msg,struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)1762 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1763 struct cm_id_private *cm_id_priv,
1764 enum ib_cm_rej_reason reason,
1765 void *ari,
1766 u8 ari_length,
1767 const void *private_data,
1768 u8 private_data_len)
1769 {
1770 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1771 rej_msg->remote_comm_id = cm_id_priv->id.remote_id;
1772
1773 switch(cm_id_priv->id.state) {
1774 case IB_CM_REQ_RCVD:
1775 rej_msg->local_comm_id = 0;
1776 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1777 break;
1778 case IB_CM_MRA_REQ_SENT:
1779 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1780 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1781 break;
1782 case IB_CM_REP_RCVD:
1783 case IB_CM_MRA_REP_SENT:
1784 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1785 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP);
1786 break;
1787 default:
1788 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1789 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER);
1790 break;
1791 }
1792
1793 rej_msg->reason = cpu_to_be16(reason);
1794 if (ari && ari_length) {
1795 cm_rej_set_reject_info_len(rej_msg, ari_length);
1796 memcpy(rej_msg->ari, ari, ari_length);
1797 }
1798
1799 if (private_data && private_data_len)
1800 memcpy(rej_msg->private_data, private_data, private_data_len);
1801 }
1802
cm_dup_req_handler(struct cm_work * work,struct cm_id_private * cm_id_priv)1803 static void cm_dup_req_handler(struct cm_work *work,
1804 struct cm_id_private *cm_id_priv)
1805 {
1806 struct ib_mad_send_buf *msg = NULL;
1807 int ret;
1808
1809 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1810 counter[CM_REQ_COUNTER]);
1811
1812 /* Quick state check to discard duplicate REQs. */
1813 if (cm_id_priv->id.state == IB_CM_REQ_RCVD)
1814 return;
1815
1816 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1817 if (ret)
1818 return;
1819
1820 spin_lock_irq(&cm_id_priv->lock);
1821 switch (cm_id_priv->id.state) {
1822 case IB_CM_MRA_REQ_SENT:
1823 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1824 CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1825 cm_id_priv->private_data,
1826 cm_id_priv->private_data_len);
1827 break;
1828 case IB_CM_TIMEWAIT:
1829 cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv,
1830 IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);
1831 break;
1832 default:
1833 goto unlock;
1834 }
1835 spin_unlock_irq(&cm_id_priv->lock);
1836
1837 ret = ib_post_send_mad(msg, NULL);
1838 if (ret)
1839 goto free;
1840 return;
1841
1842 unlock: spin_unlock_irq(&cm_id_priv->lock);
1843 free: cm_free_msg(msg);
1844 }
1845
cm_match_req(struct cm_work * work,struct cm_id_private * cm_id_priv)1846 static struct cm_id_private * cm_match_req(struct cm_work *work,
1847 struct cm_id_private *cm_id_priv)
1848 {
1849 struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1850 struct cm_timewait_info *timewait_info;
1851 struct cm_req_msg *req_msg;
1852 struct ib_cm_id *cm_id;
1853
1854 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1855
1856 /* Check for possible duplicate REQ. */
1857 spin_lock_irq(&cm.lock);
1858 timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1859 if (timewait_info) {
1860 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
1861 timewait_info->work.remote_id);
1862 spin_unlock_irq(&cm.lock);
1863 if (cur_cm_id_priv) {
1864 cm_dup_req_handler(work, cur_cm_id_priv);
1865 cm_deref_id(cur_cm_id_priv);
1866 }
1867 return NULL;
1868 }
1869
1870 /* Check for stale connections. */
1871 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1872 if (timewait_info) {
1873 cm_cleanup_timewait(cm_id_priv->timewait_info);
1874 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
1875 timewait_info->work.remote_id);
1876
1877 spin_unlock_irq(&cm.lock);
1878 cm_issue_rej(work->port, work->mad_recv_wc,
1879 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
1880 NULL, 0);
1881 if (cur_cm_id_priv) {
1882 cm_id = &cur_cm_id_priv->id;
1883 ib_send_cm_dreq(cm_id, NULL, 0);
1884 cm_deref_id(cur_cm_id_priv);
1885 }
1886 return NULL;
1887 }
1888
1889 /* Find matching listen request. */
1890 listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
1891 req_msg->service_id);
1892 if (!listen_cm_id_priv) {
1893 cm_cleanup_timewait(cm_id_priv->timewait_info);
1894 spin_unlock_irq(&cm.lock);
1895 cm_issue_rej(work->port, work->mad_recv_wc,
1896 IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
1897 NULL, 0);
1898 goto out;
1899 }
1900 atomic_inc(&listen_cm_id_priv->refcount);
1901 atomic_inc(&cm_id_priv->refcount);
1902 cm_id_priv->id.state = IB_CM_REQ_RCVD;
1903 atomic_inc(&cm_id_priv->work_count);
1904 spin_unlock_irq(&cm.lock);
1905 out:
1906 return listen_cm_id_priv;
1907 }
1908
1909 /*
1910 * Work-around for inter-subnet connections. If the LIDs are permissive,
1911 * we need to override the LID/SL data in the REQ with the LID information
1912 * in the work completion.
1913 */
cm_process_routed_req(struct cm_req_msg * req_msg,struct ib_wc * wc)1914 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
1915 {
1916 if (!cm_req_get_primary_subnet_local(req_msg)) {
1917 if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
1918 req_msg->primary_local_lid = ib_lid_be16(wc->slid);
1919 cm_req_set_primary_sl(req_msg, wc->sl);
1920 }
1921
1922 if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE)
1923 req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1924 }
1925
1926 if (!cm_req_get_alt_subnet_local(req_msg)) {
1927 if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
1928 req_msg->alt_local_lid = ib_lid_be16(wc->slid);
1929 cm_req_set_alt_sl(req_msg, wc->sl);
1930 }
1931
1932 if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE)
1933 req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1934 }
1935 }
1936
cm_req_handler(struct cm_work * work)1937 static int cm_req_handler(struct cm_work *work)
1938 {
1939 struct ib_cm_id *cm_id;
1940 struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
1941 struct cm_req_msg *req_msg;
1942 const struct ib_global_route *grh;
1943 const struct ib_gid_attr *gid_attr;
1944 int ret;
1945
1946 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1947
1948 cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
1949 if (IS_ERR(cm_id))
1950 return PTR_ERR(cm_id);
1951
1952 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1953 cm_id_priv->id.remote_id = req_msg->local_comm_id;
1954 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
1955 work->mad_recv_wc->recv_buf.grh,
1956 &cm_id_priv->av);
1957 if (ret)
1958 goto destroy;
1959 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1960 id.local_id);
1961 if (IS_ERR(cm_id_priv->timewait_info)) {
1962 ret = PTR_ERR(cm_id_priv->timewait_info);
1963 goto destroy;
1964 }
1965 cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id;
1966 cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid;
1967 cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg);
1968
1969 listen_cm_id_priv = cm_match_req(work, cm_id_priv);
1970 if (!listen_cm_id_priv) {
1971 ret = -EINVAL;
1972 goto free_timeinfo;
1973 }
1974
1975 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
1976 cm_id_priv->id.context = listen_cm_id_priv->id.context;
1977 cm_id_priv->id.service_id = req_msg->service_id;
1978 cm_id_priv->id.service_mask = ~cpu_to_be64(0);
1979
1980 cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
1981
1982 memset(&work->path[0], 0, sizeof(work->path[0]));
1983 if (cm_req_has_alt_path(req_msg))
1984 memset(&work->path[1], 0, sizeof(work->path[1]));
1985 grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
1986 gid_attr = grh->sgid_attr;
1987
1988 if (gid_attr &&
1989 rdma_protocol_roce(work->port->cm_dev->ib_device,
1990 work->port->port_num)) {
1991 work->path[0].rec_type =
1992 sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
1993 } else {
1994 cm_path_set_rec_type(work->port->cm_dev->ib_device,
1995 work->port->port_num,
1996 &work->path[0],
1997 &req_msg->primary_local_gid);
1998 }
1999 if (cm_req_has_alt_path(req_msg))
2000 work->path[1].rec_type = work->path[0].rec_type;
2001 cm_format_paths_from_req(req_msg, &work->path[0],
2002 &work->path[1]);
2003 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2004 sa_path_set_dmac(&work->path[0],
2005 cm_id_priv->av.ah_attr.roce.dmac);
2006 work->path[0].hop_limit = grh->hop_limit;
2007 ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av,
2008 cm_id_priv);
2009 if (ret) {
2010 int err;
2011
2012 err = rdma_query_gid(work->port->cm_dev->ib_device,
2013 work->port->port_num, 0,
2014 &work->path[0].sgid);
2015 if (err)
2016 ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
2017 NULL, 0, NULL, 0);
2018 else
2019 ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
2020 &work->path[0].sgid,
2021 sizeof(work->path[0].sgid),
2022 NULL, 0);
2023 goto rejected;
2024 }
2025 if (cm_req_has_alt_path(req_msg)) {
2026 ret = cm_init_av_by_path(&work->path[1], NULL,
2027 &cm_id_priv->alt_av, cm_id_priv);
2028 if (ret) {
2029 ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
2030 &work->path[0].sgid,
2031 sizeof(work->path[0].sgid), NULL, 0);
2032 goto rejected;
2033 }
2034 }
2035 cm_id_priv->tid = req_msg->hdr.tid;
2036 cm_id_priv->timeout_ms = cm_convert_to_ms(
2037 cm_req_get_local_resp_timeout(req_msg));
2038 cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg);
2039 cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg);
2040 cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg);
2041 cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg);
2042 cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg);
2043 cm_id_priv->pkey = req_msg->pkey;
2044 cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg);
2045 cm_id_priv->retry_count = cm_req_get_retry_count(req_msg);
2046 cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
2047 cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2048
2049 cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2050 cm_process_work(cm_id_priv, work);
2051 cm_deref_id(listen_cm_id_priv);
2052 return 0;
2053
2054 rejected:
2055 atomic_dec(&cm_id_priv->refcount);
2056 cm_deref_id(listen_cm_id_priv);
2057 free_timeinfo:
2058 kfree(cm_id_priv->timewait_info);
2059 destroy:
2060 ib_destroy_cm_id(cm_id);
2061 return ret;
2062 }
2063
cm_format_rep(struct cm_rep_msg * rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_rep_param * param)2064 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2065 struct cm_id_private *cm_id_priv,
2066 struct ib_cm_rep_param *param)
2067 {
2068 cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
2069 rep_msg->local_comm_id = cm_id_priv->id.local_id;
2070 rep_msg->remote_comm_id = cm_id_priv->id.remote_id;
2071 cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn));
2072 rep_msg->resp_resources = param->responder_resources;
2073 cm_rep_set_target_ack_delay(rep_msg,
2074 cm_id_priv->av.port->cm_dev->ack_delay);
2075 cm_rep_set_failover(rep_msg, param->failover_accepted);
2076 cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count);
2077 rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
2078
2079 if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2080 rep_msg->initiator_depth = param->initiator_depth;
2081 cm_rep_set_flow_ctrl(rep_msg, param->flow_control);
2082 cm_rep_set_srq(rep_msg, param->srq);
2083 cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num));
2084 } else {
2085 cm_rep_set_srq(rep_msg, 1);
2086 cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param->qp_num));
2087 }
2088
2089 if (param->private_data && param->private_data_len)
2090 memcpy(rep_msg->private_data, param->private_data,
2091 param->private_data_len);
2092 }
2093
ib_send_cm_rep(struct ib_cm_id * cm_id,struct ib_cm_rep_param * param)2094 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2095 struct ib_cm_rep_param *param)
2096 {
2097 struct cm_id_private *cm_id_priv;
2098 struct ib_mad_send_buf *msg;
2099 struct cm_rep_msg *rep_msg;
2100 unsigned long flags;
2101 int ret;
2102
2103 if (param->private_data &&
2104 param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2105 return -EINVAL;
2106
2107 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2108 spin_lock_irqsave(&cm_id_priv->lock, flags);
2109 if (cm_id->state != IB_CM_REQ_RCVD &&
2110 cm_id->state != IB_CM_MRA_REQ_SENT) {
2111 ret = -EINVAL;
2112 goto out;
2113 }
2114
2115 ret = cm_alloc_msg(cm_id_priv, &msg);
2116 if (ret)
2117 goto out;
2118
2119 rep_msg = (struct cm_rep_msg *) msg->mad;
2120 cm_format_rep(rep_msg, cm_id_priv, param);
2121 msg->timeout_ms = cm_id_priv->timeout_ms;
2122 msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2123
2124 ret = ib_post_send_mad(msg, NULL);
2125 if (ret) {
2126 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2127 cm_free_msg(msg);
2128 return ret;
2129 }
2130
2131 cm_id->state = IB_CM_REP_SENT;
2132 cm_id_priv->msg = msg;
2133 cm_id_priv->initiator_depth = param->initiator_depth;
2134 cm_id_priv->responder_resources = param->responder_resources;
2135 cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg);
2136 cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2137
2138 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2139 return ret;
2140 }
2141 EXPORT_SYMBOL(ib_send_cm_rep);
2142
cm_format_rtu(struct cm_rtu_msg * rtu_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2143 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2144 struct cm_id_private *cm_id_priv,
2145 const void *private_data,
2146 u8 private_data_len)
2147 {
2148 cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2149 rtu_msg->local_comm_id = cm_id_priv->id.local_id;
2150 rtu_msg->remote_comm_id = cm_id_priv->id.remote_id;
2151
2152 if (private_data && private_data_len)
2153 memcpy(rtu_msg->private_data, private_data, private_data_len);
2154 }
2155
ib_send_cm_rtu(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2156 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2157 const void *private_data,
2158 u8 private_data_len)
2159 {
2160 struct cm_id_private *cm_id_priv;
2161 struct ib_mad_send_buf *msg;
2162 unsigned long flags;
2163 void *data;
2164 int ret;
2165
2166 if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2167 return -EINVAL;
2168
2169 data = cm_copy_private_data(private_data, private_data_len);
2170 if (IS_ERR(data))
2171 return PTR_ERR(data);
2172
2173 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2174 spin_lock_irqsave(&cm_id_priv->lock, flags);
2175 if (cm_id->state != IB_CM_REP_RCVD &&
2176 cm_id->state != IB_CM_MRA_REP_SENT) {
2177 ret = -EINVAL;
2178 goto error;
2179 }
2180
2181 ret = cm_alloc_msg(cm_id_priv, &msg);
2182 if (ret)
2183 goto error;
2184
2185 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2186 private_data, private_data_len);
2187
2188 ret = ib_post_send_mad(msg, NULL);
2189 if (ret) {
2190 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2191 cm_free_msg(msg);
2192 kfree(data);
2193 return ret;
2194 }
2195
2196 cm_id->state = IB_CM_ESTABLISHED;
2197 cm_set_private_data(cm_id_priv, data, private_data_len);
2198 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2199 return 0;
2200
2201 error: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2202 kfree(data);
2203 return ret;
2204 }
2205 EXPORT_SYMBOL(ib_send_cm_rtu);
2206
cm_format_rep_event(struct cm_work * work,enum ib_qp_type qp_type)2207 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2208 {
2209 struct cm_rep_msg *rep_msg;
2210 struct ib_cm_rep_event_param *param;
2211
2212 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2213 param = &work->cm_event.param.rep_rcvd;
2214 param->remote_ca_guid = rep_msg->local_ca_guid;
2215 param->remote_qkey = be32_to_cpu(rep_msg->local_qkey);
2216 param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2217 param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg));
2218 param->responder_resources = rep_msg->initiator_depth;
2219 param->initiator_depth = rep_msg->resp_resources;
2220 param->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
2221 param->failover_accepted = cm_rep_get_failover(rep_msg);
2222 param->flow_control = cm_rep_get_flow_ctrl(rep_msg);
2223 param->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
2224 param->srq = cm_rep_get_srq(rep_msg);
2225 work->cm_event.private_data = &rep_msg->private_data;
2226 }
2227
cm_dup_rep_handler(struct cm_work * work)2228 static void cm_dup_rep_handler(struct cm_work *work)
2229 {
2230 struct cm_id_private *cm_id_priv;
2231 struct cm_rep_msg *rep_msg;
2232 struct ib_mad_send_buf *msg = NULL;
2233 int ret;
2234
2235 rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2236 cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id,
2237 rep_msg->local_comm_id);
2238 if (!cm_id_priv)
2239 return;
2240
2241 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2242 counter[CM_REP_COUNTER]);
2243 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2244 if (ret)
2245 goto deref;
2246
2247 spin_lock_irq(&cm_id_priv->lock);
2248 if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2249 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2250 cm_id_priv->private_data,
2251 cm_id_priv->private_data_len);
2252 else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2253 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2254 CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2255 cm_id_priv->private_data,
2256 cm_id_priv->private_data_len);
2257 else
2258 goto unlock;
2259 spin_unlock_irq(&cm_id_priv->lock);
2260
2261 ret = ib_post_send_mad(msg, NULL);
2262 if (ret)
2263 goto free;
2264 goto deref;
2265
2266 unlock: spin_unlock_irq(&cm_id_priv->lock);
2267 free: cm_free_msg(msg);
2268 deref: cm_deref_id(cm_id_priv);
2269 }
2270
cm_rep_handler(struct cm_work * work)2271 static int cm_rep_handler(struct cm_work *work)
2272 {
2273 struct cm_id_private *cm_id_priv;
2274 struct cm_rep_msg *rep_msg;
2275 int ret;
2276 struct cm_id_private *cur_cm_id_priv;
2277 struct ib_cm_id *cm_id;
2278 struct cm_timewait_info *timewait_info;
2279
2280 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2281 cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0);
2282 if (!cm_id_priv) {
2283 cm_dup_rep_handler(work);
2284 return -EINVAL;
2285 }
2286
2287 cm_format_rep_event(work, cm_id_priv->qp_type);
2288
2289 spin_lock_irq(&cm_id_priv->lock);
2290 switch (cm_id_priv->id.state) {
2291 case IB_CM_REQ_SENT:
2292 case IB_CM_MRA_REQ_RCVD:
2293 break;
2294 default:
2295 spin_unlock_irq(&cm_id_priv->lock);
2296 ret = -EINVAL;
2297 goto error;
2298 }
2299
2300 cm_id_priv->timewait_info->work.remote_id = rep_msg->local_comm_id;
2301 cm_id_priv->timewait_info->remote_ca_guid = rep_msg->local_ca_guid;
2302 cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2303
2304 spin_lock(&cm.lock);
2305 /* Check for duplicate REP. */
2306 if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2307 spin_unlock(&cm.lock);
2308 spin_unlock_irq(&cm_id_priv->lock);
2309 ret = -EINVAL;
2310 goto error;
2311 }
2312 /* Check for a stale connection. */
2313 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2314 if (timewait_info) {
2315 rb_erase(&cm_id_priv->timewait_info->remote_id_node,
2316 &cm.remote_id_table);
2317 cm_id_priv->timewait_info->inserted_remote_id = 0;
2318 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
2319 timewait_info->work.remote_id);
2320
2321 spin_unlock(&cm.lock);
2322 spin_unlock_irq(&cm_id_priv->lock);
2323 cm_issue_rej(work->port, work->mad_recv_wc,
2324 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2325 NULL, 0);
2326 ret = -EINVAL;
2327 if (cur_cm_id_priv) {
2328 cm_id = &cur_cm_id_priv->id;
2329 ib_send_cm_dreq(cm_id, NULL, 0);
2330 cm_deref_id(cur_cm_id_priv);
2331 }
2332
2333 goto error;
2334 }
2335 spin_unlock(&cm.lock);
2336
2337 cm_id_priv->id.state = IB_CM_REP_RCVD;
2338 cm_id_priv->id.remote_id = rep_msg->local_comm_id;
2339 cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2340 cm_id_priv->initiator_depth = rep_msg->resp_resources;
2341 cm_id_priv->responder_resources = rep_msg->initiator_depth;
2342 cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg);
2343 cm_id_priv->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
2344 cm_id_priv->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
2345 cm_id_priv->av.timeout =
2346 cm_ack_timeout(cm_id_priv->target_ack_delay,
2347 cm_id_priv->av.timeout - 1);
2348 cm_id_priv->alt_av.timeout =
2349 cm_ack_timeout(cm_id_priv->target_ack_delay,
2350 cm_id_priv->alt_av.timeout - 1);
2351
2352 /* todo: handle peer_to_peer */
2353
2354 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2355 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2356 if (!ret)
2357 list_add_tail(&work->list, &cm_id_priv->work_list);
2358 spin_unlock_irq(&cm_id_priv->lock);
2359
2360 if (ret)
2361 cm_process_work(cm_id_priv, work);
2362 else
2363 cm_deref_id(cm_id_priv);
2364 return 0;
2365
2366 error:
2367 cm_deref_id(cm_id_priv);
2368 return ret;
2369 }
2370
cm_establish_handler(struct cm_work * work)2371 static int cm_establish_handler(struct cm_work *work)
2372 {
2373 struct cm_id_private *cm_id_priv;
2374 int ret;
2375
2376 /* See comment in cm_establish about lookup. */
2377 cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2378 if (!cm_id_priv)
2379 return -EINVAL;
2380
2381 spin_lock_irq(&cm_id_priv->lock);
2382 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2383 spin_unlock_irq(&cm_id_priv->lock);
2384 goto out;
2385 }
2386
2387 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2388 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2389 if (!ret)
2390 list_add_tail(&work->list, &cm_id_priv->work_list);
2391 spin_unlock_irq(&cm_id_priv->lock);
2392
2393 if (ret)
2394 cm_process_work(cm_id_priv, work);
2395 else
2396 cm_deref_id(cm_id_priv);
2397 return 0;
2398 out:
2399 cm_deref_id(cm_id_priv);
2400 return -EINVAL;
2401 }
2402
cm_rtu_handler(struct cm_work * work)2403 static int cm_rtu_handler(struct cm_work *work)
2404 {
2405 struct cm_id_private *cm_id_priv;
2406 struct cm_rtu_msg *rtu_msg;
2407 int ret;
2408
2409 rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2410 cm_id_priv = cm_acquire_id(rtu_msg->remote_comm_id,
2411 rtu_msg->local_comm_id);
2412 if (!cm_id_priv)
2413 return -EINVAL;
2414
2415 work->cm_event.private_data = &rtu_msg->private_data;
2416
2417 spin_lock_irq(&cm_id_priv->lock);
2418 if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2419 cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2420 spin_unlock_irq(&cm_id_priv->lock);
2421 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2422 counter[CM_RTU_COUNTER]);
2423 goto out;
2424 }
2425 cm_id_priv->id.state = IB_CM_ESTABLISHED;
2426
2427 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2428 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2429 if (!ret)
2430 list_add_tail(&work->list, &cm_id_priv->work_list);
2431 spin_unlock_irq(&cm_id_priv->lock);
2432
2433 if (ret)
2434 cm_process_work(cm_id_priv, work);
2435 else
2436 cm_deref_id(cm_id_priv);
2437 return 0;
2438 out:
2439 cm_deref_id(cm_id_priv);
2440 return -EINVAL;
2441 }
2442
cm_format_dreq(struct cm_dreq_msg * dreq_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2443 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2444 struct cm_id_private *cm_id_priv,
2445 const void *private_data,
2446 u8 private_data_len)
2447 {
2448 cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2449 cm_form_tid(cm_id_priv));
2450 dreq_msg->local_comm_id = cm_id_priv->id.local_id;
2451 dreq_msg->remote_comm_id = cm_id_priv->id.remote_id;
2452 cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv->remote_qpn);
2453
2454 if (private_data && private_data_len)
2455 memcpy(dreq_msg->private_data, private_data, private_data_len);
2456 }
2457
ib_send_cm_dreq(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2458 int ib_send_cm_dreq(struct ib_cm_id *cm_id,
2459 const void *private_data,
2460 u8 private_data_len)
2461 {
2462 struct cm_id_private *cm_id_priv;
2463 struct ib_mad_send_buf *msg;
2464 unsigned long flags;
2465 int ret;
2466
2467 if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2468 return -EINVAL;
2469
2470 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2471 spin_lock_irqsave(&cm_id_priv->lock, flags);
2472 if (cm_id->state != IB_CM_ESTABLISHED) {
2473 ret = -EINVAL;
2474 goto out;
2475 }
2476
2477 if (cm_id->lap_state == IB_CM_LAP_SENT ||
2478 cm_id->lap_state == IB_CM_MRA_LAP_RCVD)
2479 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2480
2481 ret = cm_alloc_msg(cm_id_priv, &msg);
2482 if (ret) {
2483 cm_enter_timewait(cm_id_priv);
2484 goto out;
2485 }
2486
2487 cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2488 private_data, private_data_len);
2489 msg->timeout_ms = cm_id_priv->timeout_ms;
2490 msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2491
2492 ret = ib_post_send_mad(msg, NULL);
2493 if (ret) {
2494 cm_enter_timewait(cm_id_priv);
2495 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2496 cm_free_msg(msg);
2497 return ret;
2498 }
2499
2500 cm_id->state = IB_CM_DREQ_SENT;
2501 cm_id_priv->msg = msg;
2502 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2503 return ret;
2504 }
2505 EXPORT_SYMBOL(ib_send_cm_dreq);
2506
cm_format_drep(struct cm_drep_msg * drep_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2507 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2508 struct cm_id_private *cm_id_priv,
2509 const void *private_data,
2510 u8 private_data_len)
2511 {
2512 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2513 drep_msg->local_comm_id = cm_id_priv->id.local_id;
2514 drep_msg->remote_comm_id = cm_id_priv->id.remote_id;
2515
2516 if (private_data && private_data_len)
2517 memcpy(drep_msg->private_data, private_data, private_data_len);
2518 }
2519
ib_send_cm_drep(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2520 int ib_send_cm_drep(struct ib_cm_id *cm_id,
2521 const void *private_data,
2522 u8 private_data_len)
2523 {
2524 struct cm_id_private *cm_id_priv;
2525 struct ib_mad_send_buf *msg;
2526 unsigned long flags;
2527 void *data;
2528 int ret;
2529
2530 if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2531 return -EINVAL;
2532
2533 data = cm_copy_private_data(private_data, private_data_len);
2534 if (IS_ERR(data))
2535 return PTR_ERR(data);
2536
2537 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2538 spin_lock_irqsave(&cm_id_priv->lock, flags);
2539 if (cm_id->state != IB_CM_DREQ_RCVD) {
2540 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2541 kfree(data);
2542 return -EINVAL;
2543 }
2544
2545 cm_set_private_data(cm_id_priv, data, private_data_len);
2546 cm_enter_timewait(cm_id_priv);
2547
2548 ret = cm_alloc_msg(cm_id_priv, &msg);
2549 if (ret)
2550 goto out;
2551
2552 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2553 private_data, private_data_len);
2554
2555 ret = ib_post_send_mad(msg, NULL);
2556 if (ret) {
2557 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2558 cm_free_msg(msg);
2559 return ret;
2560 }
2561
2562 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2563 return ret;
2564 }
2565 EXPORT_SYMBOL(ib_send_cm_drep);
2566
cm_issue_drep(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)2567 static int cm_issue_drep(struct cm_port *port,
2568 struct ib_mad_recv_wc *mad_recv_wc)
2569 {
2570 struct ib_mad_send_buf *msg = NULL;
2571 struct cm_dreq_msg *dreq_msg;
2572 struct cm_drep_msg *drep_msg;
2573 int ret;
2574
2575 ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2576 if (ret)
2577 return ret;
2578
2579 dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2580 drep_msg = (struct cm_drep_msg *) msg->mad;
2581
2582 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2583 drep_msg->remote_comm_id = dreq_msg->local_comm_id;
2584 drep_msg->local_comm_id = dreq_msg->remote_comm_id;
2585
2586 ret = ib_post_send_mad(msg, NULL);
2587 if (ret)
2588 cm_free_msg(msg);
2589
2590 return ret;
2591 }
2592
cm_dreq_handler(struct cm_work * work)2593 static int cm_dreq_handler(struct cm_work *work)
2594 {
2595 struct cm_id_private *cm_id_priv;
2596 struct cm_dreq_msg *dreq_msg;
2597 struct ib_mad_send_buf *msg = NULL;
2598 int ret;
2599
2600 dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2601 cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id,
2602 dreq_msg->local_comm_id);
2603 if (!cm_id_priv) {
2604 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2605 counter[CM_DREQ_COUNTER]);
2606 cm_issue_drep(work->port, work->mad_recv_wc);
2607 return -EINVAL;
2608 }
2609
2610 work->cm_event.private_data = &dreq_msg->private_data;
2611
2612 spin_lock_irq(&cm_id_priv->lock);
2613 if (cm_id_priv->local_qpn != cm_dreq_get_remote_qpn(dreq_msg))
2614 goto unlock;
2615
2616 switch (cm_id_priv->id.state) {
2617 case IB_CM_REP_SENT:
2618 case IB_CM_DREQ_SENT:
2619 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2620 break;
2621 case IB_CM_ESTABLISHED:
2622 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2623 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2624 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2625 break;
2626 case IB_CM_MRA_REP_RCVD:
2627 break;
2628 case IB_CM_TIMEWAIT:
2629 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2630 counter[CM_DREQ_COUNTER]);
2631 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2632 if (IS_ERR(msg))
2633 goto unlock;
2634
2635 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2636 cm_id_priv->private_data,
2637 cm_id_priv->private_data_len);
2638 spin_unlock_irq(&cm_id_priv->lock);
2639
2640 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2641 ib_post_send_mad(msg, NULL))
2642 cm_free_msg(msg);
2643 goto deref;
2644 case IB_CM_DREQ_RCVD:
2645 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2646 counter[CM_DREQ_COUNTER]);
2647 goto unlock;
2648 default:
2649 goto unlock;
2650 }
2651 cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2652 cm_id_priv->tid = dreq_msg->hdr.tid;
2653 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2654 if (!ret)
2655 list_add_tail(&work->list, &cm_id_priv->work_list);
2656 spin_unlock_irq(&cm_id_priv->lock);
2657
2658 if (ret)
2659 cm_process_work(cm_id_priv, work);
2660 else
2661 cm_deref_id(cm_id_priv);
2662 return 0;
2663
2664 unlock: spin_unlock_irq(&cm_id_priv->lock);
2665 deref: cm_deref_id(cm_id_priv);
2666 return -EINVAL;
2667 }
2668
cm_drep_handler(struct cm_work * work)2669 static int cm_drep_handler(struct cm_work *work)
2670 {
2671 struct cm_id_private *cm_id_priv;
2672 struct cm_drep_msg *drep_msg;
2673 int ret;
2674
2675 drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2676 cm_id_priv = cm_acquire_id(drep_msg->remote_comm_id,
2677 drep_msg->local_comm_id);
2678 if (!cm_id_priv)
2679 return -EINVAL;
2680
2681 work->cm_event.private_data = &drep_msg->private_data;
2682
2683 spin_lock_irq(&cm_id_priv->lock);
2684 if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2685 cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2686 spin_unlock_irq(&cm_id_priv->lock);
2687 goto out;
2688 }
2689 cm_enter_timewait(cm_id_priv);
2690
2691 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2692 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2693 if (!ret)
2694 list_add_tail(&work->list, &cm_id_priv->work_list);
2695 spin_unlock_irq(&cm_id_priv->lock);
2696
2697 if (ret)
2698 cm_process_work(cm_id_priv, work);
2699 else
2700 cm_deref_id(cm_id_priv);
2701 return 0;
2702 out:
2703 cm_deref_id(cm_id_priv);
2704 return -EINVAL;
2705 }
2706
ib_send_cm_rej(struct ib_cm_id * cm_id,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2707 int ib_send_cm_rej(struct ib_cm_id *cm_id,
2708 enum ib_cm_rej_reason reason,
2709 void *ari,
2710 u8 ari_length,
2711 const void *private_data,
2712 u8 private_data_len)
2713 {
2714 struct cm_id_private *cm_id_priv;
2715 struct ib_mad_send_buf *msg;
2716 unsigned long flags;
2717 int ret;
2718
2719 if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2720 (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2721 return -EINVAL;
2722
2723 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2724
2725 spin_lock_irqsave(&cm_id_priv->lock, flags);
2726 switch (cm_id->state) {
2727 case IB_CM_REQ_SENT:
2728 case IB_CM_MRA_REQ_RCVD:
2729 case IB_CM_REQ_RCVD:
2730 case IB_CM_MRA_REQ_SENT:
2731 case IB_CM_REP_RCVD:
2732 case IB_CM_MRA_REP_SENT:
2733 ret = cm_alloc_msg(cm_id_priv, &msg);
2734 if (!ret)
2735 cm_format_rej((struct cm_rej_msg *) msg->mad,
2736 cm_id_priv, reason, ari, ari_length,
2737 private_data, private_data_len);
2738
2739 cm_reset_to_idle(cm_id_priv);
2740 break;
2741 case IB_CM_REP_SENT:
2742 case IB_CM_MRA_REP_RCVD:
2743 ret = cm_alloc_msg(cm_id_priv, &msg);
2744 if (!ret)
2745 cm_format_rej((struct cm_rej_msg *) msg->mad,
2746 cm_id_priv, reason, ari, ari_length,
2747 private_data, private_data_len);
2748
2749 cm_enter_timewait(cm_id_priv);
2750 break;
2751 default:
2752 ret = -EINVAL;
2753 goto out;
2754 }
2755
2756 if (ret)
2757 goto out;
2758
2759 ret = ib_post_send_mad(msg, NULL);
2760 if (ret)
2761 cm_free_msg(msg);
2762
2763 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2764 return ret;
2765 }
2766 EXPORT_SYMBOL(ib_send_cm_rej);
2767
cm_format_rej_event(struct cm_work * work)2768 static void cm_format_rej_event(struct cm_work *work)
2769 {
2770 struct cm_rej_msg *rej_msg;
2771 struct ib_cm_rej_event_param *param;
2772
2773 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2774 param = &work->cm_event.param.rej_rcvd;
2775 param->ari = rej_msg->ari;
2776 param->ari_length = cm_rej_get_reject_info_len(rej_msg);
2777 param->reason = __be16_to_cpu(rej_msg->reason);
2778 work->cm_event.private_data = &rej_msg->private_data;
2779 }
2780
cm_acquire_rejected_id(struct cm_rej_msg * rej_msg)2781 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2782 {
2783 struct cm_timewait_info *timewait_info;
2784 struct cm_id_private *cm_id_priv;
2785 __be32 remote_id;
2786
2787 remote_id = rej_msg->local_comm_id;
2788
2789 if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_TIMEOUT) {
2790 spin_lock_irq(&cm.lock);
2791 timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari),
2792 remote_id);
2793 if (!timewait_info) {
2794 spin_unlock_irq(&cm.lock);
2795 return NULL;
2796 }
2797 cm_id_priv = idr_find(&cm.local_id_table, (__force int)
2798 (timewait_info->work.local_id ^
2799 cm.random_id_operand));
2800 if (cm_id_priv) {
2801 if (cm_id_priv->id.remote_id == remote_id)
2802 atomic_inc(&cm_id_priv->refcount);
2803 else
2804 cm_id_priv = NULL;
2805 }
2806 spin_unlock_irq(&cm.lock);
2807 } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ)
2808 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0);
2809 else
2810 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id);
2811
2812 return cm_id_priv;
2813 }
2814
cm_rej_handler(struct cm_work * work)2815 static int cm_rej_handler(struct cm_work *work)
2816 {
2817 struct cm_id_private *cm_id_priv;
2818 struct cm_rej_msg *rej_msg;
2819 int ret;
2820
2821 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2822 cm_id_priv = cm_acquire_rejected_id(rej_msg);
2823 if (!cm_id_priv)
2824 return -EINVAL;
2825
2826 cm_format_rej_event(work);
2827
2828 spin_lock_irq(&cm_id_priv->lock);
2829 switch (cm_id_priv->id.state) {
2830 case IB_CM_REQ_SENT:
2831 case IB_CM_MRA_REQ_RCVD:
2832 case IB_CM_REP_SENT:
2833 case IB_CM_MRA_REP_RCVD:
2834 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2835 /* fall through */
2836 case IB_CM_REQ_RCVD:
2837 case IB_CM_MRA_REQ_SENT:
2838 if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_STALE_CONN)
2839 cm_enter_timewait(cm_id_priv);
2840 else
2841 cm_reset_to_idle(cm_id_priv);
2842 break;
2843 case IB_CM_DREQ_SENT:
2844 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2845 /* fall through */
2846 case IB_CM_REP_RCVD:
2847 case IB_CM_MRA_REP_SENT:
2848 cm_enter_timewait(cm_id_priv);
2849 break;
2850 case IB_CM_ESTABLISHED:
2851 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
2852 cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
2853 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
2854 ib_cancel_mad(cm_id_priv->av.port->mad_agent,
2855 cm_id_priv->msg);
2856 cm_enter_timewait(cm_id_priv);
2857 break;
2858 }
2859 /* fall through */
2860 default:
2861 spin_unlock_irq(&cm_id_priv->lock);
2862 ret = -EINVAL;
2863 goto out;
2864 }
2865
2866 ret = atomic_inc_and_test(&cm_id_priv->work_count);
2867 if (!ret)
2868 list_add_tail(&work->list, &cm_id_priv->work_list);
2869 spin_unlock_irq(&cm_id_priv->lock);
2870
2871 if (ret)
2872 cm_process_work(cm_id_priv, work);
2873 else
2874 cm_deref_id(cm_id_priv);
2875 return 0;
2876 out:
2877 cm_deref_id(cm_id_priv);
2878 return -EINVAL;
2879 }
2880
ib_send_cm_mra(struct ib_cm_id * cm_id,u8 service_timeout,const void * private_data,u8 private_data_len)2881 int ib_send_cm_mra(struct ib_cm_id *cm_id,
2882 u8 service_timeout,
2883 const void *private_data,
2884 u8 private_data_len)
2885 {
2886 struct cm_id_private *cm_id_priv;
2887 struct ib_mad_send_buf *msg;
2888 enum ib_cm_state cm_state;
2889 enum ib_cm_lap_state lap_state;
2890 enum cm_msg_response msg_response;
2891 void *data;
2892 unsigned long flags;
2893 int ret;
2894
2895 if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
2896 return -EINVAL;
2897
2898 data = cm_copy_private_data(private_data, private_data_len);
2899 if (IS_ERR(data))
2900 return PTR_ERR(data);
2901
2902 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2903
2904 spin_lock_irqsave(&cm_id_priv->lock, flags);
2905 switch(cm_id_priv->id.state) {
2906 case IB_CM_REQ_RCVD:
2907 cm_state = IB_CM_MRA_REQ_SENT;
2908 lap_state = cm_id->lap_state;
2909 msg_response = CM_MSG_RESPONSE_REQ;
2910 break;
2911 case IB_CM_REP_RCVD:
2912 cm_state = IB_CM_MRA_REP_SENT;
2913 lap_state = cm_id->lap_state;
2914 msg_response = CM_MSG_RESPONSE_REP;
2915 break;
2916 case IB_CM_ESTABLISHED:
2917 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
2918 cm_state = cm_id->state;
2919 lap_state = IB_CM_MRA_LAP_SENT;
2920 msg_response = CM_MSG_RESPONSE_OTHER;
2921 break;
2922 }
2923 default:
2924 ret = -EINVAL;
2925 goto error1;
2926 }
2927
2928 if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
2929 ret = cm_alloc_msg(cm_id_priv, &msg);
2930 if (ret)
2931 goto error1;
2932
2933 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2934 msg_response, service_timeout,
2935 private_data, private_data_len);
2936 ret = ib_post_send_mad(msg, NULL);
2937 if (ret)
2938 goto error2;
2939 }
2940
2941 cm_id->state = cm_state;
2942 cm_id->lap_state = lap_state;
2943 cm_id_priv->service_timeout = service_timeout;
2944 cm_set_private_data(cm_id_priv, data, private_data_len);
2945 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2946 return 0;
2947
2948 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2949 kfree(data);
2950 return ret;
2951
2952 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2953 kfree(data);
2954 cm_free_msg(msg);
2955 return ret;
2956 }
2957 EXPORT_SYMBOL(ib_send_cm_mra);
2958
cm_acquire_mraed_id(struct cm_mra_msg * mra_msg)2959 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
2960 {
2961 switch (cm_mra_get_msg_mraed(mra_msg)) {
2962 case CM_MSG_RESPONSE_REQ:
2963 return cm_acquire_id(mra_msg->remote_comm_id, 0);
2964 case CM_MSG_RESPONSE_REP:
2965 case CM_MSG_RESPONSE_OTHER:
2966 return cm_acquire_id(mra_msg->remote_comm_id,
2967 mra_msg->local_comm_id);
2968 default:
2969 return NULL;
2970 }
2971 }
2972
cm_mra_handler(struct cm_work * work)2973 static int cm_mra_handler(struct cm_work *work)
2974 {
2975 struct cm_id_private *cm_id_priv;
2976 struct cm_mra_msg *mra_msg;
2977 int timeout, ret;
2978
2979 mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
2980 cm_id_priv = cm_acquire_mraed_id(mra_msg);
2981 if (!cm_id_priv)
2982 return -EINVAL;
2983
2984 work->cm_event.private_data = &mra_msg->private_data;
2985 work->cm_event.param.mra_rcvd.service_timeout =
2986 cm_mra_get_service_timeout(mra_msg);
2987 timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) +
2988 cm_convert_to_ms(cm_id_priv->av.timeout);
2989
2990 spin_lock_irq(&cm_id_priv->lock);
2991 switch (cm_id_priv->id.state) {
2992 case IB_CM_REQ_SENT:
2993 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ ||
2994 ib_modify_mad(cm_id_priv->av.port->mad_agent,
2995 cm_id_priv->msg, timeout))
2996 goto out;
2997 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
2998 break;
2999 case IB_CM_REP_SENT:
3000 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP ||
3001 ib_modify_mad(cm_id_priv->av.port->mad_agent,
3002 cm_id_priv->msg, timeout))
3003 goto out;
3004 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3005 break;
3006 case IB_CM_ESTABLISHED:
3007 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER ||
3008 cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3009 ib_modify_mad(cm_id_priv->av.port->mad_agent,
3010 cm_id_priv->msg, timeout)) {
3011 if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3012 atomic_long_inc(&work->port->
3013 counter_group[CM_RECV_DUPLICATES].
3014 counter[CM_MRA_COUNTER]);
3015 goto out;
3016 }
3017 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3018 break;
3019 case IB_CM_MRA_REQ_RCVD:
3020 case IB_CM_MRA_REP_RCVD:
3021 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3022 counter[CM_MRA_COUNTER]);
3023 /* fall through */
3024 default:
3025 goto out;
3026 }
3027
3028 cm_id_priv->msg->context[1] = (void *) (unsigned long)
3029 cm_id_priv->id.state;
3030 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3031 if (!ret)
3032 list_add_tail(&work->list, &cm_id_priv->work_list);
3033 spin_unlock_irq(&cm_id_priv->lock);
3034
3035 if (ret)
3036 cm_process_work(cm_id_priv, work);
3037 else
3038 cm_deref_id(cm_id_priv);
3039 return 0;
3040 out:
3041 spin_unlock_irq(&cm_id_priv->lock);
3042 cm_deref_id(cm_id_priv);
3043 return -EINVAL;
3044 }
3045
cm_format_lap(struct cm_lap_msg * lap_msg,struct cm_id_private * cm_id_priv,struct sa_path_rec * alternate_path,const void * private_data,u8 private_data_len)3046 static void cm_format_lap(struct cm_lap_msg *lap_msg,
3047 struct cm_id_private *cm_id_priv,
3048 struct sa_path_rec *alternate_path,
3049 const void *private_data,
3050 u8 private_data_len)
3051 {
3052 bool alt_ext = false;
3053
3054 if (alternate_path->rec_type == SA_PATH_REC_TYPE_OPA)
3055 alt_ext = opa_is_extended_lid(alternate_path->opa.dlid,
3056 alternate_path->opa.slid);
3057 cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID,
3058 cm_form_tid(cm_id_priv));
3059 lap_msg->local_comm_id = cm_id_priv->id.local_id;
3060 lap_msg->remote_comm_id = cm_id_priv->id.remote_id;
3061 cm_lap_set_remote_qpn(lap_msg, cm_id_priv->remote_qpn);
3062 /* todo: need remote CM response timeout */
3063 cm_lap_set_remote_resp_timeout(lap_msg, 0x1F);
3064 lap_msg->alt_local_lid =
3065 htons(ntohl(sa_path_get_slid(alternate_path)));
3066 lap_msg->alt_remote_lid =
3067 htons(ntohl(sa_path_get_dlid(alternate_path)));
3068 lap_msg->alt_local_gid = alternate_path->sgid;
3069 lap_msg->alt_remote_gid = alternate_path->dgid;
3070 if (alt_ext) {
3071 lap_msg->alt_local_gid.global.interface_id
3072 = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.slid));
3073 lap_msg->alt_remote_gid.global.interface_id
3074 = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.dlid));
3075 }
3076 cm_lap_set_flow_label(lap_msg, alternate_path->flow_label);
3077 cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class);
3078 lap_msg->alt_hop_limit = alternate_path->hop_limit;
3079 cm_lap_set_packet_rate(lap_msg, alternate_path->rate);
3080 cm_lap_set_sl(lap_msg, alternate_path->sl);
3081 cm_lap_set_subnet_local(lap_msg, 1); /* local only... */
3082 cm_lap_set_local_ack_timeout(lap_msg,
3083 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
3084 alternate_path->packet_life_time));
3085
3086 if (private_data && private_data_len)
3087 memcpy(lap_msg->private_data, private_data, private_data_len);
3088 }
3089
ib_send_cm_lap(struct ib_cm_id * cm_id,struct sa_path_rec * alternate_path,const void * private_data,u8 private_data_len)3090 int ib_send_cm_lap(struct ib_cm_id *cm_id,
3091 struct sa_path_rec *alternate_path,
3092 const void *private_data,
3093 u8 private_data_len)
3094 {
3095 struct cm_id_private *cm_id_priv;
3096 struct ib_mad_send_buf *msg;
3097 unsigned long flags;
3098 int ret;
3099
3100 if (private_data && private_data_len > IB_CM_LAP_PRIVATE_DATA_SIZE)
3101 return -EINVAL;
3102
3103 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3104 spin_lock_irqsave(&cm_id_priv->lock, flags);
3105 if (cm_id->state != IB_CM_ESTABLISHED ||
3106 (cm_id->lap_state != IB_CM_LAP_UNINIT &&
3107 cm_id->lap_state != IB_CM_LAP_IDLE)) {
3108 ret = -EINVAL;
3109 goto out;
3110 }
3111
3112 ret = cm_init_av_by_path(alternate_path, NULL, &cm_id_priv->alt_av,
3113 cm_id_priv);
3114 if (ret)
3115 goto out;
3116 cm_id_priv->alt_av.timeout =
3117 cm_ack_timeout(cm_id_priv->target_ack_delay,
3118 cm_id_priv->alt_av.timeout - 1);
3119
3120 ret = cm_alloc_msg(cm_id_priv, &msg);
3121 if (ret)
3122 goto out;
3123
3124 cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv,
3125 alternate_path, private_data, private_data_len);
3126 msg->timeout_ms = cm_id_priv->timeout_ms;
3127 msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED;
3128
3129 ret = ib_post_send_mad(msg, NULL);
3130 if (ret) {
3131 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3132 cm_free_msg(msg);
3133 return ret;
3134 }
3135
3136 cm_id->lap_state = IB_CM_LAP_SENT;
3137 cm_id_priv->msg = msg;
3138
3139 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3140 return ret;
3141 }
3142 EXPORT_SYMBOL(ib_send_cm_lap);
3143
cm_format_path_lid_from_lap(struct cm_lap_msg * lap_msg,struct sa_path_rec * path)3144 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3145 struct sa_path_rec *path)
3146 {
3147 u32 lid;
3148
3149 if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3150 sa_path_set_dlid(path, htonl(ntohs(lap_msg->alt_local_lid)));
3151 sa_path_set_slid(path, htonl(ntohs(lap_msg->alt_remote_lid)));
3152 } else {
3153 lid = opa_get_lid_from_gid(&lap_msg->alt_local_gid);
3154 sa_path_set_dlid(path, cpu_to_be32(lid));
3155
3156 lid = opa_get_lid_from_gid(&lap_msg->alt_remote_gid);
3157 sa_path_set_slid(path, cpu_to_be32(lid));
3158 }
3159 }
3160
cm_format_path_from_lap(struct cm_id_private * cm_id_priv,struct sa_path_rec * path,struct cm_lap_msg * lap_msg)3161 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3162 struct sa_path_rec *path,
3163 struct cm_lap_msg *lap_msg)
3164 {
3165 path->dgid = lap_msg->alt_local_gid;
3166 path->sgid = lap_msg->alt_remote_gid;
3167 path->flow_label = cm_lap_get_flow_label(lap_msg);
3168 path->hop_limit = lap_msg->alt_hop_limit;
3169 path->traffic_class = cm_lap_get_traffic_class(lap_msg);
3170 path->reversible = 1;
3171 path->pkey = cm_id_priv->pkey;
3172 path->sl = cm_lap_get_sl(lap_msg);
3173 path->mtu_selector = IB_SA_EQ;
3174 path->mtu = cm_id_priv->path_mtu;
3175 path->rate_selector = IB_SA_EQ;
3176 path->rate = cm_lap_get_packet_rate(lap_msg);
3177 path->packet_life_time_selector = IB_SA_EQ;
3178 path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg);
3179 path->packet_life_time -= (path->packet_life_time > 0);
3180 cm_format_path_lid_from_lap(lap_msg, path);
3181 }
3182
cm_lap_handler(struct cm_work * work)3183 static int cm_lap_handler(struct cm_work *work)
3184 {
3185 struct cm_id_private *cm_id_priv;
3186 struct cm_lap_msg *lap_msg;
3187 struct ib_cm_lap_event_param *param;
3188 struct ib_mad_send_buf *msg = NULL;
3189 int ret;
3190
3191 /* Currently Alternate path messages are not supported for
3192 * RoCE link layer.
3193 */
3194 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3195 work->port->port_num))
3196 return -EINVAL;
3197
3198 /* todo: verify LAP request and send reject APR if invalid. */
3199 lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3200 cm_id_priv = cm_acquire_id(lap_msg->remote_comm_id,
3201 lap_msg->local_comm_id);
3202 if (!cm_id_priv)
3203 return -EINVAL;
3204
3205 param = &work->cm_event.param.lap_rcvd;
3206 memset(&work->path[0], 0, sizeof(work->path[1]));
3207 cm_path_set_rec_type(work->port->cm_dev->ib_device,
3208 work->port->port_num,
3209 &work->path[0],
3210 &lap_msg->alt_local_gid);
3211 param->alternate_path = &work->path[0];
3212 cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3213 work->cm_event.private_data = &lap_msg->private_data;
3214
3215 spin_lock_irq(&cm_id_priv->lock);
3216 if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3217 goto unlock;
3218
3219 switch (cm_id_priv->id.lap_state) {
3220 case IB_CM_LAP_UNINIT:
3221 case IB_CM_LAP_IDLE:
3222 break;
3223 case IB_CM_MRA_LAP_SENT:
3224 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3225 counter[CM_LAP_COUNTER]);
3226 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3227 if (IS_ERR(msg))
3228 goto unlock;
3229
3230 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3231 CM_MSG_RESPONSE_OTHER,
3232 cm_id_priv->service_timeout,
3233 cm_id_priv->private_data,
3234 cm_id_priv->private_data_len);
3235 spin_unlock_irq(&cm_id_priv->lock);
3236
3237 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3238 ib_post_send_mad(msg, NULL))
3239 cm_free_msg(msg);
3240 goto deref;
3241 case IB_CM_LAP_RCVD:
3242 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3243 counter[CM_LAP_COUNTER]);
3244 goto unlock;
3245 default:
3246 goto unlock;
3247 }
3248
3249 ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3250 work->mad_recv_wc->recv_buf.grh,
3251 &cm_id_priv->av);
3252 if (ret)
3253 goto unlock;
3254
3255 ret = cm_init_av_by_path(param->alternate_path, NULL,
3256 &cm_id_priv->alt_av, cm_id_priv);
3257 if (ret)
3258 goto unlock;
3259
3260 cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3261 cm_id_priv->tid = lap_msg->hdr.tid;
3262 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3263 if (!ret)
3264 list_add_tail(&work->list, &cm_id_priv->work_list);
3265 spin_unlock_irq(&cm_id_priv->lock);
3266
3267 if (ret)
3268 cm_process_work(cm_id_priv, work);
3269 else
3270 cm_deref_id(cm_id_priv);
3271 return 0;
3272
3273 unlock: spin_unlock_irq(&cm_id_priv->lock);
3274 deref: cm_deref_id(cm_id_priv);
3275 return -EINVAL;
3276 }
3277
cm_format_apr(struct cm_apr_msg * apr_msg,struct cm_id_private * cm_id_priv,enum ib_cm_apr_status status,void * info,u8 info_length,const void * private_data,u8 private_data_len)3278 static void cm_format_apr(struct cm_apr_msg *apr_msg,
3279 struct cm_id_private *cm_id_priv,
3280 enum ib_cm_apr_status status,
3281 void *info,
3282 u8 info_length,
3283 const void *private_data,
3284 u8 private_data_len)
3285 {
3286 cm_format_mad_hdr(&apr_msg->hdr, CM_APR_ATTR_ID, cm_id_priv->tid);
3287 apr_msg->local_comm_id = cm_id_priv->id.local_id;
3288 apr_msg->remote_comm_id = cm_id_priv->id.remote_id;
3289 apr_msg->ap_status = (u8) status;
3290
3291 if (info && info_length) {
3292 apr_msg->info_length = info_length;
3293 memcpy(apr_msg->info, info, info_length);
3294 }
3295
3296 if (private_data && private_data_len)
3297 memcpy(apr_msg->private_data, private_data, private_data_len);
3298 }
3299
ib_send_cm_apr(struct ib_cm_id * cm_id,enum ib_cm_apr_status status,void * info,u8 info_length,const void * private_data,u8 private_data_len)3300 int ib_send_cm_apr(struct ib_cm_id *cm_id,
3301 enum ib_cm_apr_status status,
3302 void *info,
3303 u8 info_length,
3304 const void *private_data,
3305 u8 private_data_len)
3306 {
3307 struct cm_id_private *cm_id_priv;
3308 struct ib_mad_send_buf *msg;
3309 unsigned long flags;
3310 int ret;
3311
3312 if ((private_data && private_data_len > IB_CM_APR_PRIVATE_DATA_SIZE) ||
3313 (info && info_length > IB_CM_APR_INFO_LENGTH))
3314 return -EINVAL;
3315
3316 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3317 spin_lock_irqsave(&cm_id_priv->lock, flags);
3318 if (cm_id->state != IB_CM_ESTABLISHED ||
3319 (cm_id->lap_state != IB_CM_LAP_RCVD &&
3320 cm_id->lap_state != IB_CM_MRA_LAP_SENT)) {
3321 ret = -EINVAL;
3322 goto out;
3323 }
3324
3325 ret = cm_alloc_msg(cm_id_priv, &msg);
3326 if (ret)
3327 goto out;
3328
3329 cm_format_apr((struct cm_apr_msg *) msg->mad, cm_id_priv, status,
3330 info, info_length, private_data, private_data_len);
3331 ret = ib_post_send_mad(msg, NULL);
3332 if (ret) {
3333 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3334 cm_free_msg(msg);
3335 return ret;
3336 }
3337
3338 cm_id->lap_state = IB_CM_LAP_IDLE;
3339 out: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3340 return ret;
3341 }
3342 EXPORT_SYMBOL(ib_send_cm_apr);
3343
cm_apr_handler(struct cm_work * work)3344 static int cm_apr_handler(struct cm_work *work)
3345 {
3346 struct cm_id_private *cm_id_priv;
3347 struct cm_apr_msg *apr_msg;
3348 int ret;
3349
3350 /* Currently Alternate path messages are not supported for
3351 * RoCE link layer.
3352 */
3353 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3354 work->port->port_num))
3355 return -EINVAL;
3356
3357 apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3358 cm_id_priv = cm_acquire_id(apr_msg->remote_comm_id,
3359 apr_msg->local_comm_id);
3360 if (!cm_id_priv)
3361 return -EINVAL; /* Unmatched reply. */
3362
3363 work->cm_event.param.apr_rcvd.ap_status = apr_msg->ap_status;
3364 work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info;
3365 work->cm_event.param.apr_rcvd.info_len = apr_msg->info_length;
3366 work->cm_event.private_data = &apr_msg->private_data;
3367
3368 spin_lock_irq(&cm_id_priv->lock);
3369 if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3370 (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3371 cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3372 spin_unlock_irq(&cm_id_priv->lock);
3373 goto out;
3374 }
3375 cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3376 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3377 cm_id_priv->msg = NULL;
3378
3379 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3380 if (!ret)
3381 list_add_tail(&work->list, &cm_id_priv->work_list);
3382 spin_unlock_irq(&cm_id_priv->lock);
3383
3384 if (ret)
3385 cm_process_work(cm_id_priv, work);
3386 else
3387 cm_deref_id(cm_id_priv);
3388 return 0;
3389 out:
3390 cm_deref_id(cm_id_priv);
3391 return -EINVAL;
3392 }
3393
cm_timewait_handler(struct cm_work * work)3394 static int cm_timewait_handler(struct cm_work *work)
3395 {
3396 struct cm_timewait_info *timewait_info;
3397 struct cm_id_private *cm_id_priv;
3398 int ret;
3399
3400 timewait_info = (struct cm_timewait_info *)work;
3401 spin_lock_irq(&cm.lock);
3402 list_del(&timewait_info->list);
3403 spin_unlock_irq(&cm.lock);
3404
3405 cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3406 timewait_info->work.remote_id);
3407 if (!cm_id_priv)
3408 return -EINVAL;
3409
3410 spin_lock_irq(&cm_id_priv->lock);
3411 if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3412 cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3413 spin_unlock_irq(&cm_id_priv->lock);
3414 goto out;
3415 }
3416 cm_id_priv->id.state = IB_CM_IDLE;
3417 ret = atomic_inc_and_test(&cm_id_priv->work_count);
3418 if (!ret)
3419 list_add_tail(&work->list, &cm_id_priv->work_list);
3420 spin_unlock_irq(&cm_id_priv->lock);
3421
3422 if (ret)
3423 cm_process_work(cm_id_priv, work);
3424 else
3425 cm_deref_id(cm_id_priv);
3426 return 0;
3427 out:
3428 cm_deref_id(cm_id_priv);
3429 return -EINVAL;
3430 }
3431
cm_format_sidr_req(struct cm_sidr_req_msg * sidr_req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_req_param * param)3432 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3433 struct cm_id_private *cm_id_priv,
3434 struct ib_cm_sidr_req_param *param)
3435 {
3436 cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3437 cm_form_tid(cm_id_priv));
3438 sidr_req_msg->request_id = cm_id_priv->id.local_id;
3439 sidr_req_msg->pkey = param->path->pkey;
3440 sidr_req_msg->service_id = param->service_id;
3441
3442 if (param->private_data && param->private_data_len)
3443 memcpy(sidr_req_msg->private_data, param->private_data,
3444 param->private_data_len);
3445 }
3446
ib_send_cm_sidr_req(struct ib_cm_id * cm_id,struct ib_cm_sidr_req_param * param)3447 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3448 struct ib_cm_sidr_req_param *param)
3449 {
3450 struct cm_id_private *cm_id_priv;
3451 struct ib_mad_send_buf *msg;
3452 unsigned long flags;
3453 int ret;
3454
3455 if (!param->path || (param->private_data &&
3456 param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3457 return -EINVAL;
3458
3459 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3460 ret = cm_init_av_by_path(param->path, param->sgid_attr,
3461 &cm_id_priv->av,
3462 cm_id_priv);
3463 if (ret)
3464 goto out;
3465
3466 cm_id->service_id = param->service_id;
3467 cm_id->service_mask = ~cpu_to_be64(0);
3468 cm_id_priv->timeout_ms = param->timeout_ms;
3469 cm_id_priv->max_cm_retries = param->max_cm_retries;
3470 ret = cm_alloc_msg(cm_id_priv, &msg);
3471 if (ret)
3472 goto out;
3473
3474 cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv,
3475 param);
3476 msg->timeout_ms = cm_id_priv->timeout_ms;
3477 msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;
3478
3479 spin_lock_irqsave(&cm_id_priv->lock, flags);
3480 if (cm_id->state == IB_CM_IDLE)
3481 ret = ib_post_send_mad(msg, NULL);
3482 else
3483 ret = -EINVAL;
3484
3485 if (ret) {
3486 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3487 cm_free_msg(msg);
3488 goto out;
3489 }
3490 cm_id->state = IB_CM_SIDR_REQ_SENT;
3491 cm_id_priv->msg = msg;
3492 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3493 out:
3494 return ret;
3495 }
3496 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3497
cm_format_sidr_req_event(struct cm_work * work,const struct cm_id_private * rx_cm_id,struct ib_cm_id * listen_id)3498 static void cm_format_sidr_req_event(struct cm_work *work,
3499 const struct cm_id_private *rx_cm_id,
3500 struct ib_cm_id *listen_id)
3501 {
3502 struct cm_sidr_req_msg *sidr_req_msg;
3503 struct ib_cm_sidr_req_event_param *param;
3504
3505 sidr_req_msg = (struct cm_sidr_req_msg *)
3506 work->mad_recv_wc->recv_buf.mad;
3507 param = &work->cm_event.param.sidr_req_rcvd;
3508 param->pkey = __be16_to_cpu(sidr_req_msg->pkey);
3509 param->listen_id = listen_id;
3510 param->service_id = sidr_req_msg->service_id;
3511 param->bth_pkey = cm_get_bth_pkey(work);
3512 param->port = work->port->port_num;
3513 param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3514 work->cm_event.private_data = &sidr_req_msg->private_data;
3515 }
3516
cm_sidr_req_handler(struct cm_work * work)3517 static int cm_sidr_req_handler(struct cm_work *work)
3518 {
3519 struct ib_cm_id *cm_id;
3520 struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
3521 struct cm_sidr_req_msg *sidr_req_msg;
3522 struct ib_wc *wc;
3523 int ret;
3524
3525 cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
3526 if (IS_ERR(cm_id))
3527 return PTR_ERR(cm_id);
3528 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3529
3530 /* Record SGID/SLID and request ID for lookup. */
3531 sidr_req_msg = (struct cm_sidr_req_msg *)
3532 work->mad_recv_wc->recv_buf.mad;
3533 wc = work->mad_recv_wc->wc;
3534 cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
3535 cm_id_priv->av.dgid.global.interface_id = 0;
3536 ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3537 work->mad_recv_wc->recv_buf.grh,
3538 &cm_id_priv->av);
3539 if (ret)
3540 goto out;
3541 cm_id_priv->id.remote_id = sidr_req_msg->request_id;
3542 cm_id_priv->tid = sidr_req_msg->hdr.tid;
3543 atomic_inc(&cm_id_priv->work_count);
3544
3545 spin_lock_irq(&cm.lock);
3546 cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3547 if (cur_cm_id_priv) {
3548 spin_unlock_irq(&cm.lock);
3549 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3550 counter[CM_SIDR_REQ_COUNTER]);
3551 goto out; /* Duplicate message. */
3552 }
3553 cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3554 cur_cm_id_priv = cm_find_listen(cm_id->device,
3555 sidr_req_msg->service_id);
3556 if (!cur_cm_id_priv) {
3557 spin_unlock_irq(&cm.lock);
3558 cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED);
3559 goto out; /* No match. */
3560 }
3561 atomic_inc(&cur_cm_id_priv->refcount);
3562 atomic_inc(&cm_id_priv->refcount);
3563 spin_unlock_irq(&cm.lock);
3564
3565 cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler;
3566 cm_id_priv->id.context = cur_cm_id_priv->id.context;
3567 cm_id_priv->id.service_id = sidr_req_msg->service_id;
3568 cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3569
3570 cm_format_sidr_req_event(work, cm_id_priv, &cur_cm_id_priv->id);
3571 cm_process_work(cm_id_priv, work);
3572 cm_deref_id(cur_cm_id_priv);
3573 return 0;
3574 out:
3575 ib_destroy_cm_id(&cm_id_priv->id);
3576 return -EINVAL;
3577 }
3578
cm_format_sidr_rep(struct cm_sidr_rep_msg * sidr_rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3579 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3580 struct cm_id_private *cm_id_priv,
3581 struct ib_cm_sidr_rep_param *param)
3582 {
3583 cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3584 cm_id_priv->tid);
3585 sidr_rep_msg->request_id = cm_id_priv->id.remote_id;
3586 sidr_rep_msg->status = param->status;
3587 cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param->qp_num));
3588 sidr_rep_msg->service_id = cm_id_priv->id.service_id;
3589 sidr_rep_msg->qkey = cpu_to_be32(param->qkey);
3590
3591 if (param->info && param->info_length)
3592 memcpy(sidr_rep_msg->info, param->info, param->info_length);
3593
3594 if (param->private_data && param->private_data_len)
3595 memcpy(sidr_rep_msg->private_data, param->private_data,
3596 param->private_data_len);
3597 }
3598
ib_send_cm_sidr_rep(struct ib_cm_id * cm_id,struct ib_cm_sidr_rep_param * param)3599 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3600 struct ib_cm_sidr_rep_param *param)
3601 {
3602 struct cm_id_private *cm_id_priv;
3603 struct ib_mad_send_buf *msg;
3604 unsigned long flags;
3605 int ret;
3606
3607 if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3608 (param->private_data &&
3609 param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3610 return -EINVAL;
3611
3612 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3613 spin_lock_irqsave(&cm_id_priv->lock, flags);
3614 if (cm_id->state != IB_CM_SIDR_REQ_RCVD) {
3615 ret = -EINVAL;
3616 goto error;
3617 }
3618
3619 ret = cm_alloc_msg(cm_id_priv, &msg);
3620 if (ret)
3621 goto error;
3622
3623 cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3624 param);
3625 ret = ib_post_send_mad(msg, NULL);
3626 if (ret) {
3627 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3628 cm_free_msg(msg);
3629 return ret;
3630 }
3631 cm_id->state = IB_CM_IDLE;
3632 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3633
3634 spin_lock_irqsave(&cm.lock, flags);
3635 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3636 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3637 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3638 }
3639 spin_unlock_irqrestore(&cm.lock, flags);
3640 return 0;
3641
3642 error: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3643 return ret;
3644 }
3645 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3646
cm_format_sidr_rep_event(struct cm_work * work,const struct cm_id_private * cm_id_priv)3647 static void cm_format_sidr_rep_event(struct cm_work *work,
3648 const struct cm_id_private *cm_id_priv)
3649 {
3650 struct cm_sidr_rep_msg *sidr_rep_msg;
3651 struct ib_cm_sidr_rep_event_param *param;
3652
3653 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3654 work->mad_recv_wc->recv_buf.mad;
3655 param = &work->cm_event.param.sidr_rep_rcvd;
3656 param->status = sidr_rep_msg->status;
3657 param->qkey = be32_to_cpu(sidr_rep_msg->qkey);
3658 param->qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg));
3659 param->info = &sidr_rep_msg->info;
3660 param->info_len = sidr_rep_msg->info_length;
3661 param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3662 work->cm_event.private_data = &sidr_rep_msg->private_data;
3663 }
3664
cm_sidr_rep_handler(struct cm_work * work)3665 static int cm_sidr_rep_handler(struct cm_work *work)
3666 {
3667 struct cm_sidr_rep_msg *sidr_rep_msg;
3668 struct cm_id_private *cm_id_priv;
3669
3670 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3671 work->mad_recv_wc->recv_buf.mad;
3672 cm_id_priv = cm_acquire_id(sidr_rep_msg->request_id, 0);
3673 if (!cm_id_priv)
3674 return -EINVAL; /* Unmatched reply. */
3675
3676 spin_lock_irq(&cm_id_priv->lock);
3677 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3678 spin_unlock_irq(&cm_id_priv->lock);
3679 goto out;
3680 }
3681 cm_id_priv->id.state = IB_CM_IDLE;
3682 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3683 spin_unlock_irq(&cm_id_priv->lock);
3684
3685 cm_format_sidr_rep_event(work, cm_id_priv);
3686 cm_process_work(cm_id_priv, work);
3687 return 0;
3688 out:
3689 cm_deref_id(cm_id_priv);
3690 return -EINVAL;
3691 }
3692
cm_process_send_error(struct ib_mad_send_buf * msg,enum ib_wc_status wc_status)3693 static void cm_process_send_error(struct ib_mad_send_buf *msg,
3694 enum ib_wc_status wc_status)
3695 {
3696 struct cm_id_private *cm_id_priv;
3697 struct ib_cm_event cm_event;
3698 enum ib_cm_state state;
3699 int ret;
3700
3701 memset(&cm_event, 0, sizeof cm_event);
3702 cm_id_priv = msg->context[0];
3703
3704 /* Discard old sends or ones without a response. */
3705 spin_lock_irq(&cm_id_priv->lock);
3706 state = (enum ib_cm_state) (unsigned long) msg->context[1];
3707 if (msg != cm_id_priv->msg || state != cm_id_priv->id.state)
3708 goto discard;
3709
3710 switch (state) {
3711 case IB_CM_REQ_SENT:
3712 case IB_CM_MRA_REQ_RCVD:
3713 cm_reset_to_idle(cm_id_priv);
3714 cm_event.event = IB_CM_REQ_ERROR;
3715 break;
3716 case IB_CM_REP_SENT:
3717 case IB_CM_MRA_REP_RCVD:
3718 cm_reset_to_idle(cm_id_priv);
3719 cm_event.event = IB_CM_REP_ERROR;
3720 break;
3721 case IB_CM_DREQ_SENT:
3722 cm_enter_timewait(cm_id_priv);
3723 cm_event.event = IB_CM_DREQ_ERROR;
3724 break;
3725 case IB_CM_SIDR_REQ_SENT:
3726 cm_id_priv->id.state = IB_CM_IDLE;
3727 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3728 break;
3729 default:
3730 goto discard;
3731 }
3732 spin_unlock_irq(&cm_id_priv->lock);
3733 cm_event.param.send_status = wc_status;
3734
3735 /* No other events can occur on the cm_id at this point. */
3736 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3737 cm_free_msg(msg);
3738 if (ret)
3739 ib_destroy_cm_id(&cm_id_priv->id);
3740 return;
3741 discard:
3742 spin_unlock_irq(&cm_id_priv->lock);
3743 cm_free_msg(msg);
3744 }
3745
cm_send_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_wc * mad_send_wc)3746 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3747 struct ib_mad_send_wc *mad_send_wc)
3748 {
3749 struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3750 struct cm_port *port;
3751 u16 attr_index;
3752
3753 port = mad_agent->context;
3754 attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3755 msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3756
3757 /*
3758 * If the send was in response to a received message (context[0] is not
3759 * set to a cm_id), and is not a REJ, then it is a send that was
3760 * manually retried.
3761 */
3762 if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
3763 msg->retries = 1;
3764
3765 atomic_long_add(1 + msg->retries,
3766 &port->counter_group[CM_XMIT].counter[attr_index]);
3767 if (msg->retries)
3768 atomic_long_add(msg->retries,
3769 &port->counter_group[CM_XMIT_RETRIES].
3770 counter[attr_index]);
3771
3772 switch (mad_send_wc->status) {
3773 case IB_WC_SUCCESS:
3774 case IB_WC_WR_FLUSH_ERR:
3775 cm_free_msg(msg);
3776 break;
3777 default:
3778 if (msg->context[0] && msg->context[1])
3779 cm_process_send_error(msg, mad_send_wc->status);
3780 else
3781 cm_free_msg(msg);
3782 break;
3783 }
3784 }
3785
cm_work_handler(struct work_struct * _work)3786 static void cm_work_handler(struct work_struct *_work)
3787 {
3788 struct cm_work *work = container_of(_work, struct cm_work, work.work);
3789 int ret;
3790
3791 switch (work->cm_event.event) {
3792 case IB_CM_REQ_RECEIVED:
3793 ret = cm_req_handler(work);
3794 break;
3795 case IB_CM_MRA_RECEIVED:
3796 ret = cm_mra_handler(work);
3797 break;
3798 case IB_CM_REJ_RECEIVED:
3799 ret = cm_rej_handler(work);
3800 break;
3801 case IB_CM_REP_RECEIVED:
3802 ret = cm_rep_handler(work);
3803 break;
3804 case IB_CM_RTU_RECEIVED:
3805 ret = cm_rtu_handler(work);
3806 break;
3807 case IB_CM_USER_ESTABLISHED:
3808 ret = cm_establish_handler(work);
3809 break;
3810 case IB_CM_DREQ_RECEIVED:
3811 ret = cm_dreq_handler(work);
3812 break;
3813 case IB_CM_DREP_RECEIVED:
3814 ret = cm_drep_handler(work);
3815 break;
3816 case IB_CM_SIDR_REQ_RECEIVED:
3817 ret = cm_sidr_req_handler(work);
3818 break;
3819 case IB_CM_SIDR_REP_RECEIVED:
3820 ret = cm_sidr_rep_handler(work);
3821 break;
3822 case IB_CM_LAP_RECEIVED:
3823 ret = cm_lap_handler(work);
3824 break;
3825 case IB_CM_APR_RECEIVED:
3826 ret = cm_apr_handler(work);
3827 break;
3828 case IB_CM_TIMEWAIT_EXIT:
3829 ret = cm_timewait_handler(work);
3830 break;
3831 default:
3832 ret = -EINVAL;
3833 break;
3834 }
3835 if (ret)
3836 cm_free_work(work);
3837 }
3838
cm_establish(struct ib_cm_id * cm_id)3839 static int cm_establish(struct ib_cm_id *cm_id)
3840 {
3841 struct cm_id_private *cm_id_priv;
3842 struct cm_work *work;
3843 unsigned long flags;
3844 int ret = 0;
3845 struct cm_device *cm_dev;
3846
3847 cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3848 if (!cm_dev)
3849 return -ENODEV;
3850
3851 work = kmalloc(sizeof *work, GFP_ATOMIC);
3852 if (!work)
3853 return -ENOMEM;
3854
3855 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3856 spin_lock_irqsave(&cm_id_priv->lock, flags);
3857 switch (cm_id->state)
3858 {
3859 case IB_CM_REP_SENT:
3860 case IB_CM_MRA_REP_RCVD:
3861 cm_id->state = IB_CM_ESTABLISHED;
3862 break;
3863 case IB_CM_ESTABLISHED:
3864 ret = -EISCONN;
3865 break;
3866 default:
3867 ret = -EINVAL;
3868 break;
3869 }
3870 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3871
3872 if (ret) {
3873 kfree(work);
3874 goto out;
3875 }
3876
3877 /*
3878 * The CM worker thread may try to destroy the cm_id before it
3879 * can execute this work item. To prevent potential deadlock,
3880 * we need to find the cm_id once we're in the context of the
3881 * worker thread, rather than holding a reference on it.
3882 */
3883 INIT_DELAYED_WORK(&work->work, cm_work_handler);
3884 work->local_id = cm_id->local_id;
3885 work->remote_id = cm_id->remote_id;
3886 work->mad_recv_wc = NULL;
3887 work->cm_event.event = IB_CM_USER_ESTABLISHED;
3888
3889 /* Check if the device started its remove_one */
3890 spin_lock_irqsave(&cm.lock, flags);
3891 if (!cm_dev->going_down) {
3892 queue_delayed_work(cm.wq, &work->work, 0);
3893 } else {
3894 kfree(work);
3895 ret = -ENODEV;
3896 }
3897 spin_unlock_irqrestore(&cm.lock, flags);
3898
3899 out:
3900 return ret;
3901 }
3902
cm_migrate(struct ib_cm_id * cm_id)3903 static int cm_migrate(struct ib_cm_id *cm_id)
3904 {
3905 struct cm_id_private *cm_id_priv;
3906 struct cm_av tmp_av;
3907 unsigned long flags;
3908 int tmp_send_port_not_ready;
3909 int ret = 0;
3910
3911 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3912 spin_lock_irqsave(&cm_id_priv->lock, flags);
3913 if (cm_id->state == IB_CM_ESTABLISHED &&
3914 (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3915 cm_id->lap_state == IB_CM_LAP_IDLE)) {
3916 cm_id->lap_state = IB_CM_LAP_IDLE;
3917 /* Swap address vector */
3918 tmp_av = cm_id_priv->av;
3919 cm_id_priv->av = cm_id_priv->alt_av;
3920 cm_id_priv->alt_av = tmp_av;
3921 /* Swap port send ready state */
3922 tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready;
3923 cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready;
3924 cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready;
3925 } else
3926 ret = -EINVAL;
3927 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3928
3929 return ret;
3930 }
3931
ib_cm_notify(struct ib_cm_id * cm_id,enum ib_event_type event)3932 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3933 {
3934 int ret;
3935
3936 switch (event) {
3937 case IB_EVENT_COMM_EST:
3938 ret = cm_establish(cm_id);
3939 break;
3940 case IB_EVENT_PATH_MIG:
3941 ret = cm_migrate(cm_id);
3942 break;
3943 default:
3944 ret = -EINVAL;
3945 }
3946 return ret;
3947 }
3948 EXPORT_SYMBOL(ib_cm_notify);
3949
cm_recv_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_buf * send_buf,struct ib_mad_recv_wc * mad_recv_wc)3950 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3951 struct ib_mad_send_buf *send_buf,
3952 struct ib_mad_recv_wc *mad_recv_wc)
3953 {
3954 struct cm_port *port = mad_agent->context;
3955 struct cm_work *work;
3956 enum ib_cm_event_type event;
3957 bool alt_path = false;
3958 u16 attr_id;
3959 int paths = 0;
3960 int going_down = 0;
3961
3962 switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
3963 case CM_REQ_ATTR_ID:
3964 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
3965 mad_recv_wc->recv_buf.mad);
3966 paths = 1 + (alt_path != 0);
3967 event = IB_CM_REQ_RECEIVED;
3968 break;
3969 case CM_MRA_ATTR_ID:
3970 event = IB_CM_MRA_RECEIVED;
3971 break;
3972 case CM_REJ_ATTR_ID:
3973 event = IB_CM_REJ_RECEIVED;
3974 break;
3975 case CM_REP_ATTR_ID:
3976 event = IB_CM_REP_RECEIVED;
3977 break;
3978 case CM_RTU_ATTR_ID:
3979 event = IB_CM_RTU_RECEIVED;
3980 break;
3981 case CM_DREQ_ATTR_ID:
3982 event = IB_CM_DREQ_RECEIVED;
3983 break;
3984 case CM_DREP_ATTR_ID:
3985 event = IB_CM_DREP_RECEIVED;
3986 break;
3987 case CM_SIDR_REQ_ATTR_ID:
3988 event = IB_CM_SIDR_REQ_RECEIVED;
3989 break;
3990 case CM_SIDR_REP_ATTR_ID:
3991 event = IB_CM_SIDR_REP_RECEIVED;
3992 break;
3993 case CM_LAP_ATTR_ID:
3994 paths = 1;
3995 event = IB_CM_LAP_RECEIVED;
3996 break;
3997 case CM_APR_ATTR_ID:
3998 event = IB_CM_APR_RECEIVED;
3999 break;
4000 default:
4001 ib_free_recv_mad(mad_recv_wc);
4002 return;
4003 }
4004
4005 attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4006 atomic_long_inc(&port->counter_group[CM_RECV].
4007 counter[attr_id - CM_ATTR_ID_OFFSET]);
4008
4009 work = kmalloc(sizeof(*work) + sizeof(struct sa_path_rec) * paths,
4010 GFP_KERNEL);
4011 if (!work) {
4012 ib_free_recv_mad(mad_recv_wc);
4013 return;
4014 }
4015
4016 INIT_DELAYED_WORK(&work->work, cm_work_handler);
4017 work->cm_event.event = event;
4018 work->mad_recv_wc = mad_recv_wc;
4019 work->port = port;
4020
4021 /* Check if the device started its remove_one */
4022 spin_lock_irq(&cm.lock);
4023 if (!port->cm_dev->going_down)
4024 queue_delayed_work(cm.wq, &work->work, 0);
4025 else
4026 going_down = 1;
4027 spin_unlock_irq(&cm.lock);
4028
4029 if (going_down) {
4030 kfree(work);
4031 ib_free_recv_mad(mad_recv_wc);
4032 }
4033 }
4034
cm_init_qp_init_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4035 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4036 struct ib_qp_attr *qp_attr,
4037 int *qp_attr_mask)
4038 {
4039 unsigned long flags;
4040 int ret;
4041
4042 spin_lock_irqsave(&cm_id_priv->lock, flags);
4043 switch (cm_id_priv->id.state) {
4044 case IB_CM_REQ_SENT:
4045 case IB_CM_MRA_REQ_RCVD:
4046 case IB_CM_REQ_RCVD:
4047 case IB_CM_MRA_REQ_SENT:
4048 case IB_CM_REP_RCVD:
4049 case IB_CM_MRA_REP_SENT:
4050 case IB_CM_REP_SENT:
4051 case IB_CM_MRA_REP_RCVD:
4052 case IB_CM_ESTABLISHED:
4053 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4054 IB_QP_PKEY_INDEX | IB_QP_PORT;
4055 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4056 if (cm_id_priv->responder_resources)
4057 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4058 IB_ACCESS_REMOTE_ATOMIC;
4059 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4060 qp_attr->port_num = cm_id_priv->av.port->port_num;
4061 ret = 0;
4062 break;
4063 default:
4064 ret = -EINVAL;
4065 break;
4066 }
4067 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4068 return ret;
4069 }
4070
cm_init_qp_rtr_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4071 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4072 struct ib_qp_attr *qp_attr,
4073 int *qp_attr_mask)
4074 {
4075 unsigned long flags;
4076 int ret;
4077
4078 spin_lock_irqsave(&cm_id_priv->lock, flags);
4079 switch (cm_id_priv->id.state) {
4080 case IB_CM_REQ_RCVD:
4081 case IB_CM_MRA_REQ_SENT:
4082 case IB_CM_REP_RCVD:
4083 case IB_CM_MRA_REP_SENT:
4084 case IB_CM_REP_SENT:
4085 case IB_CM_MRA_REP_RCVD:
4086 case IB_CM_ESTABLISHED:
4087 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4088 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4089 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4090 qp_attr->path_mtu = cm_id_priv->path_mtu;
4091 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4092 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4093 if (cm_id_priv->qp_type == IB_QPT_RC ||
4094 cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4095 *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4096 IB_QP_MIN_RNR_TIMER;
4097 qp_attr->max_dest_rd_atomic =
4098 cm_id_priv->responder_resources;
4099 qp_attr->min_rnr_timer = 0;
4100 }
4101 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4102 *qp_attr_mask |= IB_QP_ALT_PATH;
4103 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4104 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4105 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4106 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4107 }
4108 ret = 0;
4109 break;
4110 default:
4111 ret = -EINVAL;
4112 break;
4113 }
4114 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4115 return ret;
4116 }
4117
cm_init_qp_rts_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4118 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4119 struct ib_qp_attr *qp_attr,
4120 int *qp_attr_mask)
4121 {
4122 unsigned long flags;
4123 int ret;
4124
4125 spin_lock_irqsave(&cm_id_priv->lock, flags);
4126 switch (cm_id_priv->id.state) {
4127 /* Allow transition to RTS before sending REP */
4128 case IB_CM_REQ_RCVD:
4129 case IB_CM_MRA_REQ_SENT:
4130
4131 case IB_CM_REP_RCVD:
4132 case IB_CM_MRA_REP_SENT:
4133 case IB_CM_REP_SENT:
4134 case IB_CM_MRA_REP_RCVD:
4135 case IB_CM_ESTABLISHED:
4136 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4137 *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4138 qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4139 switch (cm_id_priv->qp_type) {
4140 case IB_QPT_RC:
4141 case IB_QPT_XRC_INI:
4142 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4143 IB_QP_MAX_QP_RD_ATOMIC;
4144 qp_attr->retry_cnt = cm_id_priv->retry_count;
4145 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4146 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4147 /* fall through */
4148 case IB_QPT_XRC_TGT:
4149 *qp_attr_mask |= IB_QP_TIMEOUT;
4150 qp_attr->timeout = cm_id_priv->av.timeout;
4151 break;
4152 default:
4153 break;
4154 }
4155 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4156 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4157 qp_attr->path_mig_state = IB_MIG_REARM;
4158 }
4159 } else {
4160 *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4161 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4162 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4163 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4164 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4165 qp_attr->path_mig_state = IB_MIG_REARM;
4166 }
4167 ret = 0;
4168 break;
4169 default:
4170 ret = -EINVAL;
4171 break;
4172 }
4173 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4174 return ret;
4175 }
4176
ib_cm_init_qp_attr(struct ib_cm_id * cm_id,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4177 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4178 struct ib_qp_attr *qp_attr,
4179 int *qp_attr_mask)
4180 {
4181 struct cm_id_private *cm_id_priv;
4182 int ret;
4183
4184 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4185 switch (qp_attr->qp_state) {
4186 case IB_QPS_INIT:
4187 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4188 break;
4189 case IB_QPS_RTR:
4190 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4191 break;
4192 case IB_QPS_RTS:
4193 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4194 break;
4195 default:
4196 ret = -EINVAL;
4197 break;
4198 }
4199 return ret;
4200 }
4201 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4202
cm_show_counter(struct kobject * obj,struct attribute * attr,char * buf)4203 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
4204 char *buf)
4205 {
4206 struct cm_counter_group *group;
4207 struct cm_counter_attribute *cm_attr;
4208
4209 group = container_of(obj, struct cm_counter_group, obj);
4210 cm_attr = container_of(attr, struct cm_counter_attribute, attr);
4211
4212 return sprintf(buf, "%ld\n",
4213 atomic_long_read(&group->counter[cm_attr->index]));
4214 }
4215
4216 static const struct sysfs_ops cm_counter_ops = {
4217 .show = cm_show_counter
4218 };
4219
4220 static struct kobj_type cm_counter_obj_type = {
4221 .sysfs_ops = &cm_counter_ops,
4222 .default_attrs = cm_counter_default_attrs
4223 };
4224
cm_devnode(struct device * dev,umode_t * mode)4225 static char *cm_devnode(struct device *dev, umode_t *mode)
4226 {
4227 if (mode)
4228 *mode = 0666;
4229 return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
4230 }
4231
4232 struct class cm_class = {
4233 .name = "infiniband_cm",
4234 .devnode = cm_devnode,
4235 };
4236 EXPORT_SYMBOL(cm_class);
4237
cm_create_port_fs(struct cm_port * port)4238 static int cm_create_port_fs(struct cm_port *port)
4239 {
4240 int i, ret;
4241
4242 for (i = 0; i < CM_COUNTER_GROUPS; i++) {
4243 ret = ib_port_register_module_stat(port->cm_dev->ib_device,
4244 port->port_num,
4245 &port->counter_group[i].obj,
4246 &cm_counter_obj_type,
4247 counter_group_names[i]);
4248 if (ret)
4249 goto error;
4250 }
4251
4252 return 0;
4253
4254 error:
4255 while (i--)
4256 ib_port_unregister_module_stat(&port->counter_group[i].obj);
4257 return ret;
4258
4259 }
4260
cm_remove_port_fs(struct cm_port * port)4261 static void cm_remove_port_fs(struct cm_port *port)
4262 {
4263 int i;
4264
4265 for (i = 0; i < CM_COUNTER_GROUPS; i++)
4266 ib_port_unregister_module_stat(&port->counter_group[i].obj);
4267
4268 }
4269
cm_add_one(struct ib_device * ib_device)4270 static void cm_add_one(struct ib_device *ib_device)
4271 {
4272 struct cm_device *cm_dev;
4273 struct cm_port *port;
4274 struct ib_mad_reg_req reg_req = {
4275 .mgmt_class = IB_MGMT_CLASS_CM,
4276 .mgmt_class_version = IB_CM_CLASS_VERSION,
4277 };
4278 struct ib_port_modify port_modify = {
4279 .set_port_cap_mask = IB_PORT_CM_SUP
4280 };
4281 unsigned long flags __writeonly;
4282 int ret;
4283 int count = 0;
4284 u8 i;
4285
4286 cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4287 GFP_KERNEL);
4288 if (!cm_dev)
4289 return;
4290
4291 cm_dev->ib_device = ib_device;
4292 cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4293 cm_dev->going_down = 0;
4294
4295 set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4296 for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4297 if (!rdma_cap_ib_cm(ib_device, i))
4298 continue;
4299
4300 port = kzalloc(sizeof *port, GFP_KERNEL);
4301 if (!port)
4302 goto error1;
4303
4304 cm_dev->port[i-1] = port;
4305 port->cm_dev = cm_dev;
4306 port->port_num = i;
4307
4308 INIT_LIST_HEAD(&port->cm_priv_prim_list);
4309 INIT_LIST_HEAD(&port->cm_priv_altr_list);
4310
4311 ret = cm_create_port_fs(port);
4312 if (ret)
4313 goto error1;
4314
4315 port->mad_agent = ib_register_mad_agent(ib_device, i,
4316 IB_QPT_GSI,
4317 ®_req,
4318 0,
4319 cm_send_handler,
4320 cm_recv_handler,
4321 port,
4322 0);
4323 if (IS_ERR(port->mad_agent))
4324 goto error2;
4325
4326 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4327 if (ret)
4328 goto error3;
4329
4330 count++;
4331 }
4332
4333 if (!count)
4334 goto free;
4335
4336 ib_set_client_data(ib_device, &cm_client, cm_dev);
4337
4338 write_lock_irqsave(&cm.device_lock, flags);
4339 list_add_tail(&cm_dev->list, &cm.device_list);
4340 write_unlock_irqrestore(&cm.device_lock, flags);
4341 return;
4342
4343 error3:
4344 ib_unregister_mad_agent(port->mad_agent);
4345 error2:
4346 cm_remove_port_fs(port);
4347 error1:
4348 port_modify.set_port_cap_mask = 0;
4349 port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4350 kfree(port);
4351 while (--i) {
4352 if (!rdma_cap_ib_cm(ib_device, i))
4353 continue;
4354
4355 port = cm_dev->port[i-1];
4356 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4357 ib_unregister_mad_agent(port->mad_agent);
4358 cm_remove_port_fs(port);
4359 kfree(port);
4360 }
4361 free:
4362 kfree(cm_dev);
4363 }
4364
cm_remove_one(struct ib_device * ib_device,void * client_data)4365 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4366 {
4367 struct cm_device *cm_dev = client_data;
4368 struct cm_port *port;
4369 struct cm_id_private *cm_id_priv;
4370 struct ib_mad_agent *cur_mad_agent;
4371 struct ib_port_modify port_modify = {
4372 .clr_port_cap_mask = IB_PORT_CM_SUP
4373 };
4374 unsigned long flags __writeonly;
4375 int i;
4376
4377 if (!cm_dev)
4378 return;
4379
4380 write_lock_irqsave(&cm.device_lock, flags);
4381 list_del(&cm_dev->list);
4382 write_unlock_irqrestore(&cm.device_lock, flags);
4383
4384 spin_lock_irq(&cm.lock);
4385 cm_dev->going_down = 1;
4386 spin_unlock_irq(&cm.lock);
4387
4388 for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4389 if (!rdma_cap_ib_cm(ib_device, i))
4390 continue;
4391
4392 port = cm_dev->port[i-1];
4393 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4394 /* Mark all the cm_id's as not valid */
4395 spin_lock_irq(&cm.lock);
4396 list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list)
4397 cm_id_priv->altr_send_port_not_ready = 1;
4398 list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list)
4399 cm_id_priv->prim_send_port_not_ready = 1;
4400 spin_unlock_irq(&cm.lock);
4401 /*
4402 * We flush the queue here after the going_down set, this
4403 * verify that no new works will be queued in the recv handler,
4404 * after that we can call the unregister_mad_agent
4405 */
4406 flush_workqueue(cm.wq);
4407 spin_lock_irq(&cm.state_lock);
4408 cur_mad_agent = port->mad_agent;
4409 port->mad_agent = NULL;
4410 spin_unlock_irq(&cm.state_lock);
4411 ib_unregister_mad_agent(cur_mad_agent);
4412 cm_remove_port_fs(port);
4413 kfree(port);
4414 }
4415
4416 kfree(cm_dev);
4417 }
4418
ib_cm_init(void)4419 static int __init ib_cm_init(void)
4420 {
4421 int ret;
4422
4423 memset(&cm, 0, sizeof cm);
4424 INIT_LIST_HEAD(&cm.device_list);
4425 rwlock_init(&cm.device_lock);
4426 spin_lock_init(&cm.lock);
4427 spin_lock_init(&cm.state_lock);
4428 cm.listen_service_table = RB_ROOT;
4429 cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4430 cm.remote_id_table = RB_ROOT;
4431 cm.remote_qp_table = RB_ROOT;
4432 cm.remote_sidr_table = RB_ROOT;
4433 idr_init(&cm.local_id_table);
4434 get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4435 INIT_LIST_HEAD(&cm.timewait_list);
4436
4437 ret = class_register(&cm_class);
4438 if (ret) {
4439 ret = -ENOMEM;
4440 goto error1;
4441 }
4442
4443 cm.wq = create_workqueue("ib_cm");
4444 if (!cm.wq) {
4445 ret = -ENOMEM;
4446 goto error2;
4447 }
4448
4449 ret = ib_register_client(&cm_client);
4450 if (ret)
4451 goto error3;
4452
4453 return 0;
4454 error3:
4455 destroy_workqueue(cm.wq);
4456 error2:
4457 class_unregister(&cm_class);
4458 error1:
4459 idr_destroy(&cm.local_id_table);
4460 return ret;
4461 }
4462
ib_cm_cleanup(void)4463 static void __exit ib_cm_cleanup(void)
4464 {
4465 struct cm_timewait_info *timewait_info, *tmp;
4466
4467 spin_lock_irq(&cm.lock);
4468 list_for_each_entry(timewait_info, &cm.timewait_list, list)
4469 cancel_delayed_work(&timewait_info->work.work);
4470 spin_unlock_irq(&cm.lock);
4471
4472 ib_unregister_client(&cm_client);
4473 destroy_workqueue(cm.wq);
4474
4475 list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4476 cancel_delayed_work_sync(&timewait_info->work.work);
4477 list_del(&timewait_info->list);
4478 kfree(timewait_info);
4479 }
4480
4481 class_unregister(&cm_class);
4482 idr_destroy(&cm.local_id_table);
4483 }
4484
4485 module_init_order(ib_cm_init, SI_ORDER_SECOND);
4486 module_exit_order(ib_cm_cleanup, SI_ORDER_SECOND);
4487
4488