1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 /* AUTHOR Edward Bortnikov
37 *
38 * DESCRIPTION
39 * The lower-level MAD transport interface implementation
40 * that allows sending a single MAD/receiving a callback
41 * when a single MAD is received.
42 */
43
44 #if HAVE_CONFIG_H
45 # include <config.h>
46 #endif /* HAVE_CONFIG_H */
47
48 #include <sys/types.h>
49 #include <sys/stat.h>
50 #include <sys/ioctl.h>
51 #include <fcntl.h>
52 #include <errno.h>
53 #include <stdlib.h>
54 #include <string.h>
55
56 #include <vendor/osm_vendor_api.h>
57 #include <vendor/osm_vendor_mlx_transport.h>
58 #include <vendor/osm_vendor_mlx_dispatcher.h>
59 #include <vendor/osm_vendor_mlx_svc.h>
60 #include <vendor/osm_ts_useraccess.h>
61
62 typedef struct _osmv_TOPSPIN_transport_mgr_ {
63 int device_fd;
64 osm_ts_user_mad_filter filter;
65 cl_thread_t receiver;
66 } osmv_TOPSPIN_transport_mgr_t;
67
68 static void
69 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
70 IN struct ib_mad *p_mad,
71 IN uint8_t is_smi,
72 OUT osm_mad_addr_t * p_mad_addr);
73
74 static void
75 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
76 IN uint8_t is_smi,
77 OUT struct ib_mad *p_mad);
78
__osmv_TOPSPIN_receiver_thr(void * p_ctx)79 void __osmv_TOPSPIN_receiver_thr(void *p_ctx)
80 {
81 int ts_ret_code;
82 struct ib_mad mad;
83 osm_mad_addr_t mad_addr;
84 osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
85 ib_api_status_t status = IB_SUCCESS;
86
87 OSM_LOG_ENTER(p_bo->p_vendor->p_log);
88
89 /* Make sure the p_bo object is still relevant */
90 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
91 return;
92
93 /* we set the type of cancelation for this thread */
94 /* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */
95
96 while (1) {
97 /* Make sure the p_bo object is still relevant */
98 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
99 return;
100
101 /* we read one mad at a time and pass it to the read callback function */
102 ts_ret_code =
103 read(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->
104 p_transp_mgr))->
105 device_fd, &mad, sizeof(mad));
106 /* Make sure the p_bo object is still relevant */
107 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
108 return;
109
110 if (ts_ret_code != sizeof(mad)) {
111 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
112 "__osmv_TOPSPIN_receiver_thr: ERR 6803: "
113 "error with read, bytes = %d, errno = %d\n",
114 ts_ret_code, errno);
115 break;
116 } else {
117 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
118 "__osmv_TOPSPIN_receiver_thr: "
119 "MAD QPN:%d SLID:0x%04x class:0x%02x "
120 "method:0x%02x attr:0x%04x status:0x%04x "
121 "tid:0x%016" PRIx64 "\n",
122 mad.dqpn,
123 cl_ntoh16(mad.slid),
124 mad.mgmt_class,
125 mad.r_method,
126 cl_ntoh16(mad.attribute_id),
127 cl_ntoh16(mad.status),
128 cl_ntoh64(mad.transaction_id));
129
130 /* first arrange an address */
131 __osmv_TOPSPIN_mad_addr_to_osm_addr(p_bo->p_vendor,
132 &mad,
133 (((ib_mad_t *) &
134 mad)->
135 mgmt_class ==
136 IB_MCLASS_SUBN_LID)
137 ||
138 (((ib_mad_t *) &
139 mad)->
140 mgmt_class ==
141 IB_MCLASS_SUBN_DIR),
142 &mad_addr);
143
144 /* call the receiver callback */
145
146 status =
147 osmv_dispatch_mad((osm_bind_handle_t) p_bo,
148 (void *)&mad, &mad_addr);
149
150 /* Make sure the p_bo object is still relevant */
151 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
152 return;
153
154 if (IB_INTERRUPTED == status) {
155
156 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
157 "__osmv_TOPSPIN_receiver_thr: "
158 "The bind handle %p is being closed. "
159 "Breaking the loop.\n", p_bo);
160 break;
161 }
162 }
163 }
164
165 OSM_LOG_EXIT(p_bo->p_vendor->p_log);
166 }
167
168 /*
169 * NAME
170 * osmv_transport_init
171 *
172 * DESCRIPTION
173 * Setup the MAD transport infrastructure (filters, callbacks etc).
174 */
175
176 ib_api_status_t
osmv_transport_init(IN osm_bind_info_t * p_info,IN char hca_id[VENDOR_HCA_MAXNAMES],IN uint8_t hca_idx,IN osmv_bind_obj_t * p_bo)177 osmv_transport_init(IN osm_bind_info_t * p_info,
178 IN char hca_id[VENDOR_HCA_MAXNAMES],
179 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
180 {
181 cl_status_t cl_st;
182 char device_file[16];
183 int device_fd;
184 int ts_ioctl_ret;
185 osmv_TOPSPIN_transport_mgr_t *p_mgr =
186 malloc(sizeof(osmv_TOPSPIN_transport_mgr_t));
187 int qpn;
188
189 if (!p_mgr) {
190 return IB_INSUFFICIENT_MEMORY;
191 }
192
193 memset(p_mgr, 0, sizeof(osmv_TOPSPIN_transport_mgr_t));
194
195 /* open TopSpin file device */
196 /* HACK: assume last char in hostid is the HCA index */
197 sprintf(device_file, "/dev/ts_ua%u", hca_idx);
198 device_fd = open(device_file, O_RDWR);
199 if (device_fd < 0) {
200 fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
201 device_file, errno);
202 return IB_ERROR;
203 }
204
205 /*
206 * Create the MAD filter on this file handle.
207 */
208
209 p_mgr->filter.port = p_bo->port_num;
210 p_mgr->filter.direction = TS_IB_MAD_DIRECTION_IN;
211 p_mgr->filter.mask =
212 TS_IB_MAD_FILTER_DIRECTION |
213 TS_IB_MAD_FILTER_PORT |
214 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
215
216 switch (p_info->mad_class) {
217 case IB_MCLASS_SUBN_LID:
218 case IB_MCLASS_SUBN_DIR:
219 qpn = 0;
220 p_mgr->filter.qpn = qpn;
221 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_LID;
222 ts_ioctl_ret =
223 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
224 if (ts_ioctl_ret < 0) {
225 return IB_ERROR;
226 }
227
228 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_DIR;
229 ts_ioctl_ret =
230 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
231 if (ts_ioctl_ret < 0) {
232 return IB_ERROR;
233 }
234
235 break;
236
237 case IB_MCLASS_SUBN_ADM:
238 default:
239 qpn = 1;
240 p_mgr->filter.qpn = qpn;
241 p_mgr->filter.mgmt_class = p_info->mad_class;
242 ts_ioctl_ret =
243 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
244 if (ts_ioctl_ret < 0) {
245 return IB_ERROR;
246 }
247 break;
248 }
249
250 p_mgr->device_fd = device_fd;
251
252 p_bo->p_transp_mgr = p_mgr;
253
254 /* Initialize the magic_ptr to the pointer of the p_bo info.
255 This will be used to signal when the object is being destroyed, so no
256 real action will be done then. */
257 p_bo->magic_ptr = p_bo;
258
259 /* init receiver thread */
260 cl_st =
261 cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_receiver_thr,
262 (void *)p_bo, "osmv TOPSPIN rcv thr");
263
264 return (ib_api_status_t) cl_st;
265 }
266
267 /*
268 * NAME
269 * osmv_transport_send_mad
270 *
271 * DESCRIPTION
272 * Send a single MAD (256 byte)
273 */
274
275 ib_api_status_t
osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,IN void * p_mad,IN const osm_mad_addr_t * p_mad_addr)276 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
277 IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
278 {
279
280 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
281 osm_vendor_t const *p_vend = p_bo->p_vendor;
282 struct ib_mad ts_mad;
283 int ret;
284 ib_api_status_t status;
285
286 const ib_mad_t *p_mad_hdr = p_mad;
287
288 OSM_LOG_ENTER(p_vend->p_log);
289
290 memset(&ts_mad, 0, sizeof(ts_mad));
291
292 /* Make sure the p_bo object is still relevant */
293 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
294 return IB_INVALID_CALLBACK;
295
296 /*
297 * Copy the MAD over to the sent mad
298 */
299 memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);
300
301 /*
302 * For all sends other than directed route SM MADs,
303 * acquire an address vector for the destination.
304 */
305 if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
306
307 __osmv_TOPSPIN_osm_addr_to_mad_addr(p_mad_addr,
308 p_mad_hdr->mgmt_class ==
309 IB_MCLASS_SUBN_LID,
310 &ts_mad);
311 } else {
312 /* is a directed route - we need to construct a permissive address */
313 /* we do not need port number since it is part of the mad_hndl */
314 ts_mad.dlid = IB_LID_PERMISSIVE;
315 ts_mad.slid = IB_LID_PERMISSIVE;
316 ts_mad.sqpn = 0;
317 ts_mad.dqpn = 0;
318 }
319
320 ts_mad.port = p_bo->port_num;
321
322 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
323 "osmv_transport_mad_send: "
324 "Sending QPN:%d DLID:0x%04x class:0x%02x "
325 "method:0x%02x attr:0x%04x status:0x%04x "
326 "tid:0x%016" PRIx64 "\n",
327 ts_mad.dqpn,
328 cl_ntoh16(ts_mad.dlid),
329 ts_mad.mgmt_class,
330 ts_mad.r_method,
331 cl_ntoh16(ts_mad.attribute_id),
332 cl_ntoh16(ts_mad.status), cl_ntoh64(ts_mad.transaction_id)
333 );
334
335 /* send it */
336 ret =
337 write(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->
338 device_fd, &ts_mad, sizeof(ts_mad));
339
340 if (ret != sizeof(ts_mad)) {
341 osm_log(p_vend->p_log, OSM_LOG_ERROR,
342 "osmv_transport_mad_send: ERR 6804: "
343 "Error sending mad (%d).\n", ret);
344 status = IB_ERROR;
345 goto Exit;
346 }
347
348 status = IB_SUCCESS;
349
350 Exit:
351 OSM_LOG_EXIT(p_vend->p_log);
352 return (status);
353 }
354
355 /*
356 register a new mad type to the opened device file
357 and send a mad through - the main idea is to make
358 the filter catch it such that the read unblocks
359 */
__osm_transport_gen_dummy_mad(osmv_bind_obj_t * p_bo)360 void __osm_transport_gen_dummy_mad(osmv_bind_obj_t * p_bo)
361 {
362 struct ib_mad ts_mad;
363 osmv_TOPSPIN_transport_mgr_t *p_mgr =
364 (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);
365 struct ib_get_port_info_ioctl port_data;
366 int ts_ioctl_ret;
367
368 /* prepare the mad fields following the stored filter on the bind */
369 memset(&ts_mad, 0, sizeof(ts_mad));
370 ts_mad.format_version = 1;
371 ts_mad.mgmt_class = p_mgr->filter.mgmt_class;
372 ts_mad.attribute_id = 0x2;
373 ts_mad.class_version = 1;
374 ts_mad.r_method = cl_ntoh16(0x2);
375 ts_mad.port = p_bo->port_num;
376 ts_mad.sqpn = p_mgr->filter.qpn;
377 ts_mad.dqpn = p_mgr->filter.qpn;
378 ts_mad.slid = 0xffff;
379 /* we must send to our local lid ... */
380 port_data.port = p_bo->port_num;
381 ts_ioctl_ret = ioctl(p_mgr->device_fd, TS_IB_IOCGPORTINFO, &port_data);
382 ts_mad.dlid = port_data.port_info.lid;
383 ts_mad.transaction_id = 0x9999;
384 write(p_mgr->device_fd, &ts_mad, sizeof(ts_mad));
385 }
386
osmv_transport_done(IN const osm_bind_handle_t h_bind)387 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
388 {
389 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
390 osmv_TOPSPIN_transport_mgr_t *p_tpot_mgr =
391 (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);
392
393 CL_ASSERT(p_bo);
394
395 /* First of all - zero out the magic_ptr, so if a callback is called -
396 it'll know that we are currently closing down, and will not handle the
397 mad. */
398 p_bo->magic_ptr = 0;
399 /* usleep(3000000); */
400
401 /* seems the only way to abort a blocking read is to make it read something */
402 __osm_transport_gen_dummy_mad(p_bo);
403 cl_thread_destroy(&(p_tpot_mgr->receiver));
404 free(p_tpot_mgr);
405 }
406
407 static void
__osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,IN uint8_t is_smi,OUT struct ib_mad * p_mad)408 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
409 IN uint8_t is_smi, OUT struct ib_mad *p_mad)
410 {
411
412 /* For global destination or Multicast address: */
413 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
414 p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
415 if (is_smi) {
416 p_mad->sqpn = 0;
417 p_mad->dqpn = 0;
418 } else {
419 p_mad->sqpn = 1;
420 p_mad->dqpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
421 }
422 /*
423 HACK we limit to the first PKey Index assuming it will
424 always be the default PKey
425 */
426 p_mad->pkey_index = 0;
427 }
428
429 static void
__osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const * p_vend,IN struct ib_mad * p_mad,IN uint8_t is_smi,OUT osm_mad_addr_t * p_mad_addr)430 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
431 IN struct ib_mad *p_mad,
432 IN uint8_t is_smi,
433 OUT osm_mad_addr_t * p_mad_addr)
434 {
435 p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
436 p_mad_addr->static_rate = 0;
437 p_mad_addr->path_bits = 0;
438 if (is_smi) {
439 /* SMI */
440 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
441 p_mad_addr->addr_type.smi.port_num = p_mad->port;
442 } else {
443 /* GSI */
444 p_mad_addr->addr_type.gsi.remote_qp = cl_ntoh32(p_mad->sqpn);
445 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
446 /* There is a TAVOR limitation that only one P_KEY is supported per */
447 /* QP - so QP1 must use IB_DEFAULT_PKEY */
448 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
449 p_mad_addr->addr_type.gsi.service_level = p_mad->sl;
450
451 p_mad_addr->addr_type.gsi.global_route = FALSE;
452 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
453 /*
454 if (p_mad_addr->addr_type.gsi.global_route)
455 {
456 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
457 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
458 p_rcv_desc->grh.traffic_class,
459 p_rcv_desc->grh.flow_label);
460 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
461 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
462 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
463 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
464 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
465 }
466 */
467 }
468 }
469
470 /*
471 * NAME osm_vendor_set_sm
472 *
473 * DESCRIPTION Modifies the port info for the bound port to set the "IS_SM" bit
474 * according to the value given (TRUE or FALSE).
475 */
476 #if (defined(OSM_VENDOR_INTF_TS_NO_VAPI) || defined(OSM_VENDOR_INTF_TS))
477
osm_vendor_set_sm(IN osm_bind_handle_t h_bind,IN boolean_t is_sm_val)478 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
479 {
480 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
481 osm_vendor_t const *p_vend = p_bo->p_vendor;
482 int ts_ioctl_ret;
483 int device_fd =
484 ((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->device_fd;
485 struct ib_set_port_info_ioctl set_port_data;
486
487 OSM_LOG_ENTER(p_vend->p_log);
488
489 memset(&set_port_data, 0, sizeof(set_port_data));
490
491 set_port_data.port = p_bo->port_num;
492 set_port_data.port_info.valid_fields = IB_PORT_IS_SM;
493 set_port_data.port_info.is_sm = is_sm_val;
494 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSPORTINFO, &set_port_data);
495 if (ts_ioctl_ret < 0) {
496 osm_log(p_vend->p_log, OSM_LOG_ERROR,
497 "osm_vendor_set_sm: ERR 6805: "
498 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
499 is_sm_val, ts_ioctl_ret);
500 }
501
502 OSM_LOG_EXIT(p_vend->p_log);
503 }
504
505 #endif
506