1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 #undef __init
37 #if HAVE_CONFIG_H
38 # include <config.h>
39 #endif /* HAVE_CONFIG_H */
40
41 #include <stdlib.h>
42 #include <string.h>
43 #include <vendor/osm_vendor_ts.h>
44 #include <vendor/osm_vendor_api.h>
45 #include <vendor/osm_ts_useraccess.h>
46 #include <opensm/osm_subnet.h>
47 #include <opensm/osm_opensm.h>
48
49 /*
50 Since a race can accure on requests. Meaning - a response is received before
51 the send_callback is called - we will save both the madw_p and the fact
52 whether or not it is a response. A race can occure only on requests that did
53 not fail, and then the madw_p will be put back in the pool before the
54 callback.
55 */
__osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)56 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
57 {
58 uint64_t wrid = 0;
59
60 CL_ASSERT(p_madw->p_mad);
61
62 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
63 wrid = (wrid << 1) |
64 ib_mad_is_response(p_madw->p_mad);
65 return wrid;
66 }
67
68 void
__osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,OUT uint8_t * is_resp,OUT osm_madw_t ** pp_madw)69 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
70 OUT uint8_t * is_resp,
71 OUT osm_madw_t ** pp_madw)
72 {
73 *is_resp = wrid & 0x0000000000000001;
74 wrid = wrid >> 1;
75 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
76 }
77
78 /**********************************************************************
79 * TS MAD to OSM ADDRESS VECTOR
80 **********************************************************************/
81 void
__osm_ts_conv_mad_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,IN struct ib_mad * p_mad,IN uint8_t is_smi,OUT osm_mad_addr_t * p_mad_addr)82 __osm_ts_conv_mad_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
83 IN struct ib_mad *p_mad,
84 IN uint8_t is_smi,
85 OUT osm_mad_addr_t * p_mad_addr)
86 {
87 p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
88 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */
89 p_mad_addr->path_bits = 0; /* HACK - no way to know in TS */
90 if (is_smi) {
91 /* SMI */
92 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
93 p_mad_addr->addr_type.smi.port_num = p_mad->port;
94 } else {
95 /* GSI */
96 p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn;
97 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
98 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
99 p_mad_addr->addr_type.gsi.service_level = 0; /* HACK no way to know */
100
101 p_mad_addr->addr_type.gsi.global_route = FALSE; /* HACK no way to know */
102 /* copy the GRH data if relevant */
103 /*
104 if (p_mad_addr->addr_type.gsi.global_route)
105 {
106 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
107 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
108 p_rcv_desc->grh.traffic_class,
109 p_rcv_desc->grh.flow_label);
110 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
111 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
112 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
113 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
114 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
115 }
116 */
117 }
118 }
119
120 /**********************************************************************
121 * OSM ADDR VECTOR TO TS MAD:
122 **********************************************************************/
123 void
__osm_ts_conv_osm_addr_to_ts_addr(IN osm_mad_addr_t * p_mad_addr,IN uint8_t is_smi,OUT struct ib_mad * p_mad)124 __osm_ts_conv_osm_addr_to_ts_addr(IN osm_mad_addr_t * p_mad_addr,
125 IN uint8_t is_smi, OUT struct ib_mad *p_mad)
126 {
127
128 /* For global destination or Multicast address: */
129 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
130 p_mad->sl = 0;
131 if (is_smi) {
132 p_mad->sqpn = 0;
133 p_mad->dqpn = 0;
134 } else {
135 p_mad->sqpn = 1;
136 p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp;
137 }
138 }
139
__osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)140 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
141 {
142 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
143 osm_vendor_t *p_vend = p_bind->p_vend;
144 VAPI_ret_t status;
145 VAPI_hca_attr_t attr_mod;
146 VAPI_hca_attr_mask_t attr_mask;
147
148 OSM_LOG_ENTER(p_vend->p_log);
149
150 memset(&attr_mod, 0, sizeof(attr_mod));
151 memset(&attr_mask, 0, sizeof(attr_mask));
152
153 attr_mod.is_sm = FALSE;
154 attr_mask = HCA_ATTR_IS_SM;
155
156 status =
157 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
158 &attr_mask);
159 if (status != VAPI_OK) {
160 osm_log(p_vend->p_log, OSM_LOG_ERROR,
161 "__osm_vendor_clear_sm: ERR 5021: "
162 "Unable set 'IS_SM' bit in port attributes (%d).\n",
163 status);
164 }
165
166 OSM_LOG_EXIT(p_vend->p_log);
167 }
168
169 /**********************************************************************
170 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
171 **********************************************************************/
osm_vendor_construct(IN osm_vendor_t * const p_vend)172 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
173 {
174 memset(p_vend, 0, sizeof(*p_vend));
175 cl_thread_construct(&(p_vend->smi_bind.poller));
176 cl_thread_construct(&(p_vend->gsi_bind.poller));
177 }
178
179 /**********************************************************************
180 * DEALOCATE osm_vendor_t
181 **********************************************************************/
osm_vendor_destroy(IN osm_vendor_t * const p_vend)182 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
183 {
184 OSM_LOG_ENTER(p_vend->p_log);
185 osm_transaction_mgr_destroy(p_vend);
186
187 /* Destroy the poller threads */
188 /* HACK: can you destroy an un-initialized thread ? */
189 pthread_cancel(p_vend->smi_bind.poller.osd.id);
190 pthread_cancel(p_vend->gsi_bind.poller.osd.id);
191 cl_thread_destroy(&(p_vend->smi_bind.poller));
192 cl_thread_destroy(&(p_vend->gsi_bind.poller));
193 OSM_LOG_EXIT(p_vend->p_log);
194 }
195
196 /**********************************************************************
197 DEALLOCATE A POINTER TO osm_vendor_t
198 **********************************************************************/
osm_vendor_delete(IN osm_vendor_t ** const pp_vend)199 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
200 {
201 CL_ASSERT(pp_vend);
202
203 osm_vendor_destroy(*pp_vend);
204 free(*pp_vend);
205 *pp_vend = NULL;
206 }
207
208 /**********************************************************************
209 Initializes the vendor:
210 **********************************************************************/
211
212 ib_api_status_t
osm_vendor_init(IN osm_vendor_t * const p_vend,IN osm_log_t * const p_log,IN const uint32_t timeout)213 osm_vendor_init(IN osm_vendor_t * const p_vend,
214 IN osm_log_t * const p_log, IN const uint32_t timeout)
215 {
216 ib_api_status_t status = IB_SUCCESS;
217
218 OSM_LOG_ENTER(p_log);
219
220 p_vend->p_log = p_log;
221 p_vend->p_transaction_mgr = NULL;
222 osm_transaction_mgr_init(p_vend);
223 p_vend->timeout = timeout;
224
225 /* we use the file handle to track the binding */
226 p_vend->smi_bind.ul_dev_fd = -1;
227 p_vend->gsi_bind.ul_dev_fd = -1;
228
229 OSM_LOG_EXIT(p_log);
230 return (status);
231 }
232
233 /**********************************************************************
234 * Create and Initialize osm_vendor_t Object
235 **********************************************************************/
osm_vendor_new(IN osm_log_t * const p_log,IN const uint32_t timeout)236 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
237 IN const uint32_t timeout)
238 {
239 ib_api_status_t status;
240 osm_vendor_t *p_vend;
241
242 OSM_LOG_ENTER(p_log);
243
244 CL_ASSERT(p_log);
245
246 p_vend = malloc(sizeof(*p_vend));
247 if (p_vend != NULL) {
248 memset(p_vend, 0, sizeof(*p_vend));
249
250 status = osm_vendor_init(p_vend, p_log, timeout);
251 if (status != IB_SUCCESS) {
252 osm_vendor_delete(&p_vend);
253 }
254 } else {
255 osm_log(p_vend->p_log, OSM_LOG_ERROR,
256 "osm_vendor_new: ERR 5007: "
257 "Fail to allocate vendor object.\n");
258 }
259
260 OSM_LOG_EXIT(p_log);
261 return (p_vend);
262 }
263
264 /**********************************************************************
265 * TS RCV Thread callback
266 * HACK: - we need to make this support arbitrary size mads.
267 **********************************************************************/
268 void
__osm_ts_rcv_callback(IN osm_ts_bind_info_t * p_bind,IN osm_mad_addr_t * p_mad_addr,IN uint32_t mad_size,IN void * p_mad)269 __osm_ts_rcv_callback(IN osm_ts_bind_info_t * p_bind,
270 IN osm_mad_addr_t * p_mad_addr,
271 IN uint32_t mad_size, IN void *p_mad)
272 {
273 ib_api_status_t status;
274 osm_madw_t *p_req_madw = NULL;
275 osm_madw_t *p_madw;
276 osm_vend_wrap_t *p_new_vw;
277 ib_mad_t *p_mad_buf;
278 osm_log_t *const p_log = p_bind->p_vend->p_log;
279
280 OSM_LOG_ENTER(p_log);
281
282 /* if it is a response MAD we mustbe able to get the request */
283 if (ib_mad_is_response((ib_mad_t *) p_mad)) {
284 /* can we find a matching madw by this payload TID */
285 status =
286 osm_transaction_mgr_get_madw_for_tid(p_bind->p_vend,
287 (ib_mad_t *) p_mad,
288 &p_req_madw);
289 if (status != IB_SUCCESS) {
290 osm_log(p_log, OSM_LOG_ERROR,
291 "__osm_ts_rcv_callback: ERR 5008: "
292 "Error obtaining request madw by TID (%d).\n",
293 status);
294 p_req_madw = NULL;
295 }
296
297 if (p_req_madw == NULL) {
298 osm_log(p_log, OSM_LOG_ERROR,
299 "__osm_ts_rcv_callback: ERR 5009: "
300 "Fail to obtain request madw for receined MAD. Aborting CB.\n");
301 goto Exit;
302 }
303 }
304
305 /* do we have a request ??? */
306 if (p_req_madw == NULL) {
307
308 /* if not - get new osm_madw and arrange it. */
309 /* create the new madw in the pool */
310 p_madw = osm_mad_pool_get(p_bind->p_osm_pool,
311 (osm_bind_handle_t) p_bind,
312 mad_size, p_mad_addr);
313 if (p_madw == NULL) {
314 osm_log(p_log, OSM_LOG_ERROR,
315 "__osm_ts_rcv_callback: ERR 5010: "
316 "Error request for a new madw.\n");
317 goto Exit;
318 }
319 /* HACK: we cust to avoid the const ??? */
320 p_mad_buf = (void *)p_madw->p_mad;
321 } else {
322 /* we have the madw defined during the send and stored in the vend_wrap */
323 /* we need to make sure the wrapper is correctly init there */
324 CL_ASSERT(p_req_madw->vend_wrap.p_resp_madw != 0);
325 p_madw = p_req_madw->vend_wrap.p_resp_madw;
326
327 CL_ASSERT(p_madw->h_bind);
328 p_mad_buf =
329 osm_vendor_get(p_madw->h_bind, mad_size,
330 &p_madw->vend_wrap);
331
332 if (p_mad_buf == NULL) {
333 osm_log(p_log, OSM_LOG_ERROR,
334 "__osm_ts_rcv_callback: ERR 5011: "
335 "Unable to acquire wire MAD.\n");
336
337 goto Exit;
338 }
339
340 /*
341 Finally, attach the wire MAD to this wrapper.
342 */
343 osm_madw_set_mad(p_madw, p_mad_buf);
344 }
345
346 /* init some fields of the vendor wrapper */
347 p_new_vw = osm_madw_get_vend_ptr(p_madw);
348 p_new_vw->h_bind = p_bind;
349 p_new_vw->size = mad_size;
350 p_new_vw->p_resp_madw = NULL;
351 p_new_vw->p_mad_buf = p_mad_buf;
352
353 memcpy(p_new_vw->p_mad_buf, p_mad, mad_size);
354
355 /* attach the buffer to the wrapper */
356 p_madw->p_mad = p_mad_buf;
357
358 /* we can also make sure we marked the size and bind on the returned madw */
359 p_madw->h_bind = p_new_vw->h_bind;
360
361 /* call the CB */
362 (*(osm_vend_mad_recv_callback_t) p_bind->rcv_callback)
363 (p_madw, p_bind->client_context, p_req_madw);
364
365 Exit:
366 OSM_LOG_EXIT(p_log);
367 }
368
369 /**********************************************************************
370 * TS Send callback : invoked after each send
371 *
372 **********************************************************************/
373 void
__osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p,IN boolean_t is_resp,IN osm_madw_t * madw_p,IN IB_comp_status_t status)374 __osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p,
375 IN boolean_t is_resp,
376 IN osm_madw_t * madw_p, IN IB_comp_status_t status)
377 {
378 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
379 osm_vend_wrap_t *p_vw;
380
381 OSM_LOG_ENTER(p_log);
382
383 osm_log(p_log, OSM_LOG_DEBUG,
384 "__osm_ts_send_callback: INFO 1008: "
385 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
386
387 /* we need to handle requests and responses differently */
388 if (is_resp) {
389 if (status != IB_COMP_SUCCESS) {
390 osm_log(p_log, OSM_LOG_ERROR,
391 "__osm_ts_send_callback: ERR 5012: "
392 "Error Sending Response MADW:%p.\n", madw_p);
393 } else {
394 osm_log(p_log, OSM_LOG_DEBUG,
395 "__osm_ts_send_callback: DBG 1008: "
396 "Completed Sending Response MADW:%p.\n",
397 madw_p);
398 }
399
400 /* if we are a response - we need to clean it up */
401 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
402 } else {
403
404 /* this call back is invoked on completion of send - error or not */
405 if (status != IB_COMP_SUCCESS) {
406
407 osm_log(p_log, OSM_LOG_ERROR,
408 "__osm_ts_send_callback: ERR 5013: "
409 "Received an Error from IB_MGT Send (%d).\n",
410 status);
411
412 p_vw = osm_madw_get_vend_ptr(madw_p);
413 CL_ASSERT(p_vw);
414
415 /*
416 Return any wrappers to the pool that may have been
417 pre-emptively allocated to handle a receive.
418 */
419 if (p_vw->p_resp_madw) {
420 osm_mad_pool_put(bind_info_p->p_osm_pool,
421 p_vw->p_resp_madw);
422 p_vw->p_resp_madw = NULL;
423 }
424
425 /* invoke the CB */
426 (*(osm_vend_mad_send_err_callback_t) bind_info_p->
427 send_err_callback)
428 (bind_info_p->client_context, madw_p);
429 } else {
430 /* successful request send - do nothing - the response will need the
431 out mad */
432 osm_log(p_log, OSM_LOG_DEBUG,
433 "__osm_ts_send_callback: DBG 1008: "
434 "Completed Sending Request MADW:%p.\n", madw_p);
435 }
436 }
437
438 OSM_LOG_EXIT(p_log);
439 }
440
441 /**********************************************************************
442 * Poller thread:
443 * Always receive 256byte mads from the devcie file
444 **********************************************************************/
__osm_vendor_ts_poller(IN void * p_ptr)445 void __osm_vendor_ts_poller(IN void *p_ptr)
446 {
447 int ts_ret_code;
448 struct ib_mad mad;
449 osm_mad_addr_t mad_addr;
450 osm_ts_bind_info_t *const p_bind = (osm_ts_bind_info_t *) p_ptr;
451
452 OSM_LOG_ENTER(p_bind->p_vend->p_log);
453 /* we set the type of cancelation for this thread */
454 pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
455
456 while (1) {
457 /* we read one mad at a time and pass it to the read callback function */
458 ts_ret_code = read(p_bind->ul_dev_fd, &mad, sizeof(mad));
459 if (ts_ret_code != sizeof(mad)) {
460 osm_log(p_bind->p_vend->p_log, OSM_LOG_ERROR,
461 "__osm_vendor_ts_poller: ERR 5003: "
462 "error with read, bytes = %d, errno = %d\n",
463 ts_ret_code, errno);
464 } else {
465 osm_log(p_bind->p_vend->p_log, OSM_LOG_DEBUG,
466 "__osm_vendor_ts_poller: "
467 "MAD QPN:%d SLID:0x%04x class:0x%02x "
468 "__osm_vendor_ts_poller:0x%02x attr:0x%04x status:0x%04x "
469 "__osm_vendor_ts_poller:0x%016" PRIx64 "\n",
470 cl_ntoh32(mad.dqpn),
471 cl_ntoh16(mad.slid),
472 mad.mgmt_class,
473 mad.r_method,
474 cl_ntoh16(mad.attribute_id),
475 cl_ntoh16(mad.status),
476 cl_ntoh64(mad.transaction_id));
477
478 /* first arrange an address */
479 __osm_ts_conv_mad_rcv_desc_to_osm_addr(p_bind->p_vend,
480 &mad,
481 (((ib_mad_t *) &
482 mad)->
483 mgmt_class ==
484 IB_MCLASS_SUBN_LID)
485 ||
486 (((ib_mad_t *) &
487 mad)->
488 mgmt_class ==
489 IB_MCLASS_SUBN_DIR),
490 &mad_addr);
491
492 /* call the receiver callback */
493 /* HACK: this should be replaced with a call to the RMPP Assembly ... */
494 __osm_ts_rcv_callback(p_bind, &mad_addr, 256, &mad);
495 }
496 }
497
498 OSM_LOG_EXIT(p_bind->p_vend->p_log);
499 }
500
501 /**********************************************************************
502 * BINDs a callback (rcv and send error) for a given class and method
503 * defined by the given: osm_bind_info_t
504 **********************************************************************/
505 osm_bind_handle_t
osm_vendor_bind(IN osm_vendor_t * const p_vend,IN osm_bind_info_t * const p_user_bind,IN osm_mad_pool_t * const p_mad_pool,IN osm_vend_mad_recv_callback_t mad_recv_callback,IN osm_vend_mad_send_err_callback_t send_err_callback,IN void * context)506 osm_vendor_bind(IN osm_vendor_t * const p_vend,
507 IN osm_bind_info_t * const p_user_bind,
508 IN osm_mad_pool_t * const p_mad_pool,
509 IN osm_vend_mad_recv_callback_t mad_recv_callback,
510 IN osm_vend_mad_send_err_callback_t send_err_callback,
511 IN void *context)
512 {
513 ib_net64_t port_guid;
514 osm_ts_bind_info_t *p_bind = NULL;
515 VAPI_hca_hndl_t hca_hndl;
516 VAPI_hca_id_t hca_id;
517 uint32_t port_num;
518 ib_api_status_t status;
519 int device_fd;
520 char device_file[16];
521 osm_ts_user_mad_filter filter;
522 int ts_ioctl_ret;
523 int qpn;
524
525 OSM_LOG_ENTER(p_vend->p_log);
526
527 CL_ASSERT(p_mad_pool);
528
529 port_guid = p_user_bind->port_guid;
530
531 osm_log(p_vend->p_log, OSM_LOG_INFO,
532 "osm_vendor_bind: "
533 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
534
535 switch (p_user_bind->mad_class) {
536 case IB_MCLASS_SUBN_LID:
537 case IB_MCLASS_SUBN_DIR:
538 p_bind = &(p_vend->smi_bind);
539 qpn = 0;
540 break;
541
542 case IB_MCLASS_SUBN_ADM:
543 default:
544 p_bind = &(p_vend->gsi_bind);
545 qpn = 1;
546 break;
547 }
548
549 /* Make sure we did not previously opened the file */
550 if (p_bind->ul_dev_fd >= 0) {
551 osm_log(p_vend->p_log, OSM_LOG_ERROR,
552 "osm_vendor_bind: ERR 5004: "
553 "Already binded to port %u\n", p_bind->port_num);
554 goto Exit;
555 }
556
557 /*
558 We need to figure out what is the TS file name to attach to.
559 I guess it is following the index of the port in the table of
560 ports.
561 */
562
563 /* obtain the hca name and port num from the guid */
564 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
565 "osm_vendor_bind: "
566 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
567 cl_ntoh64(port_guid));
568 status =
569 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
570 &hca_id, &port_num);
571 if (status != IB_SUCCESS) {
572 osm_log(p_vend->p_log, OSM_LOG_ERROR,
573 "osm_vendor_bind: ERR 5005: "
574 "Fail to find port number of port guid:0x%016" PRIx64
575 "\n", port_guid);
576 goto Exit;
577 }
578
579 /* the file name is just /dev/ts_ua0: */
580 strcpy(device_file, "/dev/ts_ua0");
581
582 osm_log(p_vend->p_log, OSM_LOG_ERROR,
583 "osm_vendor_bind: " "Opening TS UL dev file:%s\n", device_file);
584
585 /* Open the file ... */
586 device_fd = open(device_file, O_RDWR);
587 if (device_fd < 0) {
588 osm_log(p_vend->p_log, OSM_LOG_ERROR,
589 "osm_vendor_bind: ERR 5006: "
590 "Fail to open TS UL dev file:%s\n", device_file);
591 goto Exit;
592 }
593
594 /* track this bind request info */
595 p_bind->ul_dev_fd = device_fd;
596 p_bind->port_num = port_num;
597 p_bind->p_vend = p_vend;
598 p_bind->client_context = context;
599 p_bind->rcv_callback = mad_recv_callback;
600 p_bind->send_err_callback = send_err_callback;
601 p_bind->p_osm_pool = p_mad_pool;
602 p_bind->hca_hndl = hca_hndl;
603
604 /*
605 * Create the MAD filter on this file handle.
606 */
607 filter.port = port_num;
608
609 filter.qpn = qpn;
610 filter.mgmt_class = p_user_bind->mad_class;
611 filter.direction = TS_IB_MAD_DIRECTION_IN;
612 filter.mask =
613 TS_IB_MAD_FILTER_DIRECTION |
614 TS_IB_MAD_FILTER_PORT |
615 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
616
617 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
618 if (ts_ioctl_ret < 0) {
619 osm_log(p_vend->p_log, OSM_LOG_ERROR,
620 "osm_vendor_bind: ERR 5014: "
621 "Fail to register MAD filter with err:%u\n",
622 ts_ioctl_ret);
623 goto Exit;
624 }
625
626 /* Initialize the listener thread for this port */
627 status = cl_thread_init(&p_bind->poller,
628 __osm_vendor_ts_poller, p_bind,
629 "osm ts poller");
630 if (status != IB_SUCCESS)
631 goto Exit;
632
633 Exit:
634 OSM_LOG_EXIT(p_vend->p_log);
635 return ((osm_bind_handle_t) p_bind);
636 }
637
638 /**********************************************************************
639 Get a mad from the lower level.
640 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
641 **********************************************************************/
osm_vendor_get(IN osm_bind_handle_t h_bind,IN const uint32_t mad_size,IN osm_vend_wrap_t * const p_vw)642 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
643 IN const uint32_t mad_size,
644 IN osm_vend_wrap_t * const p_vw)
645 {
646 ib_mad_t *p_mad;
647 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
648 osm_vendor_t *p_vend = p_bind->p_vend;
649
650 OSM_LOG_ENTER(p_vend->p_log);
651
652 CL_ASSERT(p_vw);
653
654 p_vw->size = mad_size;
655
656 /* allocate it */
657 p_mad = (ib_mad_t *) malloc(p_vw->size);
658 if (p_mad == NULL) {
659 osm_log(p_vend->p_log, OSM_LOG_ERROR,
660 "osm_vendor_get: ERR 5022: "
661 "Error Obtaining MAD buffer.\n");
662 goto Exit;
663 }
664
665 memset(p_mad, 0, p_vw->size);
666
667 /* track locally */
668 p_vw->p_mad_buf = p_mad;
669 p_vw->h_bind = h_bind;
670 p_vw->p_resp_madw = NULL;
671
672 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
673 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
674 "osm_vendor_get: "
675 "Acquired MAD %p, size = %u.\n", p_mad, p_vw->size);
676 }
677
678 Exit:
679 OSM_LOG_EXIT(p_vend->p_log);
680 return (p_mad);
681 }
682
683 /**********************************************************************
684 * Return a MAD by providing it's wrapper object.
685 **********************************************************************/
686 void
osm_vendor_put(IN osm_bind_handle_t h_bind,IN osm_vend_wrap_t * const p_vw)687 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
688 {
689 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
690 osm_vendor_t *p_vend = p_bind->p_vend;
691 osm_madw_t *p_madw;
692
693 OSM_LOG_ENTER(p_vend->p_log);
694
695 CL_ASSERT(p_vw);
696 CL_ASSERT(p_vw->p_mad_buf);
697
698 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
699 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
700 "osm_vendor_put: " "Retiring MAD %p.\n",
701 p_vw->p_mad_buf);
702 }
703
704 /*
705 * We moved the removal of the transaction to immediatly after
706 * it was looked up.
707 */
708
709 /* free the mad but the wrapper is part of the madw object */
710 free(p_vw->p_mad_buf);
711 p_vw->p_mad_buf = NULL;
712 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
713 p_madw->p_mad = NULL;
714
715 OSM_LOG_EXIT(p_vend->p_log);
716 }
717
718 /**********************************************************************
719 Actually Send a MAD
720
721 MADs are buffers of type: struct ib_mad - so they are limited by size.
722 This is for internal use by osm_vendor_send and the transaction mgr
723 retry too.
724 **********************************************************************/
725 ib_api_status_t
osm_ts_send_mad(IN osm_ts_bind_info_t * p_bind,IN osm_madw_t * const p_madw)726 osm_ts_send_mad(IN osm_ts_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
727 {
728 osm_vendor_t *const p_vend = p_bind->p_vend;
729 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
730 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
731 struct ib_mad ts_mad;
732 int ret;
733 ib_api_status_t status;
734
735 OSM_LOG_ENTER(p_vend->p_log);
736
737 /*
738 * Copy the MAD over to the sent mad
739 */
740 memcpy(&ts_mad, p_mad, 256);
741
742 /*
743 * For all sends other than directed route SM MADs,
744 * acquire an address vector for the destination.
745 */
746 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
747 __osm_ts_conv_osm_addr_to_ts_addr(p_mad_addr,
748 p_mad->mgmt_class ==
749 IB_MCLASS_SUBN_LID, &ts_mad);
750 } else {
751 /* is a directed route - we need to construct a permissive address */
752 /* we do not need port number since it is part of the mad_hndl */
753 ts_mad.dlid = IB_LID_PERMISSIVE;
754 ts_mad.slid = IB_LID_PERMISSIVE;
755 }
756 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
757 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
758 ts_mad.sqpn = 0;
759 ts_mad.dqpn = 0;
760 } else {
761 ts_mad.sqpn = 1;
762 ts_mad.dqpn = 1;
763 }
764 ts_mad.port = p_bind->port_num;
765
766 /* send it */
767 ret = write(p_bind->ul_dev_fd, &ts_mad, sizeof(ts_mad));
768
769 if (ret != sizeof(ts_mad)) {
770 osm_log(p_vend->p_log, OSM_LOG_ERROR,
771 "osm_ts_send_mad: ERR 5026: "
772 "Error sending mad (%d).\n", ret);
773 status = IB_ERROR;
774 goto Exit;
775 }
776
777 status = IB_SUCCESS;
778
779 Exit:
780 OSM_LOG_EXIT(p_vend->p_log);
781 return (status);
782 }
783
784 /**********************************************************************
785 Send a MAD through.
786
787 What is unclear to me is the need for the setting of all the MAD Wrapper
788 fields. Seems like the OSM uses these values during it's processing...
789 **********************************************************************/
790 ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,IN osm_madw_t * const p_madw,IN boolean_t const resp_expected)791 osm_vendor_send(IN osm_bind_handle_t h_bind,
792 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
793 {
794 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
795 osm_vendor_t *const p_vend = p_bind->p_vend;
796 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
797 ib_api_status_t status;
798
799 OSM_LOG_ENTER(p_vend->p_log);
800
801 /*
802 * If a response is expected to this MAD, then preallocate
803 * a mad wrapper to contain the wire MAD received in the
804 * response. Allocating a wrapper here allows for easier
805 * failure paths than after we already received the wire mad.
806 */
807 if (resp_expected == TRUE) {
808 /* we track it in the vendor wrapper */
809 p_vw->p_resp_madw =
810 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
811 if (p_vw->p_resp_madw == NULL) {
812 osm_log(p_vend->p_log, OSM_LOG_ERROR,
813 "osm_vendor_send: ERR 5024: "
814 "Unable to allocate MAD wrapper.\n");
815 status = IB_INSUFFICIENT_RESOURCES;
816 goto Exit;
817 }
818
819 /* put some minimal info on that wrapper */
820 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
821
822 /* we also want to track it in the TID based map */
823 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t *)
824 p_bind, p_madw);
825 if (status != IB_SUCCESS) {
826 osm_log(p_vend->p_log, OSM_LOG_ERROR,
827 "osm_vendor_send: ERR 5025: "
828 "Error inserting request madw by TID (%d).\n",
829 status);
830 }
831 } else
832 p_vw->p_resp_madw = NULL;
833
834 /* do the actual send */
835 /* HACK: to be replaced by call to RMPP Segmentation */
836 status = osm_ts_send_mad(p_bind, p_madw);
837
838 /* we do not get an asycn callback so call it ourselves */
839 /* this will handle all cleanup if neccessary */
840 __osm_ts_send_callback(p_bind, !resp_expected, p_madw, status);
841
842 Exit:
843 OSM_LOG_EXIT(p_vend->p_log);
844 return (status);
845 }
846
847 /**********************************************************************
848 * the idea here is to change the content of the bind such that it
849 * will hold the local address used for sending directed route by the SMA.
850 **********************************************************************/
osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)851 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
852 {
853 osm_vendor_t *p_vend = ((osm_ts_bind_info_t *) h_bind)->p_vend;
854
855 OSM_LOG_ENTER(p_vend->p_log);
856
857 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
858 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
859
860 OSM_LOG_EXIT(p_vend->p_log);
861
862 return (IB_SUCCESS);
863 }
864
osm_vendor_set_sm(IN osm_bind_handle_t h_bind,IN boolean_t is_sm_val)865 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
866 {
867 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
868 osm_vendor_t *p_vend = p_bind->p_vend;
869 VAPI_ret_t status;
870 VAPI_hca_attr_t attr_mod;
871 VAPI_hca_attr_mask_t attr_mask;
872
873 OSM_LOG_ENTER(p_vend->p_log);
874
875 memset(&attr_mod, 0, sizeof(attr_mod));
876 memset(&attr_mask, 0, sizeof(attr_mask));
877
878 attr_mod.is_sm = is_sm_val;
879 attr_mask = HCA_ATTR_IS_SM;
880
881 status =
882 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
883 &attr_mask);
884 if (status != VAPI_OK) {
885 osm_log(p_vend->p_log, OSM_LOG_ERROR,
886 "osm_vendor_set_sm: ERR 5027: "
887 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
888 is_sm_val, status);
889 }
890
891 OSM_LOG_EXIT(p_vend->p_log);
892 }
893
osm_vendor_set_debug(IN osm_vendor_t * const p_vend,IN int32_t level)894 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
895 {
896
897 }
898