1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 #if HAVE_CONFIG_H
37 # include <config.h>
38 #endif /* HAVE_CONFIG_H */
39
40 #ifdef OSM_VENDOR_INTF_MTL
41
42 #include <stdlib.h>
43 #include <string.h>
44 #include <opensm/osm_helper.h>
45 #include <opensm/osm_log.h>
46 /* HACK - I do not know how to prevent complib from loading kernel H files */
47 #undef __init
48 #include <vendor/osm_vendor_mtl.h>
49 #include <vendor/osm_vendor_api.h>
50 #include <opensm/osm_subnet.h>
51 #include <opensm/osm_opensm.h>
52 #include <vendor/osm_vendor_mtl_transaction_mgr.h>
53 #include <vendor/osm_mtl_bind.h>
54
55 /*
56 Since a race can accure on requests. Meaning - a response is received before
57 the send_callback is called - we will save both the madw_p and the fact
58 whether or not it is a response. A race can occure only on requests that did
59 not fail, and then the madw_p will be put back in the pool before the callback.
60 */
__osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)61 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
62 {
63 uint64_t wrid = 0;
64
65 CL_ASSERT(p_madw->p_mad);
66
67 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
68 wrid = (wrid << 1) |
69 ib_mad_is_response(p_madw->p_mad);
70 return wrid;
71 }
72
73 void
__osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,OUT uint8_t * is_resp,OUT osm_madw_t ** pp_madw)74 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
75 OUT uint8_t * is_resp,
76 OUT osm_madw_t ** pp_madw)
77 {
78 *is_resp = wrid & 0x0000000000000001;
79 wrid = wrid >> 1;
80 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
81 }
82
83 /**********************************************************************
84 * IB_MGT to OSM ADDRESS VECTOR
85 **********************************************************************/
86 void
__osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,IN uint8_t is_smi,OUT osm_mad_addr_t * p_mad_addr)87 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
88 IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
89 IN uint8_t is_smi,
90 OUT osm_mad_addr_t * p_mad_addr)
91 {
92 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
93 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
94 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */
95 p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
96 if (is_smi) {
97 /* SMI */
98 p_mad_addr->addr_type.smi.source_lid =
99 cl_hton16(p_rcv_desc->remote_lid);
100 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */
101 } else {
102 /* GSI */
103 /* seems to me there is a IBMGT bug reversing the QPN ... */
104 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
105 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
106
107 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
108 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
109 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
110 /* the full PKey table - than go by the index. */
111 /* since this does not seem reasonable to me I simply use the default */
112 /* There is a TAVOR limitation that only one P_KEY is supported per */
113 /* QP - so QP1 must use IB_DEFAULT_PKEY */
114 p_mad_addr->addr_type.gsi.pkey_ix = 0;
115 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
116
117 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
118 /* copy the GRH data if relevant */
119 if (p_mad_addr->addr_type.gsi.global_route) {
120 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
121 ib_grh_set_ver_class_flow(p_rcv_desc->grh.
122 IP_version,
123 p_rcv_desc->grh.
124 traffic_class,
125 p_rcv_desc->grh.
126 flow_label);
127 p_mad_addr->addr_type.gsi.grh_info.hop_limit =
128 p_rcv_desc->grh.hop_limit;
129 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
130 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
131 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
132 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
133 }
134 }
135 }
136
137 /**********************************************************************
138 * OSM ADDR VECTOR TO IB_MGT
139 **********************************************************************/
140 void
__osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,IN uint8_t is_smi,OUT IB_ud_av_t * p_av)141 __osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,
142 IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
143 {
144
145 /* For global destination or Multicast address: */
146 u_int8_t ver;
147
148 memset(p_av, 0, sizeof(IB_ud_av_t));
149
150 p_av->src_path_bits = p_mad_addr->path_bits;
151 p_av->static_rate = p_mad_addr->static_rate;
152 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
153
154 if (is_smi) {
155 p_av->sl = 0; /* Just to note we use 0 here. */
156 } else {
157 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
158 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
159
160 if (p_mad_addr->addr_type.gsi.global_route) {
161 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
162 grh_info.ver_class_flow, &ver,
163 &p_av->traffic_class,
164 &p_av->flow_label);
165 p_av->hop_limit =
166 p_mad_addr->addr_type.gsi.grh_info.hop_limit;
167 p_av->sgid_index = 0; /* we always use source GID 0 */
168 memcpy(&p_av->dgid,
169 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
170 sizeof(ib_net64_t));
171
172 }
173 }
174 }
175
__osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)176 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
177 {
178 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
179 osm_vendor_t *p_vend = p_bind->p_vend;
180 VAPI_ret_t status;
181 VAPI_hca_attr_t attr_mod;
182 VAPI_hca_attr_mask_t attr_mask;
183
184 OSM_LOG_ENTER(p_vend->p_log);
185
186 memset(&attr_mod, 0, sizeof(attr_mod));
187 memset(&attr_mask, 0, sizeof(attr_mask));
188
189 attr_mod.is_sm = FALSE;
190 attr_mask = HCA_ATTR_IS_SM;
191
192 status =
193 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
194 &attr_mask);
195 if (status != VAPI_OK) {
196 osm_log(p_vend->p_log, OSM_LOG_ERROR,
197 "__osm_vendor_clear_sm: ERR 3C21: "
198 "Unable set 'IS_SM' bit in port attributes (%d).\n",
199 status);
200 }
201
202 OSM_LOG_EXIT(p_vend->p_log);
203 }
204
205 /**********************************************************************
206 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
207 **********************************************************************/
osm_vendor_construct(IN osm_vendor_t * const p_vend)208 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
209 {
210 memset(p_vend, 0, sizeof(*p_vend));
211 }
212
213 /**********************************************************************
214 * DEALOCATE osm_vendor_t
215 **********************************************************************/
osm_vendor_destroy(IN osm_vendor_t * const p_vend)216 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
217 {
218 osm_vendor_mgt_bind_t *vendor_mgt_bind_p;
219 IB_MGT_ret_t mgt_ret;
220 OSM_LOG_ENTER(p_vend->p_log);
221
222 if (p_vend->h_al != NULL) {
223 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
224 if (vendor_mgt_bind_p->gsi_init) {
225
226 /* un register the class */
227 /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */
228 mgt_ret =
229 IB_MGT_unbind_gsi_class(vendor_mgt_bind_p->
230 gsi_mads_hdl,
231 IB_MCLASS_SUBN_ADM);
232 if (mgt_ret != IB_MGT_OK) {
233 osm_log(p_vend->p_log, OSM_LOG_ERROR,
234 "osm_vendor_destroy: ERR 3C03: "
235 "Fail to unbind the SA class.\n");
236 }
237
238 /* un bind the handle */
239 if (IB_MGT_release_handle
240 (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) {
241 osm_log(p_vend->p_log, OSM_LOG_ERROR,
242 "osm_vendor_destroy: ERR 3C02: "
243 "Fail to unbind the SA GSI handle.\n");
244 }
245 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
246 "osm_vendor_destroy: DBG 1002: "
247 "Unbind the GSI handles.\n");
248 }
249 if (vendor_mgt_bind_p->smi_init) {
250 /* first - clear the IS_SM in the capability mask */
251 __osm_vendor_clear_sm((osm_bind_handle_t)
252 (vendor_mgt_bind_p->smi_p_bind));
253
254 /* un register the class */
255 mgt_ret =
256 IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl);
257 if (mgt_ret != IB_MGT_OK) {
258 osm_log(p_vend->p_log, OSM_LOG_ERROR,
259 "osm_vendor_destroy: ERR 3C04: "
260 "Fail to unbind the SM class.\n");
261 }
262
263 /* un bind the handle */
264 if (IB_MGT_release_handle
265 (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) {
266 osm_log(p_vend->p_log, OSM_LOG_ERROR,
267 "osm_vendor_destroy: ERR 3C05: "
268 "Fail to unbind the SMI handle.\n");
269 }
270 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
271 "osm_vendor_destroy: DBG 1003: "
272 "Unbind the SMI handles.\n");
273
274 }
275 }
276 osm_transaction_mgr_destroy(p_vend);
277 /* __osm_mtl_destroy_tid_mad_map( p_vend ); */
278 OSM_LOG_EXIT(p_vend->p_log);
279 }
280
281 /**********************************************************************
282 DEALLOCATE A POINTER TO osm_vendor_t
283 **********************************************************************/
osm_vendor_delete(IN osm_vendor_t ** const pp_vend)284 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
285 {
286 CL_ASSERT(pp_vend);
287
288 osm_vendor_destroy(*pp_vend);
289 free(*pp_vend);
290 *pp_vend = NULL;
291 }
292
293 /**********************************************************************
294 * This proc actuall binds the handle to the lower level.
295 *
296 * We might have here as a result a casting of our struct to the ib_al_handle_t
297 *
298 * Q: Do we need 2 of those - one for MSI and one for GSI ?
299 * A: Yes! We should be able to do the SA too. So we need a struct!
300 *
301 **********************************************************************/
302
303 ib_api_status_t
osm_vendor_init(IN osm_vendor_t * const p_vend,IN osm_log_t * const p_log,IN const uint32_t timeout)304 osm_vendor_init(IN osm_vendor_t * const p_vend,
305 IN osm_log_t * const p_log, IN const uint32_t timeout)
306 {
307 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
308 ib_api_status_t status = IB_SUCCESS;
309
310 OSM_LOG_ENTER(p_log);
311
312 p_vend->p_log = p_log;
313
314 /*
315 * HACK: We need no handle. Assuming the driver is up.
316 */
317 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *)
318 malloc(sizeof(osm_vendor_mgt_bind_t));
319 if (ib_mgt_hdl_p == NULL) {
320 osm_log(p_vend->p_log, OSM_LOG_ERROR,
321 "osm_vendor_init: ERR 3C06: "
322 "Fail to allocate vendor mgt handle.\n");
323 goto Exit;
324 }
325
326 ib_mgt_hdl_p->smi_init = FALSE;
327 ib_mgt_hdl_p->gsi_init = FALSE;
328 /* cast it into the ib_al_handle_t h_al */
329 p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p;
330 p_vend->p_transaction_mgr = NULL;
331 osm_transaction_mgr_init(p_vend);
332 /* p_vend->madw_by_tid_map_p = NULL; */
333 /* __osm_mtl_init_tid_mad_map( p_vend ); */
334 p_vend->timeout = timeout;
335
336 Exit:
337 OSM_LOG_EXIT(p_log);
338 return (status);
339 }
340
341 /**********************************************************************
342 * Create and Initialize osm_vendor_t Object
343 **********************************************************************/
osm_vendor_new(IN osm_log_t * const p_log,IN const uint32_t timeout)344 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
345 IN const uint32_t timeout)
346 {
347 ib_api_status_t status;
348 osm_vendor_t *p_vend;
349
350 OSM_LOG_ENTER(p_log);
351
352 CL_ASSERT(p_log);
353
354 p_vend = malloc(sizeof(*p_vend));
355 if (p_vend != NULL) {
356 memset(p_vend, 0, sizeof(*p_vend));
357 status = osm_vendor_init(p_vend, p_log, timeout);
358 if (status != IB_SUCCESS) {
359 osm_vendor_delete(&p_vend);
360 }
361 } else {
362 osm_log(p_vend->p_log, OSM_LOG_ERROR,
363 "osm_vendor_new: ERR 3C07: "
364 "Fail to allocate vendor object.\n");
365 }
366
367 OSM_LOG_EXIT(p_log);
368 return (p_vend);
369 }
370
371 /**********************************************************************
372 * IB_MGT RCV callback
373 *
374 **********************************************************************/
375 void
__osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,IN void * private_ctx_p,IN void * payload_p,IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)376 __osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,
377 IN void *private_ctx_p,
378 IN void *payload_p,
379 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
380 {
381 IB_MGT_ret_t status;
382 osm_mtl_bind_info_t *bind_info_p = private_ctx_p;
383 osm_madw_t *req_madw_p = NULL;
384 osm_madw_t *madw_p;
385 osm_vend_wrap_t *p_new_vw;
386 osm_mad_addr_t mad_addr;
387 ib_mad_t *mad_buf_p;
388 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
389
390 OSM_LOG_ENTER(p_log);
391
392 /* if it is a response MAD we mustbe able to get the request */
393 if (ib_mad_is_response((ib_mad_t *) payload_p)) {
394 /* can we find a matching madw by this payload TID */
395 status =
396 osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend,
397 (ib_mad_t *) payload_p,
398 &req_madw_p);
399 if (status != IB_MGT_OK) {
400 osm_log(p_log, OSM_LOG_ERROR,
401 "__osm_mtl_rcv_callback: ERR 3C08: "
402 "Error obtaining request madw by TID (%d).\n",
403 status);
404 req_madw_p = NULL;
405 }
406
407 if (req_madw_p == NULL) {
408 osm_log(p_log, OSM_LOG_ERROR,
409 "__osm_mtl_rcv_callback: ERR 3C09: "
410 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n",
411 ((ib_mad_t *) payload_p)->method,
412 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id)
413
414 );
415 goto Exit;
416 }
417 }
418
419 /* do we have a request ??? */
420 if (req_madw_p == NULL) {
421
422 /* first arrange an address */
423 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend,
424 rcv_remote_info_p,
425 (((ib_mad_t *)
426 payload_p)->
427 mgmt_class ==
428 IB_MCLASS_SUBN_LID)
429 || (((ib_mad_t *)
430 payload_p)->
431 mgmt_class ==
432 IB_MCLASS_SUBN_DIR),
433 &mad_addr);
434
435 osm_log(p_log, OSM_LOG_ERROR,
436 "__osm_mtl_rcv_callback: : "
437 "Received MAD from QP:%X.\n",
438 cl_ntoh32(mad_addr.addr_type.gsi.remote_qp)
439 );
440
441 /* if not - get new osm_madw and arrange it. */
442 /* create the new madw in the pool */
443 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool,
444 (osm_bind_handle_t) bind_info_p,
445 MAD_BLOCK_SIZE, &mad_addr);
446 if (madw_p == NULL) {
447 osm_log(p_log, OSM_LOG_ERROR,
448 "__osm_mtl_rcv_callback: ERR 3C10: "
449 "Error request for a new madw.\n");
450 goto Exit;
451 }
452 /* HACK: we cust to avoid the const ??? */
453 mad_buf_p = (void *)madw_p->p_mad;
454 } else {
455 /* we have the madw defined during the send and stored in the vend_wrap */
456 /* we need to make sure the wrapper is correctly init there */
457 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0);
458 madw_p = req_madw_p->vend_wrap.p_resp_madw;
459
460 /* HACK: we do not Support RMPP */
461 CL_ASSERT(madw_p->h_bind);
462 mad_buf_p =
463 osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE,
464 &madw_p->vend_wrap);
465
466 if (mad_buf_p == NULL) {
467 osm_log(p_log, OSM_LOG_ERROR,
468 "__osm_mtl_rcv_callback: ERR 3C11: "
469 "Unable to acquire wire MAD.\n");
470
471 goto Exit;
472 }
473
474 /*
475 Finally, attach the wire MAD to this wrapper.
476 */
477 osm_madw_set_mad(madw_p, mad_buf_p);
478
479 /* also we need to handle the size of the mad since we did not init ... */
480 madw_p->mad_size = MAD_BLOCK_SIZE;
481 }
482
483 /* init some fields of the vendor wrapper */
484 p_new_vw = osm_madw_get_vend_ptr(madw_p);
485 p_new_vw->h_bind = bind_info_p;
486 p_new_vw->size = MAD_BLOCK_SIZE;
487 p_new_vw->p_resp_madw = NULL;
488 p_new_vw->mad_buf_p = mad_buf_p;
489
490 /* HACK: We do not support RMPP in receiving MADS */
491 memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE);
492
493 /* attach the buffer to the wrapper */
494 madw_p->p_mad = mad_buf_p;
495
496 /* we can also make sure we marked the size and bind on the returned madw */
497 madw_p->h_bind = p_new_vw->h_bind;
498
499 /* call the CB */
500 (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context,
501 req_madw_p);
502
503 Exit:
504 OSM_LOG_EXIT(p_log);
505 }
506
507 /**********************************************************************
508 * IB_MGT Send callback : invoked after each send
509 *
510 **********************************************************************/
511 void
__osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,IN u_int64_t wrid,IN IB_comp_status_t status,IN void * private_ctx_p)512 __osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,
513 IN u_int64_t wrid,
514 IN IB_comp_status_t status, IN void *private_ctx_p)
515 {
516 osm_madw_t *madw_p;
517 osm_mtl_bind_info_t *bind_info_p =
518 (osm_mtl_bind_info_t *) private_ctx_p;
519 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
520 osm_vend_wrap_t *p_vw;
521 uint8_t is_resp;
522
523 OSM_LOG_ENTER(p_log);
524
525 /* obtain the madp from the wrid */
526 __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p);
527
528 osm_log(p_log, OSM_LOG_DEBUG,
529 "__osm_mtl_send_callback: INFO 1008: "
530 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
531
532 /* we need to handle requests and responses differently */
533 if (is_resp) {
534 if (status != IB_COMP_SUCCESS) {
535 osm_log(p_log, OSM_LOG_ERROR,
536 "__osm_mtl_send_callback: ERR 3C12: "
537 "Error Sending Response MADW:%p.\n", madw_p);
538 } else {
539 osm_log(p_log, OSM_LOG_DEBUG,
540 "__osm_mtl_send_callback: DBG 1008: "
541 "Completed Sending Response MADW:%p.\n",
542 madw_p);
543 }
544
545 /* if we are a response - we need to clean it up */
546 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
547 } else {
548
549 /* this call back is invoked on completion of send - error or not */
550 if (status != IB_COMP_SUCCESS) {
551
552 osm_log(p_log, OSM_LOG_ERROR,
553 "__osm_mtl_send_callback: ERR 3C13: "
554 "Received an Error from IB_MGT Send (%d).\n",
555 status);
556
557 p_vw = osm_madw_get_vend_ptr(madw_p);
558 CL_ASSERT(p_vw);
559
560 /*
561 Return any wrappers to the pool that may have been
562 pre-emptively allocated to handle a receive.
563 */
564 if (p_vw->p_resp_madw) {
565 osm_mad_pool_put(bind_info_p->p_osm_pool,
566 p_vw->p_resp_madw);
567 p_vw->p_resp_madw = NULL;
568 }
569
570 /* invoke the CB */
571 (*bind_info_p->send_err_callback) (bind_info_p->
572 client_context,
573 madw_p);
574 } else {
575 /* successful request send - do nothing - the response will need the
576 out mad */
577 osm_log(p_log, OSM_LOG_DEBUG,
578 "__osm_mtl_send_callback: DBG 1008: "
579 "Completed Sending Request MADW:%p.\n", madw_p);
580 }
581 }
582
583 OSM_LOG_EXIT(p_log);
584 }
585
586 /**********************************************************************
587 * BINDs a callback (rcv and send error) for a given class and method
588 * defined by the given: osm_bind_info_t
589 **********************************************************************/
590 osm_bind_handle_t
osm_vendor_bind(IN osm_vendor_t * const p_vend,IN osm_bind_info_t * const p_user_bind,IN osm_mad_pool_t * const p_mad_pool,IN osm_vend_mad_recv_callback_t mad_recv_callback,IN osm_vend_mad_send_err_callback_t send_err_callback,IN void * context)591 osm_vendor_bind(IN osm_vendor_t * const p_vend,
592 IN osm_bind_info_t * const p_user_bind,
593 IN osm_mad_pool_t * const p_mad_pool,
594 IN osm_vend_mad_recv_callback_t mad_recv_callback,
595 IN osm_vend_mad_send_err_callback_t send_err_callback,
596 IN void *context)
597 {
598 ib_net64_t port_guid;
599 osm_mtl_bind_info_t *p_bind = NULL;
600 VAPI_hca_hndl_t hca_hndl;
601 VAPI_hca_id_t hca_id;
602 IB_MGT_mad_type_t mad_type;
603 uint32_t port_num;
604 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
605 IB_MGT_ret_t mgt_ret;
606
607 OSM_LOG_ENTER(p_vend->p_log);
608
609 CL_ASSERT(p_user_bind);
610 CL_ASSERT(p_mad_pool);
611 CL_ASSERT(mad_recv_callback);
612 CL_ASSERT(send_err_callback);
613
614 /* cast back the AL handle to vendor mgt bind */
615 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
616
617 port_guid = p_user_bind->port_guid;
618
619 osm_log(p_vend->p_log, OSM_LOG_INFO,
620 "osm_vendor_bind: "
621 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
622
623 /* obtain the hca name and port num from the guid */
624 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
625 "osm_vendor_bind: "
626 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
627 port_guid);
628
629 mgt_ret =
630 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
631 &hca_id, &port_num);
632 if (mgt_ret != IB_MGT_OK) {
633 osm_log(p_vend->p_log, OSM_LOG_ERROR,
634 "osm_vendor_bind: ERR 3C14: "
635 "Unable to obtain CA and port (%d).\n");
636 goto Exit;
637 }
638
639 /* create the bind object tracking this binding */
640 p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t));
641 memset(p_bind, 0, sizeof(osm_mtl_bind_info_t));
642 if (p_bind == NULL) {
643 osm_log(p_vend->p_log, OSM_LOG_ERROR,
644 "osm_vendor_bind: ERR 3C15: "
645 "Unable to allocate internal bind object.\n");
646 goto Exit;
647 }
648
649 /* track this bind request info */
650 memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t));
651 p_bind->port_num = port_num;
652 p_bind->p_vend = p_vend;
653 p_bind->client_context = context;
654 p_bind->rcv_callback = mad_recv_callback;
655 p_bind->send_err_callback = send_err_callback;
656 p_bind->p_osm_pool = p_mad_pool;
657
658 CL_ASSERT(p_bind->port_num);
659
660 /*
661 * Get the proper CLASS
662 */
663
664 switch (p_user_bind->mad_class) {
665 case IB_MCLASS_SUBN_LID:
666 case IB_MCLASS_SUBN_DIR:
667 mad_type = IB_MGT_SMI;
668 break;
669
670 case IB_MCLASS_SUBN_ADM:
671 default:
672 mad_type = IB_MGT_GSI;
673 break;
674 }
675
676 /* we split here - based on the type of MADS GSI / SMI */
677 /* HACK: we only support one class registration per SMI/GSI !!! */
678 if (mad_type == IB_MGT_SMI) {
679 /*
680 * SMI CASE
681 */
682
683 /* we do not need to bind the handle if already available */
684 if (ib_mgt_hdl_p->smi_init == FALSE) {
685
686 /* First we have to reg and get the handle for the mad */
687 osm_log(p_vend->p_log, OSM_LOG_ERROR,
688 "osm_vendor_bind: "
689 "Binding to IB_MGT SMI of %s port %u\n", hca_id,
690 port_num);
691
692 mgt_ret =
693 IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI,
694 &(ib_mgt_hdl_p->smi_mads_hdl));
695 if (IB_MGT_OK != mgt_ret) {
696 free(p_bind);
697 p_bind = NULL;
698 osm_log(p_vend->p_log, OSM_LOG_ERROR,
699 "osm_vendor_bind: ERR 3C16: "
700 "Error obtaining IB_MGT handle to SMI.\n");
701 goto Exit;
702 }
703
704 /* bind it */
705 mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl);
706 if (IB_MGT_OK != mgt_ret) {
707 free(p_bind);
708 p_bind = NULL;
709 osm_log(p_vend->p_log, OSM_LOG_ERROR,
710 "osm_vendor_bind: ERR 3C17: "
711 "Error binding IB_MGT handle to SM.\n");
712 goto Exit;
713 }
714
715 ib_mgt_hdl_p->smi_init = TRUE;
716
717 }
718
719 /* attach to this bind info */
720 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl;
721 ib_mgt_hdl_p->smi_p_bind = p_bind;
722
723 /* now register the callback */
724 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
725 &__osm_mtl_rcv_callback,
726 p_bind,
727 &__osm_mtl_send_callback,
728 p_bind,
729 IB_MGT_RCV_CB_MASK |
730 IB_MGT_SEND_CB_MASK);
731
732 } else {
733 /*
734 * GSI CASE
735 */
736
737 if (ib_mgt_hdl_p->gsi_init == FALSE) {
738 osm_log(p_vend->p_log, OSM_LOG_ERROR,
739 "osm_vendor_bind: " "Binding to IB_MGT GSI\n");
740
741 /* First we have to reg and get the handle for the mad */
742 mgt_ret =
743 IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI,
744 &(ib_mgt_hdl_p->gsi_mads_hdl));
745 if (IB_MGT_OK != mgt_ret) {
746 free(p_bind);
747 p_bind = NULL;
748 osm_log(p_vend->p_log, OSM_LOG_ERROR,
749 "osm_vendor_bind: ERR 3C20: "
750 "Error obtaining IB_MGT handle to GSI.\n");
751 goto Exit;
752 }
753
754 /* bind it */
755 mgt_ret =
756 IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl,
757 p_user_bind->mad_class);
758 if (IB_MGT_OK != mgt_ret) {
759 free(p_bind);
760 p_bind = NULL;
761 osm_log(p_vend->p_log, OSM_LOG_ERROR,
762 "osm_vendor_bind: ERR 3C22: "
763 "Error binding IB_MGT handle to GSI.\n");
764 goto Exit;
765 }
766
767 ib_mgt_hdl_p->gsi_init = TRUE;
768
769 /* attach to this bind info */
770 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
771
772 /* now register the callback */
773 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
774 &__osm_mtl_rcv_callback,
775 p_bind,
776 &__osm_mtl_send_callback,
777 p_bind,
778 IB_MGT_RCV_CB_MASK |
779 IB_MGT_SEND_CB_MASK);
780
781 } else {
782 /* we can use the existing handle */
783 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
784 mgt_ret = IB_MGT_OK;
785 }
786
787 }
788
789 if (IB_MGT_OK != mgt_ret) {
790 free(p_bind);
791 p_bind = NULL;
792 osm_log(p_vend->p_log, OSM_LOG_ERROR,
793 "osm_vendor_bind: ERR 3C23: "
794 "Error binding IB_MGT CB (%d).\n", mgt_ret);
795 goto Exit;
796 }
797
798 /* HACK: Do we need to initialize an address vector ???? */
799
800 Exit:
801 OSM_LOG_EXIT(p_vend->p_log);
802 return ((osm_bind_handle_t) p_bind);
803 }
804
805 /**********************************************************************
806 Get a mad from the lower level.
807 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
808 **********************************************************************/
osm_vendor_get(IN osm_bind_handle_t h_bind,IN const uint32_t mad_size,IN osm_vend_wrap_t * const p_vw)809 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
810 IN const uint32_t mad_size,
811 IN osm_vend_wrap_t * const p_vw)
812 {
813 ib_mad_t *mad_p;
814 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
815 osm_vendor_t *p_vend = p_bind->p_vend;
816
817 OSM_LOG_ENTER(p_vend->p_log);
818
819 CL_ASSERT(p_vw);
820 /* HACK: We know we can not send through IB_MGT */
821 CL_ASSERT(mad_size <= MAD_BLOCK_SIZE);
822
823 /* IB_MGT assumes it is 256 - we must follow */
824 p_vw->size = MAD_BLOCK_SIZE;
825
826 /* allocate it */
827 mad_p = (ib_mad_t *) malloc(p_vw->size);
828 if (mad_p == NULL) {
829 osm_log(p_vend->p_log, OSM_LOG_ERROR,
830 "osm_vendor_get: ERR 3C24: "
831 "Error Obtaining MAD buffer.\n");
832 goto Exit;
833 }
834
835 memset(mad_p, 0, p_vw->size);
836
837 /* track locally */
838 p_vw->mad_buf_p = mad_p;
839 p_vw->h_bind = h_bind;
840 p_vw->p_resp_madw = NULL;
841
842 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
843 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
844 "osm_vendor_get: "
845 "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size);
846 }
847
848 Exit:
849 OSM_LOG_EXIT(p_vend->p_log);
850 return (mad_p);
851 }
852
853 /**********************************************************************
854 * Return a MAD by providing it's wrapper object.
855 **********************************************************************/
856 void
osm_vendor_put(IN osm_bind_handle_t h_bind,IN osm_vend_wrap_t * const p_vw)857 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
858 {
859 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
860 osm_vendor_t *p_vend = p_bind->p_vend;
861 osm_madw_t *p_madw;
862
863 OSM_LOG_ENTER(p_vend->p_log);
864
865 CL_ASSERT(p_vw);
866 CL_ASSERT(p_vw->mad_buf_p);
867
868 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
869 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
870 "osm_vendor_put: " "Retiring MAD %p.\n",
871 p_vw->mad_buf_p);
872 }
873
874 /*
875 * We moved the removal of the transaction to immediatly after
876 * it was looked up.
877 */
878
879 /* free the mad but the wrapper is part of the madw object */
880 free(p_vw->mad_buf_p);
881 p_vw->mad_buf_p = NULL;
882 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
883 p_madw->p_mad = NULL;
884
885 OSM_LOG_EXIT(p_vend->p_log);
886 }
887
888 /**********************************************************************
889 Actually Send a MAD
890
891 This is for internal use by osm_vendor_send and the transaction mgr
892 retry too.
893 **********************************************************************/
894 ib_api_status_t
osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind,IN osm_madw_t * const p_madw)895 osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
896 {
897 osm_vendor_t *const p_vend = p_bind->p_vend;
898 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
899 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
900 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
901 ib_api_status_t status;
902 IB_MGT_ret_t mgt_res;
903 IB_ud_av_t av;
904 uint64_t wrid;
905 uint32_t qpn;
906
907 OSM_LOG_ENTER(p_vend->p_log);
908
909 /*
910 * For all sends other than directed route SM MADs,
911 * acquire an address vector for the destination.
912 */
913 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
914 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr,
915 p_mad->mgmt_class ==
916 IB_MCLASS_SUBN_LID, &av);
917 } else {
918 /* is a directed route - we need to construct a permissive address */
919 memset(&av, 0, sizeof(av));
920 /* we do not need port number since it is part of the mad_hndl */
921 av.dlid = IB_LID_PERMISSIVE;
922 }
923
924 wrid = __osm_set_wrid_by_p_madw(p_madw);
925
926 /* send it */
927 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
928 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
929
930 /* SMI CASE */
931 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
932 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
933 "osm_mtl_send_mad: "
934 "av.dlid 0x%X, "
935 "av.static_rate %d, "
936 "av.path_bits %d.\n",
937 cl_ntoh16(av.dlid), av.static_rate,
938 av.src_path_bits);
939 }
940
941 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad, /* actual payload */
942 &av, /* address vector */
943 wrid, /* casting the mad wrapper pointer for err cb */
944 p_vend->timeout);
945
946 } else {
947 /* GSI CASE - Support Remote QP */
948 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
949 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
950 "osm_mtl_send_mad: "
951 "av.dlid 0x%X, av.static_rate %d, "
952 "av.path_bits %d, remote qp: 0x%06X \n",
953 av.dlid,
954 av.static_rate,
955 av.src_path_bits,
956 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
957 );
958 }
959
960 /* IBMGT have a bug sending to a QP not 1 -
961 the QPN must be in network order except when it qpn 1 ... */
962 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
963
964 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad, /* actual payload */
965 &av, /* address vector */
966 wrid, /* casting the mad wrapper pointer for err cb */
967 p_vend->timeout, qpn);
968 }
969
970 if (mgt_res != IB_MGT_OK) {
971 osm_log(p_vend->p_log, OSM_LOG_ERROR,
972 "osm_mtl_send_mad: ERR 3C26: "
973 "Error sending mad (%d).\n", mgt_res);
974 if (p_vw->p_resp_madw)
975 osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw);
976 status = IB_ERROR;
977 goto Exit;
978 }
979
980 status = IB_SUCCESS;
981
982 Exit:
983 OSM_LOG_EXIT(p_vend->p_log);
984 return (status);
985 }
986
987 /**********************************************************************
988 Send a MAD through.
989
990 What is unclear to me is the need for the setting of all the MAD Wrapper
991 fields. Seems like the OSM uses these values during it's processing...
992 **********************************************************************/
993 ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,IN osm_madw_t * const p_madw,IN boolean_t const resp_expected)994 osm_vendor_send(IN osm_bind_handle_t h_bind,
995 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
996 {
997 osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind;
998 osm_vendor_t *const p_vend = p_bind->p_vend;
999 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
1000 ib_api_status_t status;
1001
1002 OSM_LOG_ENTER(p_vend->p_log);
1003
1004 /*
1005 * If a response is expected to this MAD, then preallocate
1006 * a mad wrapper to contain the wire MAD received in the
1007 * response. Allocating a wrapper here allows for easier
1008 * failure paths than after we already received the wire mad.
1009 */
1010 if (resp_expected == TRUE) {
1011 /* we track it in the vendor wrapper */
1012 p_vw->p_resp_madw =
1013 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
1014 if (p_vw->p_resp_madw == NULL) {
1015 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1016 "osm_vendor_send: ERR 3C27: "
1017 "Unable to allocate MAD wrapper.\n");
1018 status = IB_INSUFFICIENT_RESOURCES;
1019 goto Exit;
1020 }
1021
1022 /* put some minimal info on that wrapper */
1023 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
1024
1025 /* we also want to track it in the TID based map */
1026 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t)
1027 p_bind, p_madw);
1028 if (status != IB_SUCCESS) {
1029 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1030 "osm_vendor_send: ERR 3C25: "
1031 "Error inserting request madw by TID (%d).\n",
1032 status);
1033 }
1034
1035 } else
1036 p_vw->p_resp_madw = NULL;
1037
1038 /* do the actual send */
1039 status = osm_mtl_send_mad(p_bind, p_madw);
1040
1041 Exit:
1042 OSM_LOG_EXIT(p_vend->p_log);
1043 return (status);
1044 }
1045
1046 /**********************************************************************
1047 * the idea here is to change the content of the bind such that it
1048 * will hold the local address used for sending directed route by the SMA.
1049 **********************************************************************/
osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)1050 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
1051 {
1052 osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend;
1053
1054 OSM_LOG_ENTER(p_vend->p_log);
1055
1056 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
1057 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
1058
1059 OSM_LOG_EXIT(p_vend->p_log);
1060
1061 return (IB_SUCCESS);
1062 }
1063
osm_vendor_set_sm(IN osm_bind_handle_t h_bind,IN boolean_t is_sm_val)1064 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
1065 {
1066 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
1067 osm_vendor_t *p_vend = p_bind->p_vend;
1068 VAPI_ret_t status;
1069 VAPI_hca_attr_t attr_mod;
1070 VAPI_hca_attr_mask_t attr_mask;
1071
1072 OSM_LOG_ENTER(p_vend->p_log);
1073
1074 memset(&attr_mod, 0, sizeof(attr_mod));
1075 memset(&attr_mask, 0, sizeof(attr_mask));
1076
1077 attr_mod.is_sm = is_sm_val;
1078 attr_mask = HCA_ATTR_IS_SM;
1079
1080 status =
1081 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
1082 &attr_mask);
1083 if (status != VAPI_OK) {
1084 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1085 "osm_vendor_set_sm: ERR 3C28: "
1086 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
1087 is_sm_val, status);
1088 }
1089
1090 OSM_LOG_EXIT(p_vend->p_log);
1091 }
1092
osm_vendor_set_debug(IN osm_vendor_t * const p_vend,IN int32_t level)1093 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
1094 {
1095
1096 }
1097
1098 #endif /* OSM_VENDOR_INTF_TEST */
1099