1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 #include <sys/types.h>
27 #include <sys/socket.h>
28 #include <sys/sockio.h>
29 #include <sys/param.h>
30 #include <sys/stat.h>
31 #include <netinet/in.h>
32 #include <arpa/inet.h>
33 #include <net/if.h>
34 #include <unistd.h>
35 #include <stdlib.h>
36 #include <string.h>
37 #include <errno.h>
38 #include <strings.h>
39 #include <fcntl.h>
40 #include <libdladm.h>
41 #include <libdlib.h>
42 #include <libdllink.h>
43 #include <sys/ib/ibnex/ibnex_devctl.h>
44
45 #include "dapl.h"
46 #include "dapl_adapter_util.h"
47 #include "dapl_tavor_ibtf_impl.h"
48 #include "dapl_hca_util.h"
49 #include "dapl_name_service.h"
50 #define MAX_HCAS 64
51 #define PROP_HCA_GUID "hca-guid"
52 #define PROP_PORT_NUM "port-number"
53 #define PROP_PORT_PKEY "port-pkey"
54
55 #define DEVDAPLT "/dev/daplt"
56
57 /* function prototypes */
58 static DAT_RETURN dapli_process_tavor_node(char *dev_path, int *hca_idx,
59 int try_blueflame);
60 static DAT_RETURN dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr,
61 int hca_idx);
62
63 #if defined(IBHOSTS_NAMING)
64 #include <stdio.h>
65 static int dapli_process_fake_ibds(DAPL_HCA **hca_list, int hca_idx);
66 #endif /* IBHOSTS_NAMING */
67
68 static DAPL_OS_LOCK g_tavor_state_lock;
69 static struct dapls_ib_hca_state g_tavor_state[MAX_HCAS];
70 DAPL_OS_LOCK g_tavor_uar_lock;
71
72 DAT_RETURN
dapli_init_hca(IN DAPL_HCA * hca_ptr)73 dapli_init_hca(
74 IN DAPL_HCA *hca_ptr)
75 {
76 DAT_RETURN dat_status = DAT_SUCCESS;
77 int hca_idx = 0;
78 int check_for_bf = 0;
79 datalink_class_t class;
80 datalink_id_t linkid;
81 dladm_ib_attr_t ib_attr;
82 ibnex_ctl_query_hca_t query_hca;
83 int ibnex_fd = -1;
84 dladm_handle_t dlh;
85 char hca_device_path[MAXPATHLEN];
86
87 if (dladm_open(&dlh) != DLADM_STATUS_OK) {
88 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
89 "init_hca: dladm_open failed\n");
90 return (DAT_INTERNAL_ERROR);
91 }
92
93 if ((ibnex_fd = open(IBNEX_DEVCTL_DEV, O_RDONLY)) < 0) {
94 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
95 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
96 "init_hca: could not open ib nexus (%s)\n",
97 strerror(errno));
98 goto bail;
99 }
100
101 if ((dladm_name2info(dlh, hca_ptr->name, &linkid, NULL, &class,
102 NULL) != DLADM_STATUS_OK) ||
103 (class != DATALINK_CLASS_PART) ||
104 (dladm_part_info(dlh, linkid, &ib_attr,
105 DLADM_OPT_ACTIVE) != DLADM_STATUS_OK)) {
106 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
107 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
108 "init_hca: %s not found - couldn't get partition info\n",
109 hca_ptr->name);
110 goto bail;
111 }
112
113 bzero(&query_hca, sizeof (query_hca));
114 query_hca.hca_guid = ib_attr.dia_hca_guid;
115 query_hca.hca_device_path = hca_device_path;
116 query_hca.hca_device_path_alloc_sz = sizeof (hca_device_path);
117 if (ioctl(ibnex_fd, IBNEX_CTL_QUERY_HCA, &query_hca) == -1) {
118 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
119 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
120 "init_hca: %s not found; query_hca failed\n",
121 hca_ptr->name);
122 goto bail;
123 }
124
125 if (strcmp(query_hca.hca_info.hca_driver_name, "tavor") == 0)
126 dapls_init_funcs_tavor(hca_ptr);
127 else if (strcmp(query_hca.hca_info.hca_driver_name, "arbel") == 0)
128 dapls_init_funcs_arbel(hca_ptr);
129 else if (strcmp(query_hca.hca_info.hca_driver_name, "hermon") == 0) {
130 dapls_init_funcs_hermon(hca_ptr);
131 check_for_bf = 1;
132 } else {
133 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
134 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
135 "init_hca: %s not found\n", hca_ptr->name);
136 goto bail;
137 }
138
139 dat_status = dapli_process_tavor_node(hca_device_path, &hca_idx,
140 check_for_bf);
141 if (dat_status != DAT_SUCCESS) {
142 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
143 "init_hcas: %s process_tavor_node failed(0x%x)\n",
144 hca_ptr->name, dat_status);
145 goto bail;
146 }
147
148 #if defined(IBHOSTS_NAMING)
149 if (dapli_process_fake_ibds(hca_ptr, hca_idx) == 0) {
150 /* no entries were found */
151 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
152 }
153 #else
154 dat_status = dapli_process_ia(&ib_attr, hca_ptr, hca_idx);
155 #endif
156 if (dat_status != DAT_SUCCESS) {
157 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
158 "init_hcas: %s process_ia failed(0x%x)\n",
159 hca_ptr->name, dat_status);
160 goto bail;
161 }
162
163 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
164 "init_hcas: done %s\n", hca_ptr->name);
165
166 bail:
167 if (ibnex_fd != -1)
168 (void) close(ibnex_fd);
169 dladm_close(dlh);
170 return (dat_status);
171 }
172
173 static DAT_RETURN
dapli_process_tavor_node(char * dev_path,int * hca_idx,int try_blueflame)174 dapli_process_tavor_node(char *dev_path, int *hca_idx, int try_blueflame)
175 {
176 char path_buf[MAXPATHLEN];
177 int i, idx, fd;
178 #ifndef _LP64
179 int tmpfd;
180 #endif
181 size_t pagesize;
182 void *mapaddr;
183 pid_t cur_pid;
184 off64_t uarpg_offset;
185
186 dapl_os_lock(&g_tavor_state_lock);
187
188 for (idx = 0; idx < MAX_HCAS; idx++) {
189 /*
190 * page size == 0 means this entry is not occupied
191 */
192 if (g_tavor_state[idx].uarpg_size == 0) {
193 break;
194 }
195 }
196 if (idx == MAX_HCAS) {
197 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
198 "process_tavor: all hcas are being used!\n");
199 dapl_os_unlock(&g_tavor_state_lock);
200 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
201 }
202
203 for (i = 0; i < idx; i++) {
204 if (strcmp(dev_path, g_tavor_state[i].hca_path) == 0) {
205 /* no need for a refcnt */
206 idx = i;
207 goto done;
208 }
209 }
210
211 /* Add 16 to accomodate the prefix "/devices" and suffix ":devctl" */
212 if (strlen("/devices") + strlen(dev_path) + strlen(":devctl") + 1 >
213 MAXPATHLEN) {
214 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
215 "process_tavor: devfs path %s is too long\n",
216 dev_path);
217 dapl_os_unlock(&g_tavor_state_lock);
218 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
219 }
220 (void) dapl_os_strcpy(path_buf, "/devices");
221 (void) dapl_os_strcat(path_buf, dev_path);
222 (void) dapl_os_strcat(path_buf, ":devctl");
223 (void) dapl_os_strcpy(g_tavor_state[idx].hca_path, dev_path);
224
225 pagesize = (size_t)sysconf(_SC_PAGESIZE);
226 if (pagesize == 0) {
227 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
228 "process_tavor: page_size == 0\n");
229 dapl_os_unlock(&g_tavor_state_lock);
230 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
231 }
232 cur_pid = getpid();
233
234 fd = open(path_buf, O_RDWR);
235 if (fd < 0) {
236 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
237 "process_tavor: cannot open %s: %s\n",
238 path_buf, strerror(errno));
239 dapl_os_unlock(&g_tavor_state_lock);
240 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
241 }
242 #ifndef _LP64
243 /*
244 * libc can't handle fd's greater than 255, in order to
245 * ensure that these values remain available make fd > 255.
246 * Note: not needed for LP64
247 */
248 tmpfd = fcntl(fd, F_DUPFD, 256);
249 if (tmpfd < 0) {
250 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
251 "process_tavor: cannot F_DUPFD: %s\n", strerror(errno));
252 } else {
253 (void) close(fd);
254 fd = tmpfd;
255 }
256 #endif /* _LP64 */
257
258 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
259 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
260 "process_tavor: cannot F_SETFD: %s\n", strerror(errno));
261 (void) close(fd);
262 dapl_os_unlock(&g_tavor_state_lock);
263 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
264 }
265
266 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
267 MLNX_UMAP_UARPG_RSRC) * pagesize;
268
269 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE,
270 MAP_SHARED, fd, uarpg_offset);
271 if (mapaddr == MAP_FAILED) {
272 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
273 "process_tavor: mmap failed %s\n", strerror(errno));
274 (void) close(fd);
275 dapl_os_unlock(&g_tavor_state_lock);
276 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
277 }
278
279 g_tavor_state[idx].hca_fd = fd;
280 g_tavor_state[idx].uarpg_baseaddr = mapaddr;
281 g_tavor_state[idx].uarpg_size = pagesize;
282
283 if (try_blueflame == 0)
284 goto done;
285
286 /* Try to do the Hermon Blueflame page mapping */
287 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
288 MLNX_UMAP_BLUEFLAMEPG_RSRC) * pagesize;
289
290 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE,
291 MAP_SHARED, fd, uarpg_offset);
292 if (mapaddr == MAP_FAILED) {
293 /* This is not considered to be fatal. Charge on! */
294 dapl_dbg_log(DAPL_DBG_TYPE_WARN,
295 "process_tavor: mmap of blueflame page failed %s\n",
296 strerror(errno));
297 } else {
298 g_tavor_state[idx].bf_pg_baseaddr = mapaddr;
299 g_tavor_state[idx].bf_toggle = 0;
300 }
301 done:
302 dapl_os_unlock(&g_tavor_state_lock);
303
304 *hca_idx = idx;
305
306 return (DAT_SUCCESS);
307 }
308
309 static DAT_RETURN
dapli_process_ia(dladm_ib_attr_t * ib_attr,DAPL_HCA * hca_ptr,int hca_idx)310 dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr, int hca_idx)
311 {
312 struct lifreq lifreq;
313 int sfd, retval, af;
314 char addr_buf[64];
315
316 if (ib_attr->dia_hca_guid == 0 || ib_attr->dia_portnum == 0 ||
317 ib_attr->dia_pkey == 0) {
318 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
319 "process_ia: invalid properties: guid 0x%016llx, "
320 "port %d, pkey 0x%08x\n", ib_attr->dia_hca_guid,
321 ib_attr->dia_portnum, (uint_t)ib_attr->dia_pkey);
322 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
323 }
324
325 /*
326 * if an interface has both v4 and v6 addresses plumbed,
327 * we'll take the v4 address.
328 */
329 af = AF_INET;
330 again:
331 sfd = socket(af, SOCK_DGRAM, 0);
332 if (sfd < 0) {
333 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
334 "process_ia: socket failed: %s\n", strerror(errno));
335 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
336 }
337
338 /* check if name will fit in lifr_name */
339 if (dapl_os_strlen(hca_ptr->name) >= LIFNAMSIZ) {
340 (void) close(sfd);
341 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
342 "process_ia: if name overflow %s\n",
343 hca_ptr->name);
344 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
345 }
346
347 (void) dapl_os_strcpy(lifreq.lifr_name, hca_ptr->name);
348 retval = ioctl(sfd, SIOCGLIFADDR, (caddr_t)&lifreq);
349 if (retval < 0) {
350 (void) close(sfd);
351 if (af == AF_INET6) {
352 /*
353 * the interface is not plumbed.
354 */
355 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
356 "process_ia: %s: ip address not found\n",
357 lifreq.lifr_name);
358 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
359 } else {
360 /*
361 * we've failed to find a v4 address. now
362 * let's try v6.
363 */
364 af = AF_INET6;
365 goto again;
366 }
367 }
368 (void) close(sfd);
369
370 hca_ptr->tavor_idx = hca_idx;
371 hca_ptr->node_GUID = ib_attr->dia_hca_guid;
372 hca_ptr->port_num = ib_attr->dia_portnum;
373 hca_ptr->partition_key = ib_attr->dia_pkey;
374 (void) dapl_os_memcpy((void *)&hca_ptr->hca_address,
375 (void *)&lifreq.lifr_addr, sizeof (hca_ptr->hca_address));
376 hca_ptr->max_inline_send = dapls_tavor_max_inline();
377
378 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
379 "process_ia: interface %s, hca guid 0x%016llx, port %d, "
380 "pkey 0x%08x, ip addr %s\n", lifreq.lifr_name, hca_ptr->node_GUID,
381 hca_ptr->port_num, hca_ptr->partition_key, dapls_inet_ntop(
382 (struct sockaddr *)&hca_ptr->hca_address, addr_buf, 64));
383 return (DAT_SUCCESS);
384 }
385
386 void
dapls_ib_state_init(void)387 dapls_ib_state_init(void)
388 {
389 int i;
390
391 (void) dapl_os_lock_init(&g_tavor_state_lock);
392 (void) dapl_os_lock_init(&g_tavor_uar_lock);
393 (void) dapl_os_lock_init(&dapls_ib_dbp_lock);
394
395 for (i = 0; i < MAX_HCAS; i++) {
396 g_tavor_state[i].hca_fd = 0;
397 g_tavor_state[i].uarpg_baseaddr = NULL;
398 g_tavor_state[i].uarpg_size = 0;
399 g_tavor_state[i].bf_pg_baseaddr = NULL;
400 }
401 }
402
403 void
dapls_ib_state_fini(void)404 dapls_ib_state_fini(void)
405 {
406 int i, count = 0;
407
408 /*
409 * Uinitialize the per hca instance state
410 */
411 dapl_os_lock(&g_tavor_state_lock);
412 for (i = 0; i < MAX_HCAS; i++) {
413 if (g_tavor_state[i].uarpg_size == 0) {
414 dapl_os_assert(g_tavor_state[i].uarpg_baseaddr ==
415 NULL);
416 continue;
417 }
418 if (munmap(g_tavor_state[i].uarpg_baseaddr,
419 g_tavor_state[i].uarpg_size) < 0) {
420 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
421 "ib_state_fini: "
422 "munmap(0x%p, 0x%llx) failed(%d)\n",
423 g_tavor_state[i].uarpg_baseaddr,
424 g_tavor_state[i].uarpg_size, errno);
425 }
426 if ((g_tavor_state[i].bf_pg_baseaddr != NULL) &&
427 (munmap(g_tavor_state[i].bf_pg_baseaddr,
428 g_tavor_state[i].uarpg_size) < 0)) {
429 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
430 "ib_state_fini: "
431 "munmap(0x%p, 0x%llx) of blueflame failed(%d)\n",
432 g_tavor_state[i].bf_pg_baseaddr,
433 g_tavor_state[i].uarpg_size, errno);
434 }
435
436 (void) close(g_tavor_state[i].hca_fd);
437 count++;
438 }
439 dapl_os_unlock(&g_tavor_state_lock);
440
441 dapl_os_lock_destroy(&g_tavor_uar_lock);
442 dapl_os_lock_destroy(&g_tavor_state_lock);
443 dapl_os_lock_destroy(&dapls_ib_dbp_lock);
444
445 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
446 "ib_state_fini: cleaned %d hcas\n", count);
447 }
448
449 /*
450 * dapls_ib_open_hca
451 *
452 * Open HCA
453 *
454 * Input:
455 * *hca_ptr pointer to hca device
456 * *ib_hca_handle_p pointer to provide HCA handle
457 *
458 * Output:
459 * none
460 *
461 * Return:
462 * DAT_SUCCESS
463 * DAT_INSUFFICIENT_RESOURCES
464 *
465 */
466 DAT_RETURN
dapls_ib_open_hca(IN DAPL_HCA * hca_ptr,OUT ib_hca_handle_t * ib_hca_handle_p)467 dapls_ib_open_hca(
468 IN DAPL_HCA *hca_ptr,
469 OUT ib_hca_handle_t *ib_hca_handle_p)
470 {
471 dapl_ia_create_t args;
472 DAT_RETURN dat_status;
473 struct dapls_ib_hca_handle *hca_p;
474 int fd;
475 #ifndef _LP64
476 int tmpfd;
477 #endif
478 int retval;
479 struct sockaddr *s;
480 struct sockaddr_in6 *v6addr;
481 struct sockaddr_in *v4addr;
482 dapl_ia_addr_t *sap;
483
484 dat_status = dapli_init_hca(hca_ptr);
485 if (dat_status != DAT_SUCCESS) {
486 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
487 "dapls_ib_open_hca: init_hca failed %d\n", dat_status);
488 return (dat_status);
489 }
490
491 fd = open(DEVDAPLT, O_RDONLY);
492 if (fd < 0) {
493 return (DAT_INSUFFICIENT_RESOURCES);
494 }
495
496 #ifndef _LP64
497 /*
498 * libc can't handle fd's greater than 255, in order to
499 * ensure that these values remain available make fd > 255.
500 * Note: not needed for LP64
501 */
502 tmpfd = fcntl(fd, F_DUPFD, 256);
503 if (tmpfd < 0) {
504 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
505 "dapls_ib_open_hca: cannot F_DUPFD: %s\n",
506 strerror(errno));
507 } else {
508 (void) close(fd);
509 fd = tmpfd;
510 }
511 #endif /* _LP64 */
512
513 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
514 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
515 "dapls_ib_open_hca: cannot F_SETFD: %s\n", strerror(errno));
516 (void) close(fd);
517 return (DAT_INTERNAL_ERROR);
518 }
519
520 hca_p = (struct dapls_ib_hca_handle *)dapl_os_alloc(
521 sizeof (struct dapls_ib_hca_handle));
522 if (hca_p == NULL) {
523 (void) close(fd);
524 return (DAT_INSUFFICIENT_RESOURCES);
525 }
526
527 args.ia_guid = hca_ptr->node_GUID;
528 args.ia_port = hca_ptr->port_num;
529 args.ia_pkey = hca_ptr->partition_key;
530 args.ia_version = DAPL_IF_VERSION;
531 (void) dapl_os_memzero((void *)args.ia_sadata, DAPL_ATS_NBYTES);
532
533 /* pass down local ip address to be stored in SA */
534 s = (struct sockaddr *)&hca_ptr->hca_address;
535 /* LINTED: E_BAD_PTR_CAST_ALIGN */
536 sap = (dapl_ia_addr_t *)args.ia_sadata;
537 switch (s->sa_family) {
538 case AF_INET:
539 /* LINTED: E_BAD_PTR_CAST_ALIGN */
540 v4addr = (struct sockaddr_in *)s;
541 sap->iad_v4 = v4addr->sin_addr;
542 break;
543 case AF_INET6:
544 /* LINTED: E_BAD_PTR_CAST_ALIGN */
545 v6addr = (struct sockaddr_in6 *)s;
546 sap->iad_v6 = v6addr->sin6_addr;
547 break;
548 default:
549 break; /* fall through */
550 }
551
552 retval = ioctl(fd, DAPL_IA_CREATE, &args);
553 if (retval != 0) {
554 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
555 "open_hca: ia_create failed, fd %d, "
556 "guid 0x%016llx, port %d, pkey 0x%x, version %d\n",
557 fd, args.ia_guid, args.ia_port, args.ia_pkey,
558 args.ia_version);
559
560 dapl_os_free(hca_p, sizeof (*hca_p));
561 (void) close(fd);
562 return (dapls_convert_error(errno, retval));
563 }
564
565 hca_p->ia_fd = fd;
566 hca_p->ia_rnum = args.ia_resnum;
567 hca_p->hca_fd = g_tavor_state[hca_ptr->tavor_idx].hca_fd;
568 hca_p->ia_uar = g_tavor_state[hca_ptr->tavor_idx].uarpg_baseaddr;
569 hca_p->ia_bf = g_tavor_state[hca_ptr->tavor_idx].bf_pg_baseaddr;
570 hca_p->ia_bf_toggle = &g_tavor_state[hca_ptr->tavor_idx].bf_toggle;
571 *ib_hca_handle_p = hca_p;
572 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
573 "open_hca: ia_created, hca_p 0x%p, fd %d, "
574 "rnum %d, guid 0x%016llx, port %d, pkey 0x%x\n",
575 hca_p, hca_p->ia_fd, hca_p->ia_rnum, hca_ptr->node_GUID,
576 hca_ptr->port_num, hca_ptr->partition_key);
577
578 return (DAT_SUCCESS);
579 }
580
581 /*
582 * dapls_ib_close_hca
583 *
584 * Open HCA
585 *
586 * Input:
587 * ib_hca_handle provide HCA handle
588 *
589 * Output:
590 * none
591 *
592 * Return:
593 * DAT_SUCCESS
594 * DAT_INSUFFICIENT_RESOURCES
595 *
596 */
597 DAT_RETURN
dapls_ib_close_hca(IN ib_hca_handle_t ib_hca_handle)598 dapls_ib_close_hca(
599 IN ib_hca_handle_t ib_hca_handle)
600 {
601 if (ib_hca_handle == NULL) {
602 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
603 "close_hca: ib_hca_handle == NULL\n");
604 return (DAT_SUCCESS);
605 }
606 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
607 "close_hca: closing hca 0x%p, fd %d, rnum %d\n",
608 ib_hca_handle, ib_hca_handle->ia_fd, ib_hca_handle->ia_rnum);
609
610 (void) close(ib_hca_handle->ia_fd);
611 dapl_os_free((void *)ib_hca_handle,
612 sizeof (struct dapls_ib_hca_handle));
613 return (DAT_SUCCESS);
614 }
615
616 #if defined(IBHOSTS_NAMING)
617 #define LINE_LEN 256
618 static int
dapli_process_fake_ibds(DAPL_HCA * hca_ptr,int hca_idx)619 dapli_process_fake_ibds(DAPL_HCA *hca_ptr, int hca_idx)
620 {
621 char line_buf[LINE_LEN];
622 char host_buf[LINE_LEN];
623 char localhost[LINE_LEN];
624 ib_guid_t prefix;
625 ib_guid_t guid;
626 FILE *fp;
627 int count = 0;
628 DAPL_HCA *hca_ptr;
629
630 fp = fopen("/etc/dapl/ibhosts", "r");
631 if (fp == NULL) {
632 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
633 "fake_ibds: ibhosts not found!\n");
634 return (0);
635 }
636 if (gethostname(localhost, LINE_LEN) != 0) {
637 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
638 "fake_ibds: hostname not found!\n");
639 return (0);
640 }
641 while (!feof(fp)) {
642 (void) fgets(line_buf, LINE_LEN, fp);
643 sscanf(line_buf, "%s %llx %llx", host_buf, &prefix, &guid);
644 (void) sprintf(line_buf, "%s-ib%d", localhost, count + 1);
645 if (strncmp(line_buf, host_buf, strlen(line_buf)) == 0) {
646 guid &= 0xfffffffffffffff0;
647 hca_ptr->tavor_idx = hca_idx;
648 hca_ptr->node_GUID = guid;
649 hca_ptr->port_num = count + 1;
650 hca_ptr->partition_key = 0x0000ffff;
651 count++;
652 }
653 if (count >= 2) break;
654 }
655 (void) fclose(fp);
656 return (count);
657 }
658
659 #endif /* IBHOSTS_NAMING */
660