xref: /illumos-gate/usr/src/lib/udapl/udapl_tavor/tavor/dapl_tavor_hca.c (revision e3b270024f3e952233b61d3dcb4129ba75bf2720)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/socket.h>
28 #include <sys/sockio.h>
29 #include <sys/param.h>
30 #include <sys/stat.h>
31 #include <netinet/in.h>
32 #include <arpa/inet.h>
33 #include <net/if.h>
34 #include <unistd.h>
35 #include <stdlib.h>
36 #include <string.h>
37 #include <errno.h>
38 #include <strings.h>
39 #include <fcntl.h>
40 #include <libdladm.h>
41 #include <libdlib.h>
42 #include <libdllink.h>
43 #include <sys/ib/ibnex/ibnex_devctl.h>
44 
45 #include "dapl.h"
46 #include "dapl_adapter_util.h"
47 #include "dapl_tavor_ibtf_impl.h"
48 #include "dapl_hca_util.h"
49 #include "dapl_name_service.h"
50 #define	MAX_HCAS		64
51 #define	PROP_HCA_GUID		"hca-guid"
52 #define	PROP_PORT_NUM		"port-number"
53 #define	PROP_PORT_PKEY		"port-pkey"
54 
55 #define	DEVDAPLT		"/dev/daplt"
56 
57 /* function prototypes */
58 static DAT_RETURN dapli_process_tavor_node(char *dev_path, int *hca_idx,
59     int try_blueflame);
60 static DAT_RETURN dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr,
61     int hca_idx);
62 
63 #if defined(IBHOSTS_NAMING)
64 #include <stdio.h>
65 static int dapli_process_fake_ibds(DAPL_HCA **hca_list, int hca_idx);
66 #endif /* IBHOSTS_NAMING */
67 
68 static DAPL_OS_LOCK g_tavor_state_lock;
69 static struct dapls_ib_hca_state g_tavor_state[MAX_HCAS];
70 DAPL_OS_LOCK g_tavor_uar_lock;
71 DAPL_OS_LOCK dapls_ib_dbp_lock;
72 
73 DAT_RETURN
dapli_init_hca(IN DAPL_HCA * hca_ptr)74 dapli_init_hca(
75 	IN   DAPL_HCA			*hca_ptr)
76 {
77 	DAT_RETURN		dat_status = DAT_SUCCESS;
78 	int			hca_idx = 0;
79 	int			check_for_bf = 0;
80 	datalink_class_t	class;
81 	datalink_id_t		linkid;
82 	dladm_ib_attr_t		ib_attr;
83 	ibnex_ctl_query_hca_t	query_hca;
84 	int			ibnex_fd = -1;
85 	dladm_handle_t		dlh;
86 	char			hca_device_path[MAXPATHLEN];
87 
88 	if (dladm_open(&dlh) != DLADM_STATUS_OK) {
89 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
90 		    "init_hca: dladm_open failed\n");
91 		return (DAT_INTERNAL_ERROR);
92 	}
93 
94 	if ((ibnex_fd = open(IBNEX_DEVCTL_DEV, O_RDONLY)) < 0) {
95 		dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
96 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
97 		    "init_hca: could not open ib nexus (%s)\n",
98 		    strerror(errno));
99 		goto bail;
100 	}
101 
102 	if ((dladm_name2info(dlh, hca_ptr->name, &linkid, NULL, &class,
103 	    NULL) != DLADM_STATUS_OK) ||
104 	    (class != DATALINK_CLASS_PART) ||
105 	    (dladm_part_info(dlh, linkid, &ib_attr,
106 	    DLADM_OPT_ACTIVE) != DLADM_STATUS_OK)) {
107 		dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
108 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
109 		    "init_hca: %s not found - couldn't get partition info\n",
110 		    hca_ptr->name);
111 		goto bail;
112 	}
113 
114 	bzero(&query_hca, sizeof (query_hca));
115 	query_hca.hca_guid = ib_attr.dia_hca_guid;
116 	query_hca.hca_device_path = hca_device_path;
117 	query_hca.hca_device_path_alloc_sz = sizeof (hca_device_path);
118 	if (ioctl(ibnex_fd, IBNEX_CTL_QUERY_HCA, &query_hca) == -1) {
119 		dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
120 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
121 		    "init_hca: %s not found; query_hca failed\n",
122 		    hca_ptr->name);
123 		goto bail;
124 	}
125 
126 	if (strcmp(query_hca.hca_info.hca_driver_name, "tavor") == 0)
127 		dapls_init_funcs_tavor(hca_ptr);
128 	else if (strcmp(query_hca.hca_info.hca_driver_name, "arbel") == 0)
129 		dapls_init_funcs_arbel(hca_ptr);
130 	else if (strcmp(query_hca.hca_info.hca_driver_name, "hermon") == 0) {
131 		dapls_init_funcs_hermon(hca_ptr);
132 		check_for_bf = 1;
133 	} else {
134 		dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
135 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
136 		    "init_hca: %s not found\n", hca_ptr->name);
137 		goto bail;
138 	}
139 
140 	dat_status = dapli_process_tavor_node(hca_device_path, &hca_idx,
141 	    check_for_bf);
142 	if (dat_status != DAT_SUCCESS) {
143 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
144 		    "init_hcas: %s process_tavor_node failed(0x%x)\n",
145 		    hca_ptr->name, dat_status);
146 		goto bail;
147 	}
148 
149 #if defined(IBHOSTS_NAMING)
150 	if (dapli_process_fake_ibds(hca_ptr, hca_idx) == 0) {
151 		/* no entries were found */
152 		dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
153 	}
154 #else
155 	dat_status = dapli_process_ia(&ib_attr, hca_ptr, hca_idx);
156 #endif
157 	if (dat_status != DAT_SUCCESS) {
158 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
159 		    "init_hcas: %s process_ia failed(0x%x)\n",
160 		    hca_ptr->name, dat_status);
161 		goto bail;
162 	}
163 
164 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
165 	    "init_hcas: done %s\n", hca_ptr->name);
166 
167 bail:
168 	if (ibnex_fd != -1)
169 		(void) close(ibnex_fd);
170 	dladm_close(dlh);
171 	return (dat_status);
172 }
173 
174 static DAT_RETURN
dapli_process_tavor_node(char * dev_path,int * hca_idx,int try_blueflame)175 dapli_process_tavor_node(char *dev_path, int *hca_idx, int try_blueflame)
176 {
177 	char		path_buf[MAXPATHLEN];
178 	int		i, idx, fd;
179 #ifndef _LP64
180 	int		tmpfd;
181 #endif
182 	size_t		pagesize;
183 	void		*mapaddr;
184 	pid_t		cur_pid;
185 	off64_t		uarpg_offset;
186 
187 	dapl_os_lock(&g_tavor_state_lock);
188 
189 	for (idx = 0; idx < MAX_HCAS; idx++) {
190 		/*
191 		 * page size == 0 means this entry is not occupied
192 		 */
193 		if (g_tavor_state[idx].uarpg_size == 0) {
194 			break;
195 		}
196 	}
197 	if (idx == MAX_HCAS) {
198 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
199 		    "process_tavor: all hcas are being used!\n");
200 		dapl_os_unlock(&g_tavor_state_lock);
201 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
202 	}
203 
204 	for (i = 0; i < idx; i++) {
205 		if (strcmp(dev_path, g_tavor_state[i].hca_path) == 0) {
206 			/* no need for a refcnt */
207 			idx = i;
208 			goto done;
209 		}
210 	}
211 
212 	/* Add 16 to accomodate the prefix "/devices" and suffix ":devctl" */
213 	if (strlen("/devices") + strlen(dev_path) + strlen(":devctl") + 1 >
214 	    MAXPATHLEN) {
215 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
216 		    "process_tavor: devfs path %s is too long\n",
217 		    dev_path);
218 		dapl_os_unlock(&g_tavor_state_lock);
219 		return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
220 	}
221 	(void) dapl_os_strcpy(path_buf, "/devices");
222 	(void) dapl_os_strcat(path_buf, dev_path);
223 	(void) dapl_os_strcat(path_buf, ":devctl");
224 	(void) dapl_os_strcpy(g_tavor_state[idx].hca_path, dev_path);
225 
226 	pagesize = (size_t)sysconf(_SC_PAGESIZE);
227 	if (pagesize == 0) {
228 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
229 		    "process_tavor: page_size == 0\n");
230 		dapl_os_unlock(&g_tavor_state_lock);
231 		return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
232 	}
233 	cur_pid = getpid();
234 
235 	fd = open(path_buf, O_RDWR);
236 	if (fd < 0) {
237 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
238 		    "process_tavor: cannot open %s: %s\n",
239 		    path_buf, strerror(errno));
240 		dapl_os_unlock(&g_tavor_state_lock);
241 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
242 	}
243 #ifndef _LP64
244 	/*
245 	 * libc can't handle fd's greater than 255,  in order to
246 	 * ensure that these values remain available make fd > 255.
247 	 * Note: not needed for LP64
248 	 */
249 	tmpfd = fcntl(fd, F_DUPFD, 256);
250 	if (tmpfd < 0) {
251 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
252 		"process_tavor: cannot F_DUPFD: %s\n", strerror(errno));
253 	} else {
254 		(void) close(fd);
255 		fd = tmpfd;
256 	}
257 #endif	/* _LP64 */
258 
259 	if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
260 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
261 		    "process_tavor: cannot F_SETFD: %s\n", strerror(errno));
262 		(void) close(fd);
263 		dapl_os_unlock(&g_tavor_state_lock);
264 		return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
265 	}
266 
267 	uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
268 	    MLNX_UMAP_UARPG_RSRC) * pagesize;
269 
270 	mapaddr = mmap64((void  *)0, pagesize, PROT_READ | PROT_WRITE,
271 	    MAP_SHARED, fd, uarpg_offset);
272 	if (mapaddr == MAP_FAILED) {
273 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
274 		    "process_tavor: mmap failed %s\n", strerror(errno));
275 		(void) close(fd);
276 		dapl_os_unlock(&g_tavor_state_lock);
277 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
278 	}
279 
280 	g_tavor_state[idx].hca_fd = fd;
281 	g_tavor_state[idx].uarpg_baseaddr = mapaddr;
282 	g_tavor_state[idx].uarpg_size = pagesize;
283 
284 	if (try_blueflame == 0)
285 		goto done;
286 
287 	/* Try to do the Hermon Blueflame page mapping */
288 	uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
289 	    MLNX_UMAP_BLUEFLAMEPG_RSRC) * pagesize;
290 
291 	mapaddr = mmap64((void  *)0, pagesize, PROT_READ | PROT_WRITE,
292 	    MAP_SHARED, fd, uarpg_offset);
293 	if (mapaddr == MAP_FAILED) {
294 		/* This is not considered to be fatal.  Charge on! */
295 		dapl_dbg_log(DAPL_DBG_TYPE_WARN,
296 		    "process_tavor: mmap of blueflame page failed %s\n",
297 		    strerror(errno));
298 	} else {
299 		g_tavor_state[idx].bf_pg_baseaddr = mapaddr;
300 		g_tavor_state[idx].bf_toggle = 0;
301 	}
302 done:
303 	dapl_os_unlock(&g_tavor_state_lock);
304 
305 	*hca_idx = idx;
306 
307 	return (DAT_SUCCESS);
308 }
309 
310 static DAT_RETURN
dapli_process_ia(dladm_ib_attr_t * ib_attr,DAPL_HCA * hca_ptr,int hca_idx)311 dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr, int hca_idx)
312 {
313 	struct lifreq	lifreq;
314 	int		sfd, retval, af;
315 	char		addr_buf[64];
316 
317 	if (ib_attr->dia_hca_guid == 0 || ib_attr->dia_portnum == 0 ||
318 	    ib_attr->dia_pkey == 0) {
319 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
320 		    "process_ia: invalid properties: guid 0x%016llx, "
321 		    "port %d, pkey 0x%08x\n", ib_attr->dia_hca_guid,
322 		    ib_attr->dia_portnum, (uint_t)ib_attr->dia_pkey);
323 		return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
324 	}
325 
326 	/*
327 	 * if an interface has both v4 and v6 addresses plumbed,
328 	 * we'll take the v4 address.
329 	 */
330 	af = AF_INET;
331 again:
332 	sfd = socket(af, SOCK_DGRAM, 0);
333 	if (sfd < 0) {
334 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
335 		    "process_ia: socket failed: %s\n", strerror(errno));
336 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
337 	}
338 
339 	/* check if name will fit in lifr_name */
340 	if (dapl_os_strlen(hca_ptr->name) >= LIFNAMSIZ) {
341 		(void) close(sfd);
342 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
343 		    "process_ia: if name overflow %s\n",
344 		    hca_ptr->name);
345 		return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
346 	}
347 
348 	(void) dapl_os_strcpy(lifreq.lifr_name, hca_ptr->name);
349 	retval = ioctl(sfd, SIOCGLIFADDR, (caddr_t)&lifreq);
350 	if (retval < 0) {
351 		(void) close(sfd);
352 		if (af == AF_INET6) {
353 			/*
354 			 * the interface is not plumbed.
355 			 */
356 			dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
357 			    "process_ia: %s: ip address not found\n",
358 			    lifreq.lifr_name);
359 			return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
360 		} else {
361 			/*
362 			 * we've failed to find a v4 address. now
363 			 * let's try v6.
364 			 */
365 			af = AF_INET6;
366 			goto again;
367 		}
368 	}
369 	(void) close(sfd);
370 
371 	hca_ptr->tavor_idx = hca_idx;
372 	hca_ptr->node_GUID = ib_attr->dia_hca_guid;
373 	hca_ptr->port_num = ib_attr->dia_portnum;
374 	hca_ptr->partition_key = ib_attr->dia_pkey;
375 	(void) dapl_os_memcpy((void *)&hca_ptr->hca_address,
376 	    (void *)&lifreq.lifr_addr, sizeof (hca_ptr->hca_address));
377 	hca_ptr->max_inline_send = dapls_tavor_max_inline();
378 
379 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
380 	    "process_ia: interface %s, hca guid 0x%016llx, port %d, "
381 	    "pkey 0x%08x, ip addr %s\n", lifreq.lifr_name, hca_ptr->node_GUID,
382 	    hca_ptr->port_num, hca_ptr->partition_key, dapls_inet_ntop(
383 	    (struct sockaddr *)&hca_ptr->hca_address, addr_buf, 64));
384 	return (DAT_SUCCESS);
385 }
386 
387 void
dapls_ib_state_init(void)388 dapls_ib_state_init(void)
389 {
390 	int i;
391 
392 	(void) dapl_os_lock_init(&g_tavor_state_lock);
393 	(void) dapl_os_lock_init(&g_tavor_uar_lock);
394 	(void) dapl_os_lock_init(&dapls_ib_dbp_lock);
395 
396 	for (i = 0; i < MAX_HCAS; i++) {
397 		g_tavor_state[i].hca_fd = 0;
398 		g_tavor_state[i].uarpg_baseaddr = NULL;
399 		g_tavor_state[i].uarpg_size = 0;
400 		g_tavor_state[i].bf_pg_baseaddr = NULL;
401 	}
402 }
403 
404 void
dapls_ib_state_fini(void)405 dapls_ib_state_fini(void)
406 {
407 	int i, count = 0;
408 
409 	/*
410 	 * Uinitialize the per hca instance state
411 	 */
412 	dapl_os_lock(&g_tavor_state_lock);
413 	for (i = 0; i < MAX_HCAS; i++) {
414 		if (g_tavor_state[i].uarpg_size == 0) {
415 			dapl_os_assert(g_tavor_state[i].uarpg_baseaddr ==
416 			    NULL);
417 			continue;
418 		}
419 		if (munmap(g_tavor_state[i].uarpg_baseaddr,
420 		    g_tavor_state[i].uarpg_size) < 0) {
421 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
422 			    "ib_state_fini: "
423 			    "munmap(0x%p, 0x%llx) failed(%d)\n",
424 			    g_tavor_state[i].uarpg_baseaddr,
425 			    g_tavor_state[i].uarpg_size, errno);
426 		}
427 		if ((g_tavor_state[i].bf_pg_baseaddr != NULL) &&
428 		    (munmap(g_tavor_state[i].bf_pg_baseaddr,
429 		    g_tavor_state[i].uarpg_size) < 0)) {
430 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
431 			    "ib_state_fini: "
432 			    "munmap(0x%p, 0x%llx) of blueflame failed(%d)\n",
433 			    g_tavor_state[i].bf_pg_baseaddr,
434 			    g_tavor_state[i].uarpg_size, errno);
435 		}
436 
437 		(void) close(g_tavor_state[i].hca_fd);
438 		count++;
439 	}
440 	dapl_os_unlock(&g_tavor_state_lock);
441 
442 	dapl_os_lock_destroy(&g_tavor_uar_lock);
443 	dapl_os_lock_destroy(&g_tavor_state_lock);
444 	dapl_os_lock_destroy(&dapls_ib_dbp_lock);
445 
446 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
447 	    "ib_state_fini: cleaned %d hcas\n", count);
448 }
449 
450 /*
451  * dapls_ib_open_hca
452  *
453  * Open HCA
454  *
455  * Input:
456  *      *hca_ptr          pointer to hca device
457  *      *ib_hca_handle_p  pointer to provide HCA handle
458  *
459  * Output:
460  *      none
461  *
462  * Return:
463  *      DAT_SUCCESS
464  *      DAT_INSUFFICIENT_RESOURCES
465  *
466  */
467 DAT_RETURN
dapls_ib_open_hca(IN DAPL_HCA * hca_ptr,OUT ib_hca_handle_t * ib_hca_handle_p)468 dapls_ib_open_hca(
469 	IN DAPL_HCA		*hca_ptr,
470 	OUT ib_hca_handle_t	*ib_hca_handle_p)
471 {
472 	dapl_ia_create_t		args;
473 	DAT_RETURN			dat_status;
474 	struct dapls_ib_hca_handle	*hca_p;
475 	int				fd;
476 #ifndef _LP64
477 	int				tmpfd;
478 #endif
479 	int				retval;
480 	struct sockaddr *s;
481 	struct sockaddr_in6 *v6addr;
482 	struct sockaddr_in *v4addr;
483 	dapl_ia_addr_t *sap;
484 
485 	dat_status = dapli_init_hca(hca_ptr);
486 	if (dat_status != DAT_SUCCESS) {
487 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
488 		    "dapls_ib_open_hca: init_hca failed %d\n", dat_status);
489 		return (dat_status);
490 	}
491 
492 	fd = open(DEVDAPLT, O_RDONLY);
493 	if (fd < 0) {
494 		return (DAT_INSUFFICIENT_RESOURCES);
495 	}
496 
497 #ifndef _LP64
498 	/*
499 	 * libc can't handle fd's greater than 255,  in order to
500 	 * ensure that these values remain available make fd > 255.
501 	 * Note: not needed for LP64
502 	 */
503 	tmpfd = fcntl(fd, F_DUPFD, 256);
504 	if (tmpfd < 0) {
505 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
506 		    "dapls_ib_open_hca: cannot F_DUPFD: %s\n",
507 		    strerror(errno));
508 	} else {
509 		(void) close(fd);
510 		fd = tmpfd;
511 	}
512 #endif	/* _LP64 */
513 
514 	if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
515 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
516 		    "dapls_ib_open_hca: cannot F_SETFD: %s\n", strerror(errno));
517 		(void) close(fd);
518 		return (DAT_INTERNAL_ERROR);
519 	}
520 
521 	hca_p = (struct dapls_ib_hca_handle *)dapl_os_alloc(
522 	    sizeof (struct dapls_ib_hca_handle));
523 	if (hca_p == NULL) {
524 		(void) close(fd);
525 		return (DAT_INSUFFICIENT_RESOURCES);
526 	}
527 
528 	args.ia_guid = hca_ptr->node_GUID;
529 	args.ia_port = hca_ptr->port_num;
530 	args.ia_pkey = hca_ptr->partition_key;
531 	args.ia_version = DAPL_IF_VERSION;
532 	(void) dapl_os_memzero((void *)args.ia_sadata, DAPL_ATS_NBYTES);
533 
534 	/* pass down local ip address to be stored in SA */
535 	s = (struct sockaddr *)&hca_ptr->hca_address;
536 	/* LINTED: E_BAD_PTR_CAST_ALIGN */
537 	sap = (dapl_ia_addr_t *)args.ia_sadata;
538 	switch (s->sa_family) {
539 	case AF_INET:
540 		/* LINTED: E_BAD_PTR_CAST_ALIGN */
541 		v4addr = (struct sockaddr_in *)s;
542 		sap->iad_v4 = v4addr->sin_addr;
543 		break;
544 	case AF_INET6:
545 		/* LINTED: E_BAD_PTR_CAST_ALIGN */
546 		v6addr = (struct sockaddr_in6 *)s;
547 		sap->iad_v6 = v6addr->sin6_addr;
548 		break;
549 	default:
550 		break; /* fall through */
551 	}
552 
553 	retval = ioctl(fd, DAPL_IA_CREATE, &args);
554 	if (retval != 0) {
555 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
556 		    "open_hca: ia_create failed, fd %d, "
557 		    "guid 0x%016llx, port %d, pkey 0x%x, version %d\n",
558 		    fd, args.ia_guid, args.ia_port, args.ia_pkey,
559 		    args.ia_version);
560 
561 		dapl_os_free(hca_p, sizeof (*hca_p));
562 		(void) close(fd);
563 		return (dapls_convert_error(errno, retval));
564 	}
565 
566 	hca_p->ia_fd = fd;
567 	hca_p->ia_rnum = args.ia_resnum;
568 	hca_p->hca_fd = g_tavor_state[hca_ptr->tavor_idx].hca_fd;
569 	hca_p->ia_uar = g_tavor_state[hca_ptr->tavor_idx].uarpg_baseaddr;
570 	hca_p->ia_bf = g_tavor_state[hca_ptr->tavor_idx].bf_pg_baseaddr;
571 	hca_p->ia_bf_toggle = &g_tavor_state[hca_ptr->tavor_idx].bf_toggle;
572 	*ib_hca_handle_p = hca_p;
573 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
574 	    "open_hca: ia_created, hca_p 0x%p, fd %d, "
575 	    "rnum %d, guid 0x%016llx, port %d, pkey 0x%x\n",
576 	    hca_p, hca_p->ia_fd, hca_p->ia_rnum, hca_ptr->node_GUID,
577 	    hca_ptr->port_num, hca_ptr->partition_key);
578 
579 	return (DAT_SUCCESS);
580 }
581 
582 /*
583  * dapls_ib_close_hca
584  *
585  * Open HCA
586  *
587  * Input:
588  *      ib_hca_handle   provide HCA handle
589  *
590  * Output:
591  *      none
592  *
593  * Return:
594  *      DAT_SUCCESS
595  *      DAT_INSUFFICIENT_RESOURCES
596  *
597  */
598 DAT_RETURN
dapls_ib_close_hca(IN ib_hca_handle_t ib_hca_handle)599 dapls_ib_close_hca(
600 	IN ib_hca_handle_t	ib_hca_handle)
601 {
602 	if (ib_hca_handle == NULL) {
603 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
604 		    "close_hca: ib_hca_handle == NULL\n");
605 		return (DAT_SUCCESS);
606 	}
607 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
608 	    "close_hca: closing hca 0x%p, fd %d, rnum %d\n",
609 	    ib_hca_handle, ib_hca_handle->ia_fd, ib_hca_handle->ia_rnum);
610 
611 	(void) close(ib_hca_handle->ia_fd);
612 	dapl_os_free((void *)ib_hca_handle,
613 	    sizeof (struct dapls_ib_hca_handle));
614 	return (DAT_SUCCESS);
615 }
616 
617 #if defined(IBHOSTS_NAMING)
618 #define	LINE_LEN	256
619 static int
dapli_process_fake_ibds(DAPL_HCA * hca_ptr,int hca_idx)620 dapli_process_fake_ibds(DAPL_HCA *hca_ptr, int hca_idx)
621 {
622 	char		line_buf[LINE_LEN];
623 	char		host_buf[LINE_LEN];
624 	char		localhost[LINE_LEN];
625 	ib_guid_t	prefix;
626 	ib_guid_t	guid;
627 	FILE		*fp;
628 	int		count = 0;
629 	DAPL_HCA	*hca_ptr;
630 
631 	fp = fopen("/etc/dapl/ibhosts", "r");
632 	if (fp == NULL) {
633 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
634 		    "fake_ibds: ibhosts not found!\n");
635 		return (0);
636 	}
637 	if (gethostname(localhost, LINE_LEN) != 0) {
638 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
639 		    "fake_ibds: hostname not found!\n");
640 		return (0);
641 	}
642 	while (!feof(fp)) {
643 		(void) fgets(line_buf, LINE_LEN, fp);
644 		sscanf(line_buf, "%s %llx %llx", host_buf, &prefix, &guid);
645 		(void) sprintf(line_buf, "%s-ib%d", localhost, count + 1);
646 		if (strncmp(line_buf, host_buf, strlen(line_buf)) == 0) {
647 			guid &= 0xfffffffffffffff0;
648 			hca_ptr->tavor_idx = hca_idx;
649 			hca_ptr->node_GUID = guid;
650 			hca_ptr->port_num = count + 1;
651 			hca_ptr->partition_key = 0x0000ffff;
652 			count++;
653 		}
654 		if (count >= 2) break;
655 	}
656 	(void) fclose(fp);
657 	return (count);
658 }
659 
660 #endif /* IBHOSTS_NAMING */
661