1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/conf.h>
26 #include <sys/stat.h>
27 #include <sys/modctl.h>
28 #include <sys/taskq.h>
29 #include <sys/mdi_impldefs.h>
30 #include <sys/sunmdi.h>
31 #include <sys/sunpm.h>
32 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
33 #include <sys/ib/ibnex/ibnex.h>
34 #include <sys/ib/ibnex/ibnex_devctl.h>
35 #include <sys/ib/ibtl/ibti.h>
36 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
37 #include <sys/file.h>
38 #include <sys/hwconf.h>
39 #include <sys/fs/dv_node.h>
40
41 void ibnex_handle_hca_attach(void *);
42 static int ibnex_hca_bus_config_one(dev_info_t *, void *,
43 ddi_bus_config_op_t, uint_t *, dev_info_t **);
44
45 static ibnex_node_data_t *ibnex_get_cdip_info(dev_info_t *, char *,
46 dev_info_t **, ibnex_node_type_t *);
47 static int ibnex_prom_devname_to_pkey_n_portnum(
48 char *, ib_pkey_t *, uint8_t *);
49 static dev_info_t *ibnex_config_obp_args(dev_info_t *, char *);
50
51 extern int ibnex_busctl(dev_info_t *,
52 dev_info_t *, ddi_ctl_enum_t, void *, void *);
53 extern int ibnex_map_fault(dev_info_t *,
54 dev_info_t *, struct hat *, struct seg *,
55 caddr_t, struct devpage *, pfn_t, uint_t, uint_t);
56 static int ibnex_hca_bus_config(dev_info_t *, uint_t,
57 ddi_bus_config_op_t, void *, dev_info_t **);
58 static int ibnex_hca_bus_unconfig(dev_info_t *,
59 uint_t, ddi_bus_config_op_t, void *);
60 extern dev_info_t *ibnex_config_port_node(dev_info_t *, char *);
61 extern dev_info_t *ibnex_config_obp_args(dev_info_t *, char *);
62 extern int ibnex_ioc_bus_config_one(dev_info_t **, uint_t,
63 ddi_bus_config_op_t, void *, dev_info_t **, int *);
64 extern int ibnex_pseudo_config_one(
65 ibnex_node_data_t *, char *, dev_info_t *);
66 extern void ibnex_config_all_children(dev_info_t *);
67 extern void ibnex_pseudo_initnodes(void);
68
69 extern int ibnex_pseudo_mdi_config_one(int, void *, dev_info_t **,
70 char *, char *);
71 extern int ibnex_get_dip_from_guid(ib_guid_t, int,
72 ib_pkey_t, dev_info_t **);
73 extern dev_info_t *ibnex_commsvc_initnode(dev_info_t *,
74 ibdm_port_attr_t *, int, int, ib_pkey_t, int *,
75 int);
76 extern uint64_t ibnex_str2hex(char *, int, int *);
77 extern int ibnex_str2int(char *, int, int *);
78 extern void ibnex_create_hcasvc_nodes(
79 dev_info_t *, ibdm_port_attr_t *);
80 extern void ibnex_create_port_nodes(
81 dev_info_t *, ibdm_port_attr_t *);
82 extern void ibnex_create_vppa_nodes(
83 dev_info_t *, ibdm_port_attr_t *);
84 extern int ibnex_get_pkey_commsvc_index_portnum(
85 char *, int *, ib_pkey_t *, uint8_t *);
86
87 extern ibnex_t ibnex;
88 extern int ibnex_port_settling_time;
89
90 /*
91 * The bus_ops structure defines the capabilities of HCA nexus driver.
92 */
93 struct bus_ops ibnex_ci_busops = {
94 BUSO_REV,
95 nullbusmap, /* bus_map */
96 NULL, /* bus_get_intrspec */
97 NULL, /* bus_add_intrspec */
98 NULL, /* bus_remove_intrspec */
99 ibnex_map_fault, /* Map Fault */
100 ddi_no_dma_map, /* DMA related entry points */
101 NULL,
102 NULL,
103 NULL,
104 NULL,
105 NULL,
106 NULL,
107 NULL,
108 ibnex_busctl, /* bus_ctl */
109 ddi_bus_prop_op, /* bus_prop_op */
110 NULL, /* bus_get_eventcookie */
111 NULL, /* bus_add_eventcall */
112 NULL, /* bus_remove_eventcall */
113 NULL, /* bus_post_event */
114 NULL,
115 ibnex_hca_bus_config, /* bus config */
116 ibnex_hca_bus_unconfig /* bus unconfig */
117 };
118
119 /*
120 * ibnex_hca_bus_config()
121 *
122 * BUS_CONFIG_ONE:
123 * Enumerate the exact instance of the driver. Use the device node name
124 * to locate the exact instance.
125 * Query IBDM to find whether the hardware exits for the instance of the
126 * driver. If exists, create a device node and return NDI_SUCCESS.
127 *
128 * BUS_CONFIG_ALL:
129 * Enumerate all the instances of all the possible children (seen before
130 * and never seen before).
131 *
132 * BUS_CONFIG_DRIVER:
133 * Enumerate all the instances of a particular driver.
134 */
135 static int
ibnex_hca_bus_config(dev_info_t * parent,uint_t flag,ddi_bus_config_op_t op,void * devname,dev_info_t ** child)136 ibnex_hca_bus_config(dev_info_t *parent, uint_t flag,
137 ddi_bus_config_op_t op, void *devname, dev_info_t **child)
138 {
139 int ret = IBNEX_SUCCESS, circ;
140 char *srvname, nameaddr[MAXNAMELEN];
141 dev_info_t *cdip;
142 ibnex_node_data_t *node_data;
143 ibnex_port_node_t *port_node;
144
145 /*
146 * In a normal case HCA is setup as a phci.
147 * If an HCA is in maintenance mode, its phci is not set up
148 * but the driver is attached to update the firmware. In this
149 * case, do not configure the MPxIO clients.
150 */
151 if (mdi_component_is_phci(parent, NULL) == MDI_FAILURE) {
152 if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER)
153 return (NDI_SUCCESS);
154 else
155 return (NDI_FAILURE);
156 }
157
158 switch (op) {
159 case BUS_CONFIG_ONE:
160 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config: CONFIG_ONE, "
161 "parent %p", parent);
162 ret = ibnex_hca_bus_config_one(
163 parent, devname, op, &flag, child);
164 break;
165
166 case BUS_CONFIG_OBP_ARGS:
167 mdi_devi_enter(parent, &circ);
168 cdip = ibnex_config_obp_args(parent, devname);
169 if (cdip) {
170 /*
171 * Boot case.
172 * Special handling because the "devname"
173 * format for the enumerated device is
174 * different.
175 */
176 node_data = ddi_get_parent_data(cdip);
177 port_node = &node_data->node_data.port_node;
178 if (node_data->node_type ==
179 IBNEX_VPPA_COMMSVC_NODE) {
180 srvname =
181 ibnex.ibnex_vppa_comm_svc_names[
182 port_node->port_commsvc_idx];
183 (void) snprintf(nameaddr, MAXNAMELEN,
184 "ibport@%x,%x,%s",
185 port_node->port_num,
186 port_node->port_pkey, srvname);
187 }
188 devname = (void *)nameaddr;
189 } else {
190 IBTF_DPRINTF_L2("ibnex", "\thca_bus_config: "
191 "CONFIG_OBP_ARGS : invalid state!!");
192
193 ret = IBNEX_FAILURE;
194 }
195 mdi_devi_exit(parent, circ);
196 break;
197
198 case BUS_CONFIG_ALL:
199 IBTF_DPRINTF_L4("ibnex",
200 "\thca_bus_config: CONFIG_ALL parent %p", parent);
201 ibnex_config_all_children(parent);
202 break;
203
204 case BUS_CONFIG_DRIVER:
205 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config: "
206 "CONFIG_DRIVER parent %p", parent);
207 ibnex_config_all_children(parent);
208 break;
209
210 default:
211 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config: error");
212 ret = IBNEX_FAILURE;
213 break;
214 }
215
216
217 if (ret == IBNEX_SUCCESS) {
218 if (op == BUS_CONFIG_OBP_ARGS)
219 op = BUS_CONFIG_ONE;
220
221 ret = ndi_busop_bus_config(
222 parent, flag, op, devname, child, 0);
223 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config:"
224 "ndi_busop_bus_config : retval %d", ret);
225 return (ret);
226 }
227
228 return (NDI_FAILURE);
229 }
230
231 /*
232 * ibnex_hca_bus_unconfig()
233 *
234 * Unconfigure a particular device node or all instance of a device
235 * driver device or all children of IBnex
236 */
237 static int
ibnex_hca_bus_unconfig(dev_info_t * parent,uint_t flag,ddi_bus_config_op_t op,void * device_name)238 ibnex_hca_bus_unconfig(dev_info_t *parent,
239 uint_t flag, ddi_bus_config_op_t op, void *device_name)
240 {
241
242 if (ndi_busop_bus_unconfig(parent, flag, op, device_name) !=
243 DDI_SUCCESS)
244 return (DDI_FAILURE);
245
246 if ((op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) &&
247 (flag & NDI_UNCONFIG)) {
248 ibnex_node_data_t *ndp;
249 dev_info_t *dip = NULL;
250 major_t major = (major_t)(uintptr_t)device_name;
251
252 mutex_enter(&ibnex.ibnex_mutex);
253
254 if (major == -1) {
255 /*
256 * HCA dip. When major number is -1 HCA is
257 * going away cleanup all the port nodes.
258 */
259 for (ndp = ibnex.ibnex_port_node_head;
260 ndp; ndp = ndp->node_next) {
261 ibnex_port_node_t *port_node;
262
263 port_node = &ndp->node_data.port_node;
264 if (port_node->port_pdip == parent) {
265 port_node->port_pdip = NULL;
266 ndp->node_dip = NULL;
267 ndp->node_state =
268 IBNEX_CFGADM_UNCONFIGURED;
269 }
270 }
271 } else {
272 /*
273 * HCA dip. Cleanup only the port nodes that
274 * match the major number.
275 */
276 for (ndp = ibnex.ibnex_port_node_head;
277 ndp; ndp = ndp->node_next) {
278 ibnex_port_node_t *port_node;
279
280 port_node = &ndp->node_data.port_node;
281 dip = ndp->node_dip;
282 if (dip && (ddi_driver_major(dip) ==
283 major) && port_node->port_pdip ==
284 parent) {
285 port_node->port_pdip = NULL;
286 ndp->node_dip = NULL;
287 ndp->node_state =
288 IBNEX_CFGADM_UNCONFIGURED;
289 }
290 }
291 }
292 mutex_exit(&ibnex.ibnex_mutex);
293 }
294 return (DDI_SUCCESS);
295 }
296
297 /*
298 * ibnex_config_obp_args()
299 * Configures a particular port node for a IP over IB communication
300 * service.
301 * The format of the input string "devname" is
302 * port=x,pkey=y,protocol=ip,<wanboot options>
303 * Thr format of the node name created here is
304 * ibport@<Port#>,<pkey>,<service name>
305 * where pkey = 0 for port communication service nodes
306 * Returns "dev_info_t" of the "child" node just created
307 * NULL when failed to enumerate the child node
308 *
309 */
310 static dev_info_t *
ibnex_config_obp_args(dev_info_t * parent,char * devname)311 ibnex_config_obp_args(dev_info_t *parent, char *devname)
312 {
313 int ii, index;
314 int rval, iter = 0;
315 char *temp;
316 uint8_t port_num;
317 ib_guid_t hca_guid, port_guid;
318 ib_pkey_t pkey;
319 dev_info_t *cdip;
320 boolean_t displayed = B_FALSE;
321 ibdm_port_attr_t *port_attr;
322
323 IBTF_DPRINTF_L4("ibnex", "\tconfig_obp_args: %s", devname);
324
325 /* Is this OBP node for IPoIB ? */
326 temp = devname;
327 do {
328 temp = strstr(temp, ",protocol=ip");
329 if (temp == NULL)
330 break;
331
332 if (strlen(devname) > (int)((temp - devname) + 12)) {
333 if (temp[12] == ',')
334 break;
335 } else {
336 break;
337 }
338 temp++;
339 } while (temp);
340
341 if (temp == NULL)
342 return (NULL);
343 if (ibnex_prom_devname_to_pkey_n_portnum(
344 devname, &pkey, &port_num) != IBNEX_SUCCESS) {
345 return (NULL);
346 }
347 for (index = 0; index < ibnex.ibnex_nvppa_comm_svcs; index++) {
348 if (strcmp(ibnex.ibnex_vppa_comm_svc_names[index],
349 "ipib") == 0) {
350 break;
351 }
352 }
353
354 hca_guid = ibtl_ibnex_hcadip2guid(parent);
355 if ((port_attr = ibdm_ibnex_probe_hcaport(
356 hca_guid, port_num)) == NULL) {
357 IBTF_DPRINTF_L2("ibnex",
358 "\tconfig_port_node: Port does not exist");
359 return (NULL);
360 }
361
362 /* Wait until "port is up" */
363 while (port_attr->pa_state != IBT_PORT_ACTIVE) {
364 ibdm_ibnex_free_port_attr(port_attr);
365 delay(drv_usectohz(10000));
366 if ((port_attr = ibdm_ibnex_probe_hcaport(
367 hca_guid, port_num)) == NULL) {
368 return (NULL);
369 }
370 if (iter++ == 400) {
371 if (displayed == B_FALSE) {
372 cmn_err(CE_NOTE, "\tWaiting for Port %d "
373 "initialization", port_attr->pa_port_num);
374 displayed = B_TRUE;
375 }
376 }
377 }
378 IBTF_DPRINTF_L4("ibnex", "\tPort is initialized");
379
380 mutex_enter(&ibnex.ibnex_mutex);
381 port_guid = port_attr->pa_port_guid;
382 rval = ibnex_get_dip_from_guid(port_guid, index, pkey, &cdip);
383 if (rval == IBNEX_SUCCESS && cdip != NULL) {
384 IBTF_DPRINTF_L4("ibnex", "\tconfig_port_node: Node exists");
385 mutex_exit(&ibnex.ibnex_mutex);
386 ibdm_ibnex_free_port_attr(port_attr);
387 return (cdip);
388 }
389 for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
390 if (pkey == port_attr->pa_pkey_tbl[ii].pt_pkey) {
391 cdip = ibnex_commsvc_initnode(parent, port_attr,
392 index, IBNEX_VPPA_COMMSVC_NODE, pkey, &rval,
393 IBNEX_CFGADM_ENUMERATE);
394 IBTF_DPRINTF_L5("ibnex",
395 "\t ibnex_commsvc_initnode rval %x", rval);
396 break;
397 }
398 }
399 mutex_exit(&ibnex.ibnex_mutex);
400
401 ibdm_ibnex_free_port_attr(port_attr);
402 return (cdip);
403 }
404
405
406 /*
407 * ibnex_prom_devname_to_pkey_n_portnum()
408 * Parses the device node name and extracts "PKEY" and "port#"
409 * Returns IBNEX_SUCCESS/IBNEX_FAILURE
410 */
411 static int
ibnex_prom_devname_to_pkey_n_portnum(char * devname,ib_pkey_t * pkey,uint8_t * port)412 ibnex_prom_devname_to_pkey_n_portnum(
413 char *devname, ib_pkey_t *pkey, uint8_t *port)
414 {
415 int ret = IBNEX_SUCCESS;
416 char *tmp, *tmp1;
417
418 if ((tmp = strstr(devname, "port=")) != NULL) {
419 if ((tmp = strchr(++tmp, '=')) != NULL)
420 if ((tmp1 = strchr(++tmp, ',')) != NULL)
421 *port = ibnex_str2int(tmp, (tmp1 - tmp), &ret);
422 } else
423 ret = IBNEX_FAILURE;
424
425 if ((ret == IBNEX_SUCCESS) &&
426 (tmp = strstr(devname, "pkey=")) != NULL) {
427 if ((tmp = strchr(++tmp, '=')) != NULL)
428 if ((tmp1 = strchr(++tmp, ',')) != NULL)
429 *pkey = ibnex_str2hex(tmp, (tmp1 - tmp), &ret);
430 } else
431 ret = IBNEX_FAILURE;
432
433 return (ret);
434 }
435
436 static ibnex_node_data_t *
ibnex_get_cdip_info(dev_info_t * parent,char * devname,dev_info_t ** cdip,ibnex_node_type_t * type)437 ibnex_get_cdip_info(dev_info_t *parent,
438 char *devname, dev_info_t **cdip, ibnex_node_type_t *type)
439 {
440 char *device_name, *cname = NULL, *caddr = NULL;
441 int len;
442 ibnex_node_data_t *node_data = NULL;
443
444 len = strlen((char *)devname) + 1;
445 device_name = i_ddi_strdup(devname, KM_SLEEP);
446 i_ddi_parse_name(device_name, &cname, &caddr, NULL);
447
448 IBTF_DPRINTF_L4("ibnex",
449 "\tfind_child_dip: cname %s addr %s", cname, caddr);
450
451 if (strncmp(cname, IBNEX_IOC_CNAME, 3) == 0)
452 *type = IBNEX_IOC_NODE;
453 else if (strncmp(cname, IBNEX_IBPORT_CNAME, 3) == 0)
454 *type = IBNEX_HCA_CHILD_NODE;
455 else
456 *type = IBNEX_PSEUDO_NODE;
457
458 *cdip = ndi_devi_findchild(parent, devname);
459
460 IBTF_DPRINTF_L4("ibnex",
461 "\tfind_child_dip: cdip %p type %x", *cdip, *type);
462
463 if (*cdip)
464 node_data = ddi_get_parent_data(*cdip);
465 kmem_free(device_name, len);
466
467 return (node_data);
468 }
469
470 static int
ibnex_hca_bus_config_one(dev_info_t * parent,void * devname,ddi_bus_config_op_t op,uint_t * flag,dev_info_t ** child)471 ibnex_hca_bus_config_one(dev_info_t *parent, void *devname,
472 ddi_bus_config_op_t op, uint_t *flag, dev_info_t **child)
473 {
474 int ret = IBNEX_SUCCESS, len, circ, need_bus_config;
475 char *device_name, *caddr, *cname;
476 dev_info_t *cdip;
477 ibnex_node_data_t *node_data;
478 ibnex_node_type_t node_type;
479 int index;
480 uint8_t port_num;
481 ib_pkey_t pkey;
482
483 len = strlen((char *)devname) + 1;
484 device_name = i_ddi_strdup(devname, KM_SLEEP);
485 i_ddi_parse_name(device_name, &cname, &caddr, NULL);
486
487 if (caddr == NULL || (strlen(caddr) == 0)) {
488 IBTF_DPRINTF_L2("ibnex",
489 "\thca_bus_config: Invalid device node address");
490 kmem_free(device_name, len);
491 return (IBNEX_FAILURE);
492 }
493
494 ndi_devi_enter(parent, &circ);
495 node_data = ibnex_get_cdip_info(
496 parent, devname, &cdip, &node_type);
497 ndi_devi_exit(parent, circ);
498
499 if (cdip) {
500 if ((node_data) && (node_data->node_type ==
501 IBNEX_PORT_COMMSVC_NODE)) {
502 if (node_data->node_dip == NULL) {
503 node_data->node_dip = cdip;
504 node_data->node_data.port_node.port_pdip =
505 parent;
506 }
507 }
508 }
509
510 /*
511 * If child dip is present, just return
512 * from here.
513 */
514 if (cdip != NULL || (node_data != NULL &&
515 node_data->node_dip != NULL)) {
516 goto end;
517 }
518
519 switch (node_type) {
520
521 case IBNEX_IOC_NODE:
522 ret = ibnex_ioc_bus_config_one(&parent, *flag,
523 op, devname, child, &need_bus_config);
524 if (!need_bus_config) {
525 kmem_free(device_name, len);
526 return (ret);
527 }
528 break;
529
530 case IBNEX_PSEUDO_NODE:
531 ret = IBNEX_SUCCESS;
532 mdi_devi_enter(parent, &circ);
533 ibnex_pseudo_initnodes();
534 mutex_enter(&ibnex.ibnex_mutex);
535 ret = ibnex_pseudo_config_one(NULL,
536 caddr, parent);
537 mutex_exit(&ibnex.ibnex_mutex);
538 mdi_devi_exit(parent, circ);
539 break;
540
541 default:
542 if (ibnex_get_pkey_commsvc_index_portnum(devname,
543 &index, &pkey, &port_num) != IBNEX_SUCCESS) {
544 IBTF_DPRINTF_L2("ibnex",
545 "\tconfig_port_node: Invalid Service Name");
546 kmem_free(device_name, len);
547 return (IBNEX_FAILURE);
548 }
549
550 if ((pkey != 0) && (port_num != 0)) {
551 if (strcmp("ipib",
552 ibnex.ibnex_vppa_comm_svc_names[index]) == 0) {
553 IBTF_DPRINTF_L2("ibnex",
554 "Skipping IBD devices... ");
555 break;
556 }
557 }
558
559 ndi_devi_enter(parent, &circ);
560 cdip = ibnex_config_port_node(parent, devname);
561 if (cdip)
562 ret = IBNEX_SUCCESS;
563 else
564 ret = IBNEX_FAILURE;
565 ndi_devi_exit(parent, circ);
566 break;
567 }
568 end:
569 if (node_type == IBNEX_HCA_CHILD_NODE) {
570 /* Allows enumeration under PHCI */
571 *flag |= NDI_MDI_FALLBACK;
572 }
573 kmem_free(device_name, len);
574 return (ret);
575 }
576
577 void
ibnex_handle_hca_attach(void * cb_arg)578 ibnex_handle_hca_attach(void *cb_arg)
579 {
580 ib_guid_t hca_guid = *((ib_guid_t *)cb_arg);
581 dev_info_t *phci;
582 int ii, circ;
583 ibdm_hca_list_t *hca_list;
584
585 IBTF_DPRINTF_L4("ibnex", "handle_hca_attach(%llx)", hca_guid);
586
587 phci = ibtl_ibnex_hcaguid2dip(hca_guid);
588
589 /*
590 * Enumerate children of this HCA, port nodes,
591 * VPPA & HCA_SVC nodes. Use ndi_devi_enter() for
592 * locking. IB Nexus is enumerating the children
593 * of HCA, not MPXIO clients.
594 */
595 ndi_devi_enter(phci, &circ);
596 ibdm_ibnex_port_settle_wait(hca_guid, ibnex_port_settling_time);
597 hca_list = ibdm_ibnex_get_hca_info_by_guid(hca_guid);
598 if (hca_list == NULL) {
599 ndi_devi_exit(phci, circ);
600 kmem_free(cb_arg, sizeof (ib_guid_t));
601 return;
602 }
603 ibnex_create_hcasvc_nodes(phci, hca_list->hl_hca_port_attr);
604 for (ii = 0; ii < hca_list->hl_nports; ii++) {
605 ibnex_create_port_nodes(phci, &hca_list->hl_port_attr[ii]);
606 ibnex_create_vppa_nodes(phci, &hca_list->hl_port_attr[ii]);
607 }
608 ibdm_ibnex_free_hca_list(hca_list);
609 ndi_devi_exit(phci, circ);
610 kmem_free(cb_arg, sizeof (ib_guid_t));
611 }
612