1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <sys/conf.h> 26 #include <sys/stat.h> 27 #include <sys/modctl.h> 28 #include <sys/taskq.h> 29 #include <sys/mdi_impldefs.h> 30 #include <sys/sunmdi.h> 31 #include <sys/sunpm.h> 32 #include <sys/ib/mgt/ibdm/ibdm_impl.h> 33 #include <sys/ib/ibnex/ibnex.h> 34 #include <sys/ib/ibnex/ibnex_devctl.h> 35 #include <sys/ib/ibtl/ibti.h> 36 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 37 #include <sys/file.h> 38 #include <sys/hwconf.h> 39 #include <sys/fs/dv_node.h> 40 41 void ibnex_handle_hca_attach(void *); 42 static int ibnex_hca_bus_config_one(dev_info_t *, void *, 43 ddi_bus_config_op_t, uint_t *, dev_info_t **); 44 45 static ibnex_node_data_t *ibnex_get_cdip_info(dev_info_t *, char *, 46 dev_info_t **, ibnex_node_type_t *); 47 static int ibnex_prom_devname_to_pkey_n_portnum( 48 char *, ib_pkey_t *, uint8_t *); 49 static dev_info_t *ibnex_config_obp_args(dev_info_t *, char *); 50 51 extern int ibnex_busctl(dev_info_t *, 52 dev_info_t *, ddi_ctl_enum_t, void *, void *); 53 extern int ibnex_map_fault(dev_info_t *, 54 dev_info_t *, struct hat *, struct seg *, 55 caddr_t, struct devpage *, pfn_t, uint_t, uint_t); 56 static int ibnex_hca_bus_config(dev_info_t *, uint_t, 57 ddi_bus_config_op_t, void *, dev_info_t **); 58 static int ibnex_hca_bus_unconfig(dev_info_t *, 59 uint_t, ddi_bus_config_op_t, void *); 60 extern dev_info_t *ibnex_config_port_node(dev_info_t *, char *); 61 extern dev_info_t *ibnex_config_obp_args(dev_info_t *, char *); 62 extern int ibnex_ioc_bus_config_one(dev_info_t **, uint_t, 63 ddi_bus_config_op_t, void *, dev_info_t **, int *); 64 extern int ibnex_pseudo_config_one( 65 ibnex_node_data_t *, char *, dev_info_t *); 66 extern void ibnex_config_all_children(dev_info_t *); 67 extern void ibnex_pseudo_initnodes(void); 68 69 extern int ibnex_pseudo_mdi_config_one(int, void *, dev_info_t **, 70 char *, char *); 71 extern int ibnex_get_dip_from_guid(ib_guid_t, int, 72 ib_pkey_t, dev_info_t **); 73 extern dev_info_t *ibnex_commsvc_initnode(dev_info_t *, 74 ibdm_port_attr_t *, int, int, ib_pkey_t, int *, 75 int); 76 extern uint64_t ibnex_str2hex(char *, int, int *); 77 extern int ibnex_str2int(char *, int, int *); 78 extern void ibnex_create_hcasvc_nodes( 79 dev_info_t *, ibdm_port_attr_t *); 80 extern void ibnex_create_port_nodes( 81 dev_info_t *, ibdm_port_attr_t *); 82 extern void ibnex_create_vppa_nodes( 83 dev_info_t *, ibdm_port_attr_t *); 84 extern int ibnex_get_pkey_commsvc_index_portnum( 85 char *, int *, ib_pkey_t *, uint8_t *); 86 87 extern ibnex_t ibnex; 88 extern int ibnex_port_settling_time; 89 90 /* 91 * The bus_ops structure defines the capabilities of HCA nexus driver. 92 */ 93 struct bus_ops ibnex_ci_busops = { 94 BUSO_REV, 95 nullbusmap, /* bus_map */ 96 NULL, /* bus_get_intrspec */ 97 NULL, /* bus_add_intrspec */ 98 NULL, /* bus_remove_intrspec */ 99 ibnex_map_fault, /* Map Fault */ 100 ddi_no_dma_map, /* DMA related entry points */ 101 NULL, 102 NULL, 103 NULL, 104 NULL, 105 NULL, 106 NULL, 107 NULL, 108 ibnex_busctl, /* bus_ctl */ 109 ddi_bus_prop_op, /* bus_prop_op */ 110 NULL, /* bus_get_eventcookie */ 111 NULL, /* bus_add_eventcall */ 112 NULL, /* bus_remove_eventcall */ 113 NULL, /* bus_post_event */ 114 NULL, 115 ibnex_hca_bus_config, /* bus config */ 116 ibnex_hca_bus_unconfig /* bus unconfig */ 117 }; 118 119 /* 120 * ibnex_hca_bus_config() 121 * 122 * BUS_CONFIG_ONE: 123 * Enumerate the exact instance of the driver. Use the device node name 124 * to locate the exact instance. 125 * Query IBDM to find whether the hardware exits for the instance of the 126 * driver. If exists, create a device node and return NDI_SUCCESS. 127 * 128 * BUS_CONFIG_ALL: 129 * Enumerate all the instances of all the possible children (seen before 130 * and never seen before). 131 * 132 * BUS_CONFIG_DRIVER: 133 * Enumerate all the instances of a particular driver. 134 */ 135 static int 136 ibnex_hca_bus_config(dev_info_t *parent, uint_t flag, 137 ddi_bus_config_op_t op, void *devname, dev_info_t **child) 138 { 139 int ret = IBNEX_SUCCESS, circ; 140 char *srvname, nameaddr[MAXNAMELEN]; 141 dev_info_t *cdip; 142 ibnex_node_data_t *node_data; 143 ibnex_port_node_t *port_node; 144 145 /* 146 * In a normal case HCA is setup as a phci. 147 * If an HCA is in maintenance mode, its phci is not set up 148 * but the driver is attached to update the firmware. In this 149 * case, do not configure the MPxIO clients. 150 */ 151 if (mdi_component_is_phci(parent, NULL) == MDI_FAILURE) { 152 if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) 153 return (NDI_SUCCESS); 154 else 155 return (NDI_FAILURE); 156 } 157 158 switch (op) { 159 case BUS_CONFIG_ONE: 160 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config: CONFIG_ONE, " 161 "parent %p", parent); 162 ret = ibnex_hca_bus_config_one( 163 parent, devname, op, &flag, child); 164 break; 165 166 case BUS_CONFIG_OBP_ARGS: 167 mdi_devi_enter(parent, &circ); 168 cdip = ibnex_config_obp_args(parent, devname); 169 if (cdip) { 170 /* 171 * Boot case. 172 * Special handling because the "devname" 173 * format for the enumerated device is 174 * different. 175 */ 176 node_data = ddi_get_parent_data(cdip); 177 port_node = &node_data->node_data.port_node; 178 if (node_data->node_type == 179 IBNEX_VPPA_COMMSVC_NODE) { 180 srvname = 181 ibnex.ibnex_vppa_comm_svc_names[ 182 port_node->port_commsvc_idx]; 183 (void) snprintf(nameaddr, MAXNAMELEN, 184 "ibport@%x,%x,%s", 185 port_node->port_num, 186 port_node->port_pkey, srvname); 187 } 188 devname = (void *)nameaddr; 189 } else { 190 IBTF_DPRINTF_L2("ibnex", "\thca_bus_config: " 191 "CONFIG_OBP_ARGS : invalid state!!"); 192 193 ret = IBNEX_FAILURE; 194 } 195 mdi_devi_exit(parent, circ); 196 break; 197 198 case BUS_CONFIG_ALL: 199 IBTF_DPRINTF_L4("ibnex", 200 "\thca_bus_config: CONFIG_ALL parent %p", parent); 201 ibnex_config_all_children(parent); 202 break; 203 204 case BUS_CONFIG_DRIVER: 205 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config: " 206 "CONFIG_DRIVER parent %p", parent); 207 ibnex_config_all_children(parent); 208 break; 209 210 default: 211 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config: error"); 212 ret = IBNEX_FAILURE; 213 break; 214 } 215 216 217 if (ret == IBNEX_SUCCESS) { 218 if (op == BUS_CONFIG_OBP_ARGS) 219 op = BUS_CONFIG_ONE; 220 221 ret = ndi_busop_bus_config( 222 parent, flag, op, devname, child, 0); 223 IBTF_DPRINTF_L4("ibnex", "\thca_bus_config:" 224 "ndi_busop_bus_config : retval %d", ret); 225 return (ret); 226 } 227 228 return (NDI_FAILURE); 229 } 230 231 /* 232 * ibnex_hca_bus_unconfig() 233 * 234 * Unconfigure a particular device node or all instance of a device 235 * driver device or all children of IBnex 236 */ 237 static int 238 ibnex_hca_bus_unconfig(dev_info_t *parent, 239 uint_t flag, ddi_bus_config_op_t op, void *device_name) 240 { 241 242 if (ndi_busop_bus_unconfig(parent, flag, op, device_name) != 243 DDI_SUCCESS) 244 return (DDI_FAILURE); 245 246 if ((op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) && 247 (flag & NDI_UNCONFIG)) { 248 ibnex_node_data_t *ndp; 249 dev_info_t *dip = NULL; 250 major_t major = (major_t)(uintptr_t)device_name; 251 252 mutex_enter(&ibnex.ibnex_mutex); 253 254 if (major == -1) { 255 /* 256 * HCA dip. When major number is -1 HCA is 257 * going away cleanup all the port nodes. 258 */ 259 for (ndp = ibnex.ibnex_port_node_head; 260 ndp; ndp = ndp->node_next) { 261 ibnex_port_node_t *port_node; 262 263 port_node = &ndp->node_data.port_node; 264 if (port_node->port_pdip == parent) { 265 port_node->port_pdip = NULL; 266 ndp->node_dip = NULL; 267 ndp->node_state = 268 IBNEX_CFGADM_UNCONFIGURED; 269 } 270 } 271 } else { 272 /* 273 * HCA dip. Cleanup only the port nodes that 274 * match the major number. 275 */ 276 for (ndp = ibnex.ibnex_port_node_head; 277 ndp; ndp = ndp->node_next) { 278 ibnex_port_node_t *port_node; 279 280 port_node = &ndp->node_data.port_node; 281 dip = ndp->node_dip; 282 if (dip && (ddi_driver_major(dip) == 283 major) && port_node->port_pdip == 284 parent) { 285 port_node->port_pdip = NULL; 286 ndp->node_dip = NULL; 287 ndp->node_state = 288 IBNEX_CFGADM_UNCONFIGURED; 289 } 290 } 291 } 292 mutex_exit(&ibnex.ibnex_mutex); 293 } 294 return (DDI_SUCCESS); 295 } 296 297 /* 298 * ibnex_config_obp_args() 299 * Configures a particular port node for a IP over IB communication 300 * service. 301 * The format of the input string "devname" is 302 * port=x,pkey=y,protocol=ip 303 * Thr format of the node name created here is 304 * ibport@<Port#>,<pkey>,<service name> 305 * where pkey = 0 for port communication service nodes 306 * Returns "dev_info_t" of the "child" node just created 307 * NULL when failed to enumerate the child node 308 * 309 */ 310 static dev_info_t * 311 ibnex_config_obp_args(dev_info_t *parent, char *devname) 312 { 313 int ii, index; 314 int rval, iter = 0; 315 char *temp; 316 uint8_t port_num; 317 ib_guid_t hca_guid, port_guid; 318 ib_pkey_t pkey; 319 dev_info_t *cdip; 320 boolean_t displayed = B_FALSE; 321 ibdm_port_attr_t *port_attr; 322 323 IBTF_DPRINTF_L4("ibnex", "\tconfig_obp_args: %s", devname); 324 325 /* Is this OBP node for IPoIB ? */ 326 temp = devname; 327 do { 328 temp = strstr(temp, ",protocol=ip"); 329 if (temp == NULL) 330 break; 331 332 if (strlen(devname) > (int)((temp - devname) + 12)) { 333 if (temp[12] == ',') 334 break; 335 } else { 336 break; 337 } 338 temp++; 339 } while (temp); 340 341 if (temp == NULL) 342 return (NULL); 343 if (ibnex_prom_devname_to_pkey_n_portnum( 344 devname, &pkey, &port_num) != IBNEX_SUCCESS) { 345 return (NULL); 346 } 347 for (index = 0; index < ibnex.ibnex_nvppa_comm_svcs; index++) { 348 if (strcmp(ibnex.ibnex_vppa_comm_svc_names[index], 349 "ipib") == 0) { 350 break; 351 } 352 } 353 354 hca_guid = ibtl_ibnex_hcadip2guid(parent); 355 if ((port_attr = ibdm_ibnex_probe_hcaport( 356 hca_guid, port_num)) == NULL) { 357 IBTF_DPRINTF_L2("ibnex", 358 "\tconfig_port_node: Port does not exist"); 359 return (NULL); 360 } 361 362 /* Wait until "port is up" */ 363 while (port_attr->pa_state != IBT_PORT_ACTIVE) { 364 ibdm_ibnex_free_port_attr(port_attr); 365 delay(drv_usectohz(10000)); 366 if ((port_attr = ibdm_ibnex_probe_hcaport( 367 hca_guid, port_num)) == NULL) { 368 return (NULL); 369 } 370 if (iter++ == 400) { 371 if (displayed == B_FALSE) { 372 cmn_err(CE_NOTE, "\tWaiting for Port %d " 373 "initialization", port_attr->pa_port_num); 374 displayed = B_TRUE; 375 } 376 } 377 } 378 IBTF_DPRINTF_L4("ibnex", "\tPort is initialized"); 379 380 mutex_enter(&ibnex.ibnex_mutex); 381 port_guid = port_attr->pa_port_guid; 382 rval = ibnex_get_dip_from_guid(port_guid, index, pkey, &cdip); 383 if (rval == IBNEX_SUCCESS && cdip != NULL) { 384 IBTF_DPRINTF_L4("ibnex", "\tconfig_port_node: Node exists"); 385 mutex_exit(&ibnex.ibnex_mutex); 386 ibdm_ibnex_free_port_attr(port_attr); 387 return (cdip); 388 } 389 for (ii = 0; ii < port_attr->pa_npkeys; ii++) { 390 if (pkey == port_attr->pa_pkey_tbl[ii].pt_pkey) { 391 cdip = ibnex_commsvc_initnode(parent, port_attr, 392 index, IBNEX_VPPA_COMMSVC_NODE, pkey, &rval, 393 IBNEX_CFGADM_ENUMERATE); 394 IBTF_DPRINTF_L5("ibnex", 395 "\t ibnex_commsvc_initnode rval %x", rval); 396 break; 397 } 398 } 399 mutex_exit(&ibnex.ibnex_mutex); 400 401 ibdm_ibnex_free_port_attr(port_attr); 402 return (cdip); 403 } 404 405 406 /* 407 * ibnex_prom_devname_to_pkey_n_portnum() 408 * Parses the device node name and extracts "PKEY" and "port#" 409 * Returns IBNEX_SUCCESS/IBNEX_FAILURE 410 */ 411 static int 412 ibnex_prom_devname_to_pkey_n_portnum( 413 char *devname, ib_pkey_t *pkey, uint8_t *port) 414 { 415 int ret = IBNEX_SUCCESS; 416 char *tmp, *tmp1; 417 418 if ((tmp = strstr(devname, "port=")) != NULL) { 419 if ((tmp = strchr(++tmp, '=')) != NULL) 420 if ((tmp1 = strchr(++tmp, ',')) != NULL) 421 *port = ibnex_str2int(tmp, (tmp1 - tmp), &ret); 422 } else 423 ret = IBNEX_FAILURE; 424 425 if ((ret == IBNEX_SUCCESS) && 426 (tmp = strstr(devname, "pkey=")) != NULL) { 427 if ((tmp = strchr(++tmp, '=')) != NULL) 428 if ((tmp1 = strchr(++tmp, ',')) != NULL) 429 *pkey = ibnex_str2hex(tmp, (tmp1 - tmp), &ret); 430 } else 431 ret = IBNEX_FAILURE; 432 433 return (ret); 434 } 435 436 static ibnex_node_data_t * 437 ibnex_get_cdip_info(dev_info_t *parent, 438 char *devname, dev_info_t **cdip, ibnex_node_type_t *type) 439 { 440 char *device_name, *cname = NULL, *caddr = NULL; 441 int len; 442 ibnex_node_data_t *node_data = NULL; 443 444 len = strlen((char *)devname) + 1; 445 device_name = i_ddi_strdup(devname, KM_SLEEP); 446 i_ddi_parse_name(device_name, &cname, &caddr, NULL); 447 448 IBTF_DPRINTF_L4("ibnex", 449 "\tfind_child_dip: cname %s addr %s", cname, caddr); 450 451 if (strncmp(cname, IBNEX_IOC_CNAME, 3) == 0) 452 *type = IBNEX_IOC_NODE; 453 else if (strncmp(cname, IBNEX_IBPORT_CNAME, 3) == 0) 454 *type = IBNEX_HCA_CHILD_NODE; 455 else 456 *type = IBNEX_PSEUDO_NODE; 457 458 *cdip = ndi_devi_findchild(parent, devname); 459 460 IBTF_DPRINTF_L4("ibnex", 461 "\tfind_child_dip: cdip %p type %x", *cdip, *type); 462 463 if (*cdip) 464 node_data = ddi_get_parent_data(*cdip); 465 kmem_free(device_name, len); 466 467 return (node_data); 468 } 469 470 static int 471 ibnex_hca_bus_config_one(dev_info_t *parent, void *devname, 472 ddi_bus_config_op_t op, uint_t *flag, dev_info_t **child) 473 { 474 int ret = IBNEX_SUCCESS, len, circ, need_bus_config; 475 char *device_name, *caddr, *cname; 476 dev_info_t *cdip; 477 ibnex_node_data_t *node_data; 478 ibnex_node_type_t node_type; 479 int index; 480 uint8_t port_num; 481 ib_pkey_t pkey; 482 483 len = strlen((char *)devname) + 1; 484 device_name = i_ddi_strdup(devname, KM_SLEEP); 485 i_ddi_parse_name(device_name, &cname, &caddr, NULL); 486 487 if (caddr == NULL || (strlen(caddr) == 0)) { 488 IBTF_DPRINTF_L2("ibnex", 489 "\thca_bus_config: Invalid device node address"); 490 kmem_free(device_name, len); 491 return (IBNEX_FAILURE); 492 } 493 494 ndi_devi_enter(parent, &circ); 495 node_data = ibnex_get_cdip_info( 496 parent, devname, &cdip, &node_type); 497 ndi_devi_exit(parent, circ); 498 499 if (cdip) { 500 if ((node_data) && (node_data->node_type == 501 IBNEX_PORT_COMMSVC_NODE)) { 502 if (node_data->node_dip == NULL) { 503 node_data->node_dip = cdip; 504 node_data->node_data.port_node.port_pdip = 505 parent; 506 } 507 } 508 } 509 510 /* 511 * If child dip is present, just return 512 * from here. 513 */ 514 if (cdip != NULL || (node_data != NULL && 515 node_data->node_dip != NULL)) { 516 goto end; 517 } 518 519 switch (node_type) { 520 521 case IBNEX_IOC_NODE: 522 ret = ibnex_ioc_bus_config_one(&parent, *flag, 523 op, devname, child, &need_bus_config); 524 if (!need_bus_config) { 525 kmem_free(device_name, len); 526 return (ret); 527 } 528 break; 529 530 case IBNEX_PSEUDO_NODE: 531 ret = IBNEX_SUCCESS; 532 mdi_devi_enter(parent, &circ); 533 ibnex_pseudo_initnodes(); 534 mutex_enter(&ibnex.ibnex_mutex); 535 ret = ibnex_pseudo_config_one(NULL, 536 caddr, parent); 537 mutex_exit(&ibnex.ibnex_mutex); 538 mdi_devi_exit(parent, circ); 539 break; 540 541 default: 542 if (ibnex_get_pkey_commsvc_index_portnum(devname, 543 &index, &pkey, &port_num) != IBNEX_SUCCESS) { 544 IBTF_DPRINTF_L2("ibnex", 545 "\tconfig_port_node: Invalid Service Name"); 546 kmem_free(device_name, len); 547 return (IBNEX_FAILURE); 548 } 549 550 if ((pkey != 0) && (port_num != 0)) { 551 if (strcmp("ipib", 552 ibnex.ibnex_vppa_comm_svc_names[index]) == 0) { 553 IBTF_DPRINTF_L2("ibnex", 554 "Skipping IBD devices... "); 555 break; 556 } 557 } 558 559 ndi_devi_enter(parent, &circ); 560 cdip = ibnex_config_port_node(parent, devname); 561 if (cdip) 562 ret = IBNEX_SUCCESS; 563 else 564 ret = IBNEX_FAILURE; 565 ndi_devi_exit(parent, circ); 566 break; 567 } 568 end: 569 if (node_type == IBNEX_HCA_CHILD_NODE) { 570 /* Allows enumeration under PHCI */ 571 *flag |= NDI_MDI_FALLBACK; 572 } 573 kmem_free(device_name, len); 574 return (ret); 575 } 576 577 void 578 ibnex_handle_hca_attach(void *cb_arg) 579 { 580 ib_guid_t hca_guid = *((ib_guid_t *)cb_arg); 581 dev_info_t *phci; 582 int ii, circ; 583 ibdm_hca_list_t *hca_list; 584 585 IBTF_DPRINTF_L4("ibnex", "handle_hca_attach(%llx)", hca_guid); 586 587 phci = ibtl_ibnex_hcaguid2dip(hca_guid); 588 589 /* 590 * Enumerate children of this HCA, port nodes, 591 * VPPA & HCA_SVC nodes. Use ndi_devi_enter() for 592 * locking. IB Nexus is enumerating the children 593 * of HCA, not MPXIO clients. 594 */ 595 ndi_devi_enter(phci, &circ); 596 ibdm_ibnex_port_settle_wait(hca_guid, ibnex_port_settling_time); 597 hca_list = ibdm_ibnex_get_hca_info_by_guid(hca_guid); 598 if (hca_list == NULL) { 599 ndi_devi_exit(phci, circ); 600 kmem_free(cb_arg, sizeof (ib_guid_t)); 601 return; 602 } 603 ibnex_create_hcasvc_nodes(phci, hca_list->hl_hca_port_attr); 604 for (ii = 0; ii < hca_list->hl_nports; ii++) { 605 ibnex_create_port_nodes(phci, &hca_list->hl_port_attr[ii]); 606 ibnex_create_vppa_nodes(phci, &hca_list->hl_port_attr[ii]); 607 } 608 ibdm_ibnex_free_hca_list(hca_list); 609 ndi_devi_exit(phci, circ); 610 kmem_free(cb_arg, sizeof (ib_guid_t)); 611 } 612