1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright (c) 2017, Joyent, Inc. 14 * Copyright 2020 Robert Mustacchi 15 * Copyright 2020 Oxide Computer Company 16 */ 17 18 /* 19 * This module covers enumerating properties of physical NICs. At this time, as 20 * various devices are discovered that may relate to various networking gear, we 21 * will attempt to enumerate ports and transceivers under them, if requested. 22 */ 23 24 #include <strings.h> 25 #include <libdevinfo.h> 26 #include <libdladm.h> 27 #include <libdllink.h> 28 #include <libdlstat.h> 29 #include <libsff.h> 30 #include <unistd.h> 31 #include <sys/dld_ioc.h> 32 #include <sys/dld.h> 33 #include <sys/mac.h> 34 35 #include <sys/fm/protocol.h> 36 #include <fm/topo_mod.h> 37 #include <fm/topo_list.h> 38 #include <fm/topo_method.h> 39 40 #include <topo_port.h> 41 #include <topo_transceiver.h> 42 43 #include "topo_nic.h" 44 45 typedef enum { 46 NIC_PORT_UNKNOWN, 47 NIC_PORT_SFF 48 } nic_port_type_t; 49 50 static const topo_pgroup_info_t datalink_pgroup = { 51 TOPO_PGROUP_DATALINK, 52 TOPO_STABILITY_PRIVATE, 53 TOPO_STABILITY_PRIVATE, 54 1 55 }; 56 57 typedef struct nic_port_mac { 58 char npm_mac[ETHERADDRSTRL]; 59 boolean_t npm_valid; 60 topo_mod_t *npm_mod; 61 } nic_port_mac_t; 62 63 /* 64 * The following drivers have their main function be a nexus driver which 65 * enumerates children itself which are mac providers rather than having the 66 * main PCI functions actually be the device nodes. As such, when we encounter 67 * them, we need to enumerate them in a slightly different way by walking over 68 * each child of the instance. 69 */ 70 static const char *nic_nexuses[] = { 71 "t4nex", 72 NULL 73 }; 74 75 /* 76 * The first MAC address is always the primary MAC address, so we only worry 77 * about the first. Thus this function always returns B_FALSE, to terminate 78 * iteration. 79 */ 80 static boolean_t 81 nic_port_datalink_mac_cb(void *arg, dladm_macaddr_attr_t *attr) 82 { 83 nic_port_mac_t *mac = arg; 84 85 if (attr->ma_addrlen != ETHERADDRL) { 86 topo_mod_dprintf(mac->npm_mod, 87 "found address with bad length: %u\n", attr->ma_addrlen); 88 return (B_FALSE); 89 } 90 91 (void) snprintf(mac->npm_mac, sizeof (mac->npm_mac), 92 "%02x:%02x:%02x:%02x:%02x:%02x", 93 attr->ma_addr[0], attr->ma_addr[1], attr->ma_addr[2], 94 attr->ma_addr[3], attr->ma_addr[4], attr->ma_addr[5]); 95 mac->npm_valid = B_TRUE; 96 return (B_FALSE); 97 } 98 99 static int 100 nic_port_datalink_props(topo_mod_t *mod, tnode_t *port, dladm_handle_t handle, 101 datalink_id_t linkid) 102 { 103 int err; 104 dladm_status_t status; 105 uint64_t ifspeed; 106 link_duplex_t duplex; 107 link_state_t state; 108 const char *duplex_str, *state_str; 109 datalink_class_t dlclass; 110 uint32_t media; 111 char dlname[MAXLINKNAMELEN * 2]; 112 char dlerr[DLADM_STRSIZE]; 113 nic_port_mac_t mac; 114 115 status = dladm_datalink_id2info(handle, linkid, NULL, &dlclass, &media, 116 dlname, sizeof (dlname)); 117 if (status != DLADM_STATUS_OK) { 118 topo_mod_dprintf(mod, "failed to get link info: %s\n", 119 dladm_status2str(status, dlerr)); 120 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM)); 121 } 122 123 if (dlclass != DATALINK_CLASS_PHYS) { 124 return (0); 125 } 126 127 status = dladm_get_single_mac_stat(handle, linkid, "ifspeed", 128 KSTAT_DATA_UINT64, &ifspeed); 129 if (status != DLADM_STATUS_OK) { 130 topo_mod_dprintf(mod, "failed to get ifspeed: %s\n", 131 dladm_status2str(status, dlerr)); 132 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM)); 133 } 134 135 status = dladm_get_single_mac_stat(handle, linkid, "link_duplex", 136 KSTAT_DATA_UINT32, &duplex); 137 if (status != DLADM_STATUS_OK) { 138 topo_mod_dprintf(mod, "failed to get link_duplex: %s\n", 139 dladm_status2str(status, dlerr)); 140 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM)); 141 } 142 143 switch (duplex) { 144 case LINK_DUPLEX_HALF: 145 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_HALF; 146 break; 147 case LINK_DUPLEX_FULL: 148 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_FULL; 149 break; 150 default: 151 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_UNKNOWN; 152 break; 153 } 154 155 status = dladm_get_single_mac_stat(handle, linkid, "link_state", 156 KSTAT_DATA_UINT32, &state); 157 if (status != DLADM_STATUS_OK) { 158 topo_mod_dprintf(mod, "failed to get link_duplex: %s\n", 159 dladm_status2str(status, dlerr)); 160 return (topo_mod_seterrno(mod, status)); 161 } 162 163 switch (state) { 164 case LINK_STATE_UP: 165 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_UP; 166 break; 167 case LINK_STATE_DOWN: 168 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_DOWN; 169 break; 170 default: 171 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_UNKNOWN; 172 break; 173 } 174 175 /* 176 * Override the duplex if the link is down. Some devices will leave it 177 * set at half as opposed to unknown. 178 */ 179 if (state == LINK_STATE_DOWN || state == LINK_STATE_UNKNOWN) { 180 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_UNKNOWN; 181 } 182 183 mac.npm_mac[0] = '\0'; 184 mac.npm_valid = B_FALSE; 185 mac.npm_mod = mod; 186 if (media == DL_ETHER) { 187 (void) dladm_walk_macaddr(handle, linkid, &mac, 188 nic_port_datalink_mac_cb); 189 } 190 191 if (topo_pgroup_create(port, &datalink_pgroup, &err) != 0) { 192 topo_mod_dprintf(mod, "falied to create property group %s: " 193 "%s\n", TOPO_PGROUP_DATALINK, topo_strerror(err)); 194 return (topo_mod_seterrno(mod, err)); 195 } 196 197 if (topo_prop_set_uint64(port, TOPO_PGROUP_DATALINK, 198 TOPO_PGROUP_DATALINK_LINK_SPEED, TOPO_PROP_IMMUTABLE, ifspeed, 199 &err) != 0) { 200 topo_mod_dprintf(mod, "failed to set %s property: %s\n", 201 TOPO_PGROUP_DATALINK_LINK_SPEED, topo_strerror(err)); 202 return (topo_mod_seterrno(mod, err)); 203 } 204 205 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 206 TOPO_PGROUP_DATALINK_LINK_DUPLEX, TOPO_PROP_IMMUTABLE, duplex_str, 207 &err) != 0) { 208 topo_mod_dprintf(mod, "failed to set %s property: %s\n", 209 TOPO_PGROUP_DATALINK_LINK_DUPLEX, topo_strerror(err)); 210 return (topo_mod_seterrno(mod, err)); 211 } 212 213 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 214 TOPO_PGROUP_DATALINK_LINK_STATUS, TOPO_PROP_IMMUTABLE, state_str, 215 &err) != 0) { 216 topo_mod_dprintf(mod, "failed to set %s property: %s\n", 217 TOPO_PGROUP_DATALINK_LINK_STATUS, topo_strerror(err)); 218 return (topo_mod_seterrno(mod, err)); 219 } 220 221 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 222 TOPO_PGROUP_DATALINK_LINK_NAME, TOPO_PROP_IMMUTABLE, dlname, 223 &err) != 0) { 224 topo_mod_dprintf(mod, "failed to set %s propery: %s\n", 225 TOPO_PGROUP_DATALINK_LINK_NAME, topo_strerror(err)); 226 return (topo_mod_seterrno(mod, err)); 227 } 228 229 if (mac.npm_valid) { 230 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 231 TOPO_PGROUP_DATALINK_PMAC, TOPO_PROP_IMMUTABLE, 232 mac.npm_mac, &err) != 0) { 233 topo_mod_dprintf(mod, "failed to set %s propery: %s\n", 234 TOPO_PGROUP_DATALINK_PMAC, topo_strerror(err)); 235 return (topo_mod_seterrno(mod, err)); 236 } 237 } 238 239 240 return (0); 241 } 242 243 /* 244 * Create an instance of a transceiver with the specified id. We must create 245 * both its port and the transceiver node. 246 */ 247 static int 248 nic_create_transceiver(topo_mod_t *mod, tnode_t *pnode, dladm_handle_t handle, 249 datalink_id_t linkid, topo_instance_t inst, uint_t tranid, 250 nic_port_type_t port_type) 251 { 252 int ret; 253 tnode_t *port; 254 dld_ioc_gettran_t dgt; 255 dld_ioc_tranio_t dti; 256 uint8_t buf[256]; 257 char ouibuf[16]; 258 char *vendor = NULL, *part = NULL, *rev = NULL, *serial = NULL; 259 nvlist_t *nvl = NULL; 260 261 switch (port_type) { 262 case NIC_PORT_UNKNOWN: 263 ret = port_create_unknown(mod, pnode, inst, &port); 264 break; 265 case NIC_PORT_SFF: 266 ret = port_create_sff(mod, pnode, inst, &port); 267 break; 268 default: 269 return (-1); 270 } 271 272 if ((ret = nic_port_datalink_props(mod, port, handle, linkid)) != 0) 273 return (ret); 274 275 if (port_type != NIC_PORT_SFF) 276 return (0); 277 278 bzero(&dgt, sizeof (dgt)); 279 dgt.dgt_linkid = linkid; 280 dgt.dgt_tran_id = tranid; 281 282 if (ioctl(dladm_dld_fd(handle), DLDIOC_GETTRAN, &dgt) != 0) { 283 if (errno == ENOTSUP) 284 return (0); 285 return (-1); 286 } 287 288 if (dgt.dgt_present == 0) 289 return (0); 290 291 bzero(&dti, sizeof (dti)); 292 dti.dti_linkid = linkid; 293 dti.dti_tran_id = tranid; 294 dti.dti_page = 0xa0; 295 dti.dti_nbytes = sizeof (buf); 296 dti.dti_buf = (uintptr_t)buf; 297 298 if (ioctl(dladm_dld_fd(handle), DLDIOC_READTRAN, &dti) == 0) { 299 uchar_t *oui; 300 uint_t nbyte; 301 302 if (libsff_parse(buf, dti.dti_nbytes, dti.dti_page, 303 &nvl) == 0) { 304 if ((ret = nvlist_lookup_string(nvl, LIBSFF_KEY_VENDOR, 305 &vendor)) != 0 && nvlist_lookup_byte_array(nvl, 306 LIBSFF_KEY_OUI, &oui, &nbyte) == 0 && nbyte == 3) { 307 if (snprintf(ouibuf, sizeof (ouibuf), 308 "%02x:%02x:%02x", oui[0], oui[1], oui[2]) < 309 sizeof (ouibuf)) { 310 vendor = ouibuf; 311 } 312 } else if (ret != 0) { 313 vendor = NULL; 314 } 315 316 if (nvlist_lookup_string(nvl, LIBSFF_KEY_PART, 317 &part) != 0) { 318 part = NULL; 319 } 320 321 if (nvlist_lookup_string(nvl, LIBSFF_KEY_REVISION, 322 &rev) != 0) { 323 rev = NULL; 324 } 325 326 if (nvlist_lookup_string(nvl, LIBSFF_KEY_SERIAL, 327 &serial) != 0) { 328 serial = NULL; 329 } 330 } 331 } 332 333 if (transceiver_range_create(mod, port, 0, 0) != 0) { 334 nvlist_free(nvl); 335 return (-1); 336 } 337 338 if (transceiver_create_sff(mod, port, 0, dgt.dgt_usable, vendor, part, 339 rev, serial, NULL) != 0) { 340 nvlist_free(nvl); 341 return (-1); 342 } 343 344 nvlist_free(nvl); 345 return (0); 346 } 347 348 static boolean_t 349 nic_enum_link_ntrans(dladm_handle_t handle, datalink_id_t linkid, uint_t *ntran, 350 nic_port_type_t *pt) 351 { 352 dld_ioc_gettran_t dgt; 353 354 memset(&dgt, 0, sizeof (dgt)); 355 dgt.dgt_linkid = linkid; 356 dgt.dgt_tran_id = DLDIOC_GETTRAN_GETNTRAN; 357 358 if (ioctl(dladm_dld_fd(handle), DLDIOC_GETTRAN, &dgt) != 0) { 359 if (errno != ENOTSUP) { 360 return (B_FALSE); 361 } 362 *pt = NIC_PORT_UNKNOWN; 363 *ntran = 1; 364 } else { 365 *ntran = dgt.dgt_tran_id; 366 *pt = NIC_PORT_SFF; 367 } 368 369 return (B_TRUE); 370 } 371 372 static boolean_t 373 nic_enum_devinfo_linkid(dladm_handle_t handle, di_node_t din, 374 datalink_id_t *linkidp) 375 { 376 char dname[MAXNAMELEN]; 377 378 if (snprintf(dname, sizeof (dname), "%s%d", di_driver_name(din), 379 di_instance(din)) >= sizeof (dname)) { 380 return (B_FALSE); 381 } 382 383 if (dladm_dev2linkid(handle, dname, linkidp) != DLADM_STATUS_OK) 384 return (B_FALSE); 385 386 return (B_TRUE); 387 } 388 389 /* 390 * When we encounter a nexus driver we need to walk each of its children to 391 * actually get at the dladm handles and devices that we can use for this. 392 */ 393 static int 394 nic_enum_nexus(topo_mod_t *mod, tnode_t *pnode, dladm_handle_t handle, 395 di_node_t din) 396 { 397 uint_t total_ports = 0; 398 nic_port_type_t pt; 399 di_node_t child; 400 401 /* 402 * We have to iterate child nodes in two passes. The first pass is used 403 * to determine the number of children to create. FM requires that we 404 * create all the children nodes at once currently. 405 */ 406 for (child = di_child_node(din); child != DI_NODE_NIL; 407 child = di_sibling_node(child)) { 408 datalink_id_t linkid; 409 uint_t ntrans; 410 411 if (!nic_enum_devinfo_linkid(handle, child, &linkid)) 412 return (-1); 413 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt)) 414 return (-1); 415 416 total_ports += ntrans; 417 } 418 419 if (total_ports == 0) 420 return (0); 421 422 if (port_range_create(mod, pnode, 0, total_ports - 1) != 0) 423 return (-1); 424 425 total_ports = 0; 426 for (child = di_child_node(din); child != DI_NODE_NIL; 427 child = di_sibling_node(child)) { 428 datalink_id_t linkid; 429 uint_t i, ntrans; 430 431 if (!nic_enum_devinfo_linkid(handle, child, &linkid)) 432 return (-1); 433 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt)) 434 return (-1); 435 436 for (i = 0; i < ntrans; i++) { 437 if (nic_create_transceiver(mod, pnode, handle, linkid, 438 total_ports + i, i, pt) != 0) { 439 return (-1); 440 } 441 } 442 443 total_ports += ntrans; 444 } 445 446 return (0); 447 } 448 449 /* ARGSUSED */ 450 static int 451 nic_enum(topo_mod_t *mod, tnode_t *pnode, const char *name, 452 topo_instance_t min, topo_instance_t max, void *modarg, void *data) 453 { 454 di_node_t din = data; 455 datalink_id_t linkid; 456 dladm_handle_t handle; 457 uint_t ntrans, i; 458 nic_port_type_t pt; 459 const char *drv; 460 461 if (strcmp(name, NIC) != 0) { 462 topo_mod_dprintf(mod, "nic_enum: asked to enumerate unknown " 463 "component: %s\n", name); 464 return (-1); 465 } 466 467 if (din == NULL) { 468 topo_mod_dprintf(mod, "nic_enum: missing data argument\n"); 469 return (-1); 470 } 471 472 if ((handle = topo_mod_getspecific(mod)) == NULL) { 473 topo_mod_dprintf(mod, "nic_enum: failed to get nic module " 474 "specific data\n"); 475 return (-1); 476 } 477 478 /* 479 * No driver attached, just skip it. 480 */ 481 if ((drv = di_driver_name(din)) == NULL) { 482 return (0); 483 } 484 485 for (i = 0; nic_nexuses[i] != NULL; i++) { 486 if (strcmp(drv, nic_nexuses[i]) == 0) { 487 return (nic_enum_nexus(mod, pnode, handle, din)); 488 } 489 } 490 491 if (!nic_enum_devinfo_linkid(handle, din, &linkid)) 492 return (-1); 493 494 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt)) 495 return (-1); 496 497 if (ntrans == 0) 498 return (0); 499 500 if (port_range_create(mod, pnode, 0, ntrans - 1) != 0) 501 return (-1); 502 503 for (i = 0; i < ntrans; i++) { 504 if (nic_create_transceiver(mod, pnode, handle, linkid, i, i, 505 pt) != 0) { 506 return (-1); 507 } 508 } 509 510 return (0); 511 } 512 513 static const topo_modops_t nic_ops = { 514 nic_enum, NULL 515 }; 516 517 static topo_modinfo_t nic_mod = { 518 NIC, FM_FMRI_SCHEME_HC, NIC_VERSION, &nic_ops 519 }; 520 521 int 522 _topo_init(topo_mod_t *mod, topo_version_t version) 523 { 524 dladm_handle_t handle; 525 526 if (getenv("TOPONICDEBUG") != NULL) 527 topo_mod_setdebug(mod); 528 529 topo_mod_dprintf(mod, "_mod_init: " 530 "initializing %s enumerator\n", NIC); 531 532 if (version != NIC_VERSION) { 533 return (-1); 534 } 535 536 if (dladm_open(&handle) != 0) 537 return (-1); 538 539 if (topo_mod_register(mod, &nic_mod, TOPO_VERSION) != 0) { 540 dladm_close(handle); 541 return (-1); 542 } 543 544 topo_mod_setspecific(mod, handle); 545 546 return (0); 547 } 548 549 void 550 _topo_fini(topo_mod_t *mod) 551 { 552 dladm_handle_t handle; 553 554 if ((handle = topo_mod_getspecific(mod)) == NULL) 555 return; 556 557 dladm_close(handle); 558 topo_mod_setspecific(mod, NULL); 559 } 560