1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright (c) 2017, Joyent, Inc. 14 * Copyright 2020 Robert Mustacchi 15 * Copyright 2023 Oxide Computer Company 16 */ 17 18 /* 19 * This module covers enumerating properties of physical NICs. At this time, as 20 * various devices are discovered that may relate to various networking gear, we 21 * will attempt to enumerate ports and transceivers under them, if requested. 22 */ 23 24 #include <strings.h> 25 #include <libdevinfo.h> 26 #include <libdladm.h> 27 #include <libdllink.h> 28 #include <libdlstat.h> 29 #include <libsff.h> 30 #include <unistd.h> 31 #include <sys/dld_ioc.h> 32 #include <sys/dld.h> 33 #include <sys/mac.h> 34 35 #include <sys/fm/protocol.h> 36 #include <fm/topo_mod.h> 37 #include <fm/topo_list.h> 38 #include <fm/topo_method.h> 39 40 #include <topo_port.h> 41 #include <topo_transceiver.h> 42 43 #include "topo_nic.h" 44 45 typedef enum { 46 NIC_PORT_UNKNOWN, 47 NIC_PORT_SFF 48 } nic_port_type_t; 49 50 static const topo_pgroup_info_t datalink_pgroup = { 51 TOPO_PGROUP_DATALINK, 52 TOPO_STABILITY_PRIVATE, 53 TOPO_STABILITY_PRIVATE, 54 1 55 }; 56 57 typedef struct nic_port_mac { 58 char npm_mac[ETHERADDRSTRL]; 59 boolean_t npm_valid; 60 topo_mod_t *npm_mod; 61 } nic_port_mac_t; 62 63 /* 64 * The following drivers have their main function be a nexus driver which 65 * enumerates children itself which are mac providers rather than having the 66 * main PCI functions actually be the device nodes. As such, when we encounter 67 * them, we need to enumerate them in a slightly different way by walking over 68 * each child of the instance. 69 */ 70 static const char *nic_nexuses[] = { 71 "t4nex", 72 NULL 73 }; 74 75 /* 76 * The first MAC address is always the primary MAC address, so we only worry 77 * about the first. Thus this function always returns B_FALSE, to terminate 78 * iteration. 79 */ 80 static boolean_t 81 nic_port_datalink_mac_cb(void *arg, dladm_macaddr_attr_t *attr) 82 { 83 nic_port_mac_t *mac = arg; 84 85 if (attr->ma_addrlen != ETHERADDRL) { 86 topo_mod_dprintf(mac->npm_mod, 87 "found address with bad length: %u\n", attr->ma_addrlen); 88 return (B_FALSE); 89 } 90 91 (void) snprintf(mac->npm_mac, sizeof (mac->npm_mac), 92 "%02x:%02x:%02x:%02x:%02x:%02x", 93 attr->ma_addr[0], attr->ma_addr[1], attr->ma_addr[2], 94 attr->ma_addr[3], attr->ma_addr[4], attr->ma_addr[5]); 95 mac->npm_valid = B_TRUE; 96 return (B_FALSE); 97 } 98 99 static int 100 nic_port_datalink_props(topo_mod_t *mod, tnode_t *port, dladm_handle_t handle, 101 datalink_id_t linkid) 102 { 103 int err; 104 dladm_status_t status; 105 uint64_t ifspeed; 106 link_duplex_t duplex; 107 link_state_t state; 108 const char *duplex_str, *state_str, *media_str; 109 datalink_class_t dlclass; 110 uint32_t media; 111 char dlname[MAXLINKNAMELEN * 2]; 112 char dlerr[DLADM_STRSIZE], dlmedia[DLADM_PROP_VAL_MAX], *valptr[1]; 113 uint_t valcnt = 1; 114 nic_port_mac_t mac; 115 116 status = dladm_datalink_id2info(handle, linkid, NULL, &dlclass, &media, 117 dlname, sizeof (dlname)); 118 if (status != DLADM_STATUS_OK) { 119 topo_mod_dprintf(mod, "failed to get link info: %s\n", 120 dladm_status2str(status, dlerr)); 121 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM)); 122 } 123 124 if (dlclass != DATALINK_CLASS_PHYS) { 125 return (0); 126 } 127 128 status = dladm_get_single_mac_stat(handle, linkid, "ifspeed", 129 KSTAT_DATA_UINT64, &ifspeed); 130 if (status != DLADM_STATUS_OK) { 131 topo_mod_dprintf(mod, "failed to get ifspeed: %s\n", 132 dladm_status2str(status, dlerr)); 133 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM)); 134 } 135 136 status = dladm_get_single_mac_stat(handle, linkid, "link_duplex", 137 KSTAT_DATA_UINT32, &duplex); 138 if (status != DLADM_STATUS_OK) { 139 topo_mod_dprintf(mod, "failed to get link_duplex: %s\n", 140 dladm_status2str(status, dlerr)); 141 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM)); 142 } 143 144 switch (duplex) { 145 case LINK_DUPLEX_HALF: 146 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_HALF; 147 break; 148 case LINK_DUPLEX_FULL: 149 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_FULL; 150 break; 151 default: 152 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_UNKNOWN; 153 break; 154 } 155 156 status = dladm_get_single_mac_stat(handle, linkid, "link_state", 157 KSTAT_DATA_UINT32, &state); 158 if (status != DLADM_STATUS_OK) { 159 topo_mod_dprintf(mod, "failed to get link_duplex: %s\n", 160 dladm_status2str(status, dlerr)); 161 return (topo_mod_seterrno(mod, status)); 162 } 163 164 switch (state) { 165 case LINK_STATE_UP: 166 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_UP; 167 break; 168 case LINK_STATE_DOWN: 169 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_DOWN; 170 break; 171 default: 172 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_UNKNOWN; 173 break; 174 } 175 176 /* 177 * Override the duplex if the link is down. Some devices will leave it 178 * set at half as opposed to unknown. 179 */ 180 if (state == LINK_STATE_DOWN || state == LINK_STATE_UNKNOWN) { 181 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_UNKNOWN; 182 } 183 184 media_str = NULL; 185 if (state == LINK_STATE_UP) { 186 valptr[0] = dlmedia; 187 if (dladm_get_linkprop(handle, linkid, DLADM_PROP_VAL_CURRENT, 188 "media", valptr, &valcnt) == DLADM_STATUS_OK) { 189 media_str = dlmedia; 190 } 191 } 192 193 mac.npm_mac[0] = '\0'; 194 mac.npm_valid = B_FALSE; 195 mac.npm_mod = mod; 196 if (media == DL_ETHER) { 197 (void) dladm_walk_macaddr(handle, linkid, &mac, 198 nic_port_datalink_mac_cb); 199 } 200 201 if (topo_pgroup_create(port, &datalink_pgroup, &err) != 0) { 202 topo_mod_dprintf(mod, "falied to create property group %s: " 203 "%s\n", TOPO_PGROUP_DATALINK, topo_strerror(err)); 204 return (topo_mod_seterrno(mod, err)); 205 } 206 207 if (topo_prop_set_uint64(port, TOPO_PGROUP_DATALINK, 208 TOPO_PGROUP_DATALINK_LINK_SPEED, TOPO_PROP_IMMUTABLE, ifspeed, 209 &err) != 0) { 210 topo_mod_dprintf(mod, "failed to set %s property: %s\n", 211 TOPO_PGROUP_DATALINK_LINK_SPEED, topo_strerror(err)); 212 return (topo_mod_seterrno(mod, err)); 213 } 214 215 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 216 TOPO_PGROUP_DATALINK_LINK_DUPLEX, TOPO_PROP_IMMUTABLE, duplex_str, 217 &err) != 0) { 218 topo_mod_dprintf(mod, "failed to set %s property: %s\n", 219 TOPO_PGROUP_DATALINK_LINK_DUPLEX, topo_strerror(err)); 220 return (topo_mod_seterrno(mod, err)); 221 } 222 223 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 224 TOPO_PGROUP_DATALINK_LINK_STATUS, TOPO_PROP_IMMUTABLE, state_str, 225 &err) != 0) { 226 topo_mod_dprintf(mod, "failed to set %s property: %s\n", 227 TOPO_PGROUP_DATALINK_LINK_STATUS, topo_strerror(err)); 228 return (topo_mod_seterrno(mod, err)); 229 } 230 231 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 232 TOPO_PGROUP_DATALINK_LINK_NAME, TOPO_PROP_IMMUTABLE, dlname, 233 &err) != 0) { 234 topo_mod_dprintf(mod, "failed to set %s propery: %s\n", 235 TOPO_PGROUP_DATALINK_LINK_NAME, topo_strerror(err)); 236 return (topo_mod_seterrno(mod, err)); 237 } 238 239 if (media_str != NULL && topo_prop_set_string(port, 240 TOPO_PGROUP_DATALINK, TOPO_PGROUP_DATALINK_LINK_MEDIA, 241 TOPO_PROP_IMMUTABLE, media_str, &err) != 0) { 242 topo_mod_dprintf(mod, "failed to set %s propery: %s\n", 243 TOPO_PGROUP_DATALINK_LINK_MEDIA, topo_strerror(err)); 244 return (topo_mod_seterrno(mod, err)); 245 } 246 247 if (mac.npm_valid) { 248 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK, 249 TOPO_PGROUP_DATALINK_PMAC, TOPO_PROP_IMMUTABLE, 250 mac.npm_mac, &err) != 0) { 251 topo_mod_dprintf(mod, "failed to set %s propery: %s\n", 252 TOPO_PGROUP_DATALINK_PMAC, topo_strerror(err)); 253 return (topo_mod_seterrno(mod, err)); 254 } 255 } 256 257 258 return (0); 259 } 260 261 /* 262 * Create an instance of a transceiver with the specified id. We must create 263 * both its port and the transceiver node. 264 */ 265 static int 266 nic_create_transceiver(topo_mod_t *mod, tnode_t *pnode, dladm_handle_t handle, 267 datalink_id_t linkid, topo_instance_t inst, uint_t tranid, 268 nic_port_type_t port_type) 269 { 270 int ret; 271 tnode_t *port; 272 dld_ioc_gettran_t dgt; 273 dld_ioc_tranio_t dti; 274 uint8_t buf[256]; 275 char ouibuf[16]; 276 char *vendor = NULL, *part = NULL, *rev = NULL, *serial = NULL; 277 nvlist_t *nvl = NULL; 278 279 switch (port_type) { 280 case NIC_PORT_UNKNOWN: 281 ret = port_create_unknown(mod, pnode, inst, &port); 282 break; 283 case NIC_PORT_SFF: 284 ret = port_create_sff(mod, pnode, inst, &port); 285 break; 286 default: 287 return (-1); 288 } 289 290 if ((ret = nic_port_datalink_props(mod, port, handle, linkid)) != 0) 291 return (ret); 292 293 if (port_type != NIC_PORT_SFF) 294 return (0); 295 296 bzero(&dgt, sizeof (dgt)); 297 dgt.dgt_linkid = linkid; 298 dgt.dgt_tran_id = tranid; 299 300 if (ioctl(dladm_dld_fd(handle), DLDIOC_GETTRAN, &dgt) != 0) { 301 if (errno == ENOTSUP) 302 return (0); 303 return (-1); 304 } 305 306 if (dgt.dgt_present == 0) 307 return (0); 308 309 bzero(&dti, sizeof (dti)); 310 dti.dti_linkid = linkid; 311 dti.dti_tran_id = tranid; 312 dti.dti_page = 0xa0; 313 dti.dti_nbytes = sizeof (buf); 314 dti.dti_buf = (uintptr_t)buf; 315 316 if (ioctl(dladm_dld_fd(handle), DLDIOC_READTRAN, &dti) == 0) { 317 uchar_t *oui; 318 uint_t nbyte; 319 320 if (libsff_parse(buf, dti.dti_nbytes, dti.dti_page, 321 &nvl) == 0) { 322 if ((ret = nvlist_lookup_string(nvl, LIBSFF_KEY_VENDOR, 323 &vendor)) != 0 && nvlist_lookup_byte_array(nvl, 324 LIBSFF_KEY_OUI, &oui, &nbyte) == 0 && nbyte == 3) { 325 if (snprintf(ouibuf, sizeof (ouibuf), 326 "%02x:%02x:%02x", oui[0], oui[1], oui[2]) < 327 sizeof (ouibuf)) { 328 vendor = ouibuf; 329 } 330 } else if (ret != 0) { 331 vendor = NULL; 332 } 333 334 if (nvlist_lookup_string(nvl, LIBSFF_KEY_PART, 335 &part) != 0) { 336 part = NULL; 337 } 338 339 if (nvlist_lookup_string(nvl, LIBSFF_KEY_REVISION, 340 &rev) != 0) { 341 rev = NULL; 342 } 343 344 if (nvlist_lookup_string(nvl, LIBSFF_KEY_SERIAL, 345 &serial) != 0) { 346 serial = NULL; 347 } 348 } 349 } 350 351 if (transceiver_range_create(mod, port, 0, 0) != 0) { 352 nvlist_free(nvl); 353 return (-1); 354 } 355 356 if (transceiver_create_sff(mod, port, 0, dgt.dgt_usable, vendor, part, 357 rev, serial, NULL) != 0) { 358 nvlist_free(nvl); 359 return (-1); 360 } 361 362 nvlist_free(nvl); 363 return (0); 364 } 365 366 static boolean_t 367 nic_enum_link_ntrans(dladm_handle_t handle, datalink_id_t linkid, uint_t *ntran, 368 nic_port_type_t *pt) 369 { 370 dld_ioc_gettran_t dgt; 371 372 memset(&dgt, 0, sizeof (dgt)); 373 dgt.dgt_linkid = linkid; 374 dgt.dgt_tran_id = DLDIOC_GETTRAN_GETNTRAN; 375 376 if (ioctl(dladm_dld_fd(handle), DLDIOC_GETTRAN, &dgt) != 0) { 377 if (errno != ENOTSUP) { 378 return (B_FALSE); 379 } 380 *pt = NIC_PORT_UNKNOWN; 381 *ntran = 1; 382 } else { 383 *ntran = dgt.dgt_tran_id; 384 *pt = NIC_PORT_SFF; 385 } 386 387 return (B_TRUE); 388 } 389 390 static boolean_t 391 nic_enum_devinfo_linkid(dladm_handle_t handle, di_node_t din, 392 datalink_id_t *linkidp) 393 { 394 char dname[MAXNAMELEN]; 395 396 if (snprintf(dname, sizeof (dname), "%s%d", di_driver_name(din), 397 di_instance(din)) >= sizeof (dname)) { 398 return (B_FALSE); 399 } 400 401 if (dladm_dev2linkid(handle, dname, linkidp) != DLADM_STATUS_OK) 402 return (B_FALSE); 403 404 return (B_TRUE); 405 } 406 407 /* 408 * When we encounter a nexus driver we need to walk each of its children to 409 * actually get at the dladm handles and devices that we can use for this. 410 */ 411 static int 412 nic_enum_nexus(topo_mod_t *mod, tnode_t *pnode, dladm_handle_t handle, 413 di_node_t din) 414 { 415 uint_t total_ports = 0; 416 nic_port_type_t pt; 417 di_node_t child; 418 419 /* 420 * We have to iterate child nodes in two passes. The first pass is used 421 * to determine the number of children to create. FM requires that we 422 * create all the children nodes at once currently. 423 */ 424 for (child = di_child_node(din); child != DI_NODE_NIL; 425 child = di_sibling_node(child)) { 426 datalink_id_t linkid; 427 uint_t ntrans; 428 429 if (!nic_enum_devinfo_linkid(handle, child, &linkid)) 430 return (-1); 431 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt)) 432 return (-1); 433 434 total_ports += ntrans; 435 } 436 437 if (total_ports == 0) 438 return (0); 439 440 if (port_range_create(mod, pnode, 0, total_ports - 1) != 0) 441 return (-1); 442 443 total_ports = 0; 444 for (child = di_child_node(din); child != DI_NODE_NIL; 445 child = di_sibling_node(child)) { 446 datalink_id_t linkid; 447 uint_t i, ntrans; 448 449 if (!nic_enum_devinfo_linkid(handle, child, &linkid)) 450 return (-1); 451 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt)) 452 return (-1); 453 454 for (i = 0; i < ntrans; i++) { 455 if (nic_create_transceiver(mod, pnode, handle, linkid, 456 total_ports + i, i, pt) != 0) { 457 return (-1); 458 } 459 } 460 461 total_ports += ntrans; 462 } 463 464 return (0); 465 } 466 467 /* ARGSUSED */ 468 static int 469 nic_enum(topo_mod_t *mod, tnode_t *pnode, const char *name, 470 topo_instance_t min, topo_instance_t max, void *modarg, void *data) 471 { 472 di_node_t din = data; 473 datalink_id_t linkid; 474 dladm_handle_t handle; 475 uint_t ntrans, i; 476 nic_port_type_t pt; 477 const char *drv; 478 479 if (strcmp(name, NIC) != 0) { 480 topo_mod_dprintf(mod, "nic_enum: asked to enumerate unknown " 481 "component: %s\n", name); 482 return (-1); 483 } 484 485 if (din == NULL) { 486 topo_mod_dprintf(mod, "nic_enum: missing data argument\n"); 487 return (-1); 488 } 489 490 if ((handle = topo_mod_getspecific(mod)) == NULL) { 491 topo_mod_dprintf(mod, "nic_enum: failed to get nic module " 492 "specific data\n"); 493 return (-1); 494 } 495 496 /* 497 * No driver attached, just skip it. 498 */ 499 if ((drv = di_driver_name(din)) == NULL) { 500 return (0); 501 } 502 503 for (i = 0; nic_nexuses[i] != NULL; i++) { 504 if (strcmp(drv, nic_nexuses[i]) == 0) { 505 return (nic_enum_nexus(mod, pnode, handle, din)); 506 } 507 } 508 509 if (!nic_enum_devinfo_linkid(handle, din, &linkid)) 510 return (-1); 511 512 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt)) 513 return (-1); 514 515 if (ntrans == 0) 516 return (0); 517 518 if (port_range_create(mod, pnode, 0, ntrans - 1) != 0) 519 return (-1); 520 521 for (i = 0; i < ntrans; i++) { 522 if (nic_create_transceiver(mod, pnode, handle, linkid, i, i, 523 pt) != 0) { 524 return (-1); 525 } 526 } 527 528 return (0); 529 } 530 531 static const topo_modops_t nic_ops = { 532 nic_enum, NULL 533 }; 534 535 static topo_modinfo_t nic_mod = { 536 NIC, FM_FMRI_SCHEME_HC, NIC_VERSION, &nic_ops 537 }; 538 539 int 540 _topo_init(topo_mod_t *mod, topo_version_t version) 541 { 542 dladm_handle_t handle; 543 544 if (getenv("TOPONICDEBUG") != NULL) 545 topo_mod_setdebug(mod); 546 547 topo_mod_dprintf(mod, "_mod_init: " 548 "initializing %s enumerator\n", NIC); 549 550 if (version != NIC_VERSION) { 551 return (-1); 552 } 553 554 if (dladm_open(&handle) != 0) 555 return (-1); 556 557 if (topo_mod_register(mod, &nic_mod, TOPO_VERSION) != 0) { 558 dladm_close(handle); 559 return (-1); 560 } 561 562 topo_mod_setspecific(mod, handle); 563 564 return (0); 565 } 566 567 void 568 _topo_fini(topo_mod_t *mod) 569 { 570 dladm_handle_t handle; 571 572 if ((handle = topo_mod_getspecific(mod)) == NULL) 573 return; 574 575 dladm_close(handle); 576 topo_mod_setspecific(mod, NULL); 577 } 578