1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright (c) 2017, Joyent, Inc.
14 * Copyright 2020 Robert Mustacchi
15 * Copyright 2023 Oxide Computer Company
16 */
17
18 /*
19 * This module covers enumerating properties of physical NICs. At this time, as
20 * various devices are discovered that may relate to various networking gear, we
21 * will attempt to enumerate ports and transceivers under them, if requested.
22 */
23
24 #include <strings.h>
25 #include <libdevinfo.h>
26 #include <libdladm.h>
27 #include <libdllink.h>
28 #include <libdlstat.h>
29 #include <libsff.h>
30 #include <unistd.h>
31 #include <sys/dld_ioc.h>
32 #include <sys/dld.h>
33 #include <sys/mac.h>
34
35 #include <sys/fm/protocol.h>
36 #include <fm/topo_mod.h>
37 #include <fm/topo_list.h>
38 #include <fm/topo_method.h>
39
40 #include <topo_port.h>
41 #include <topo_transceiver.h>
42
43 #include "topo_nic.h"
44
45 typedef enum {
46 NIC_PORT_UNKNOWN,
47 NIC_PORT_SFF
48 } nic_port_type_t;
49
50 static const topo_pgroup_info_t datalink_pgroup = {
51 TOPO_PGROUP_DATALINK,
52 TOPO_STABILITY_PRIVATE,
53 TOPO_STABILITY_PRIVATE,
54 1
55 };
56
57 typedef struct nic_port_mac {
58 char npm_mac[ETHERADDRSTRL];
59 boolean_t npm_valid;
60 topo_mod_t *npm_mod;
61 } nic_port_mac_t;
62
63 /*
64 * The following drivers have their main function be a nexus driver which
65 * enumerates children itself which are mac providers rather than having the
66 * main PCI functions actually be the device nodes. As such, when we encounter
67 * them, we need to enumerate them in a slightly different way by walking over
68 * each child of the instance.
69 */
70 static const char *nic_nexuses[] = {
71 "t4nex",
72 NULL
73 };
74
75 /*
76 * The first MAC address is always the primary MAC address, so we only worry
77 * about the first. Thus this function always returns B_FALSE, to terminate
78 * iteration.
79 */
80 static boolean_t
nic_port_datalink_mac_cb(void * arg,dladm_macaddr_attr_t * attr)81 nic_port_datalink_mac_cb(void *arg, dladm_macaddr_attr_t *attr)
82 {
83 nic_port_mac_t *mac = arg;
84
85 if (attr->ma_addrlen != ETHERADDRL) {
86 topo_mod_dprintf(mac->npm_mod,
87 "found address with bad length: %u\n", attr->ma_addrlen);
88 return (B_FALSE);
89 }
90
91 (void) snprintf(mac->npm_mac, sizeof (mac->npm_mac),
92 "%02x:%02x:%02x:%02x:%02x:%02x",
93 attr->ma_addr[0], attr->ma_addr[1], attr->ma_addr[2],
94 attr->ma_addr[3], attr->ma_addr[4], attr->ma_addr[5]);
95 mac->npm_valid = B_TRUE;
96 return (B_FALSE);
97 }
98
99 static int
nic_port_datalink_props(topo_mod_t * mod,tnode_t * port,dladm_handle_t handle,datalink_id_t linkid)100 nic_port_datalink_props(topo_mod_t *mod, tnode_t *port, dladm_handle_t handle,
101 datalink_id_t linkid)
102 {
103 int err;
104 dladm_status_t status;
105 uint64_t ifspeed;
106 link_duplex_t duplex;
107 link_state_t state;
108 const char *duplex_str, *state_str, *media_str;
109 datalink_class_t dlclass;
110 uint32_t media;
111 char dlname[MAXLINKNAMELEN * 2];
112 char dlerr[DLADM_STRSIZE], dlmedia[DLADM_PROP_VAL_MAX], *valptr[1];
113 uint_t valcnt = 1;
114 nic_port_mac_t mac;
115
116 status = dladm_datalink_id2info(handle, linkid, NULL, &dlclass, &media,
117 dlname, sizeof (dlname));
118 if (status != DLADM_STATUS_OK) {
119 topo_mod_dprintf(mod, "failed to get link info: %s\n",
120 dladm_status2str(status, dlerr));
121 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM));
122 }
123
124 if (dlclass != DATALINK_CLASS_PHYS) {
125 return (0);
126 }
127
128 status = dladm_get_single_mac_stat(handle, linkid, "ifspeed",
129 KSTAT_DATA_UINT64, &ifspeed);
130 if (status != DLADM_STATUS_OK) {
131 topo_mod_dprintf(mod, "failed to get ifspeed: %s\n",
132 dladm_status2str(status, dlerr));
133 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM));
134 }
135
136 status = dladm_get_single_mac_stat(handle, linkid, "link_duplex",
137 KSTAT_DATA_UINT32, &duplex);
138 if (status != DLADM_STATUS_OK) {
139 topo_mod_dprintf(mod, "failed to get link_duplex: %s\n",
140 dladm_status2str(status, dlerr));
141 return (topo_mod_seterrno(mod, EMOD_UKNOWN_ENUM));
142 }
143
144 switch (duplex) {
145 case LINK_DUPLEX_HALF:
146 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_HALF;
147 break;
148 case LINK_DUPLEX_FULL:
149 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_FULL;
150 break;
151 default:
152 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_UNKNOWN;
153 break;
154 }
155
156 status = dladm_get_single_mac_stat(handle, linkid, "link_state",
157 KSTAT_DATA_UINT32, &state);
158 if (status != DLADM_STATUS_OK) {
159 topo_mod_dprintf(mod, "failed to get link_duplex: %s\n",
160 dladm_status2str(status, dlerr));
161 return (topo_mod_seterrno(mod, status));
162 }
163
164 switch (state) {
165 case LINK_STATE_UP:
166 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_UP;
167 break;
168 case LINK_STATE_DOWN:
169 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_DOWN;
170 break;
171 default:
172 state_str = TOPO_PGROUP_DATALINK_LINK_STATUS_UNKNOWN;
173 break;
174 }
175
176 /*
177 * Override the duplex if the link is down. Some devices will leave it
178 * set at half as opposed to unknown.
179 */
180 if (state == LINK_STATE_DOWN || state == LINK_STATE_UNKNOWN) {
181 duplex_str = TOPO_PGROUP_DATALINK_LINK_DUPLEX_UNKNOWN;
182 }
183
184 media_str = NULL;
185 if (state == LINK_STATE_UP) {
186 valptr[0] = dlmedia;
187 if (dladm_get_linkprop(handle, linkid, DLADM_PROP_VAL_CURRENT,
188 "media", valptr, &valcnt) == DLADM_STATUS_OK) {
189 media_str = dlmedia;
190 }
191 }
192
193 mac.npm_mac[0] = '\0';
194 mac.npm_valid = B_FALSE;
195 mac.npm_mod = mod;
196 if (media == DL_ETHER) {
197 (void) dladm_walk_macaddr(handle, linkid, &mac,
198 nic_port_datalink_mac_cb);
199 }
200
201 if (topo_pgroup_create(port, &datalink_pgroup, &err) != 0) {
202 topo_mod_dprintf(mod, "falied to create property group %s: "
203 "%s\n", TOPO_PGROUP_DATALINK, topo_strerror(err));
204 return (topo_mod_seterrno(mod, err));
205 }
206
207 if (topo_prop_set_uint64(port, TOPO_PGROUP_DATALINK,
208 TOPO_PGROUP_DATALINK_LINK_SPEED, TOPO_PROP_IMMUTABLE, ifspeed,
209 &err) != 0) {
210 topo_mod_dprintf(mod, "failed to set %s property: %s\n",
211 TOPO_PGROUP_DATALINK_LINK_SPEED, topo_strerror(err));
212 return (topo_mod_seterrno(mod, err));
213 }
214
215 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK,
216 TOPO_PGROUP_DATALINK_LINK_DUPLEX, TOPO_PROP_IMMUTABLE, duplex_str,
217 &err) != 0) {
218 topo_mod_dprintf(mod, "failed to set %s property: %s\n",
219 TOPO_PGROUP_DATALINK_LINK_DUPLEX, topo_strerror(err));
220 return (topo_mod_seterrno(mod, err));
221 }
222
223 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK,
224 TOPO_PGROUP_DATALINK_LINK_STATUS, TOPO_PROP_IMMUTABLE, state_str,
225 &err) != 0) {
226 topo_mod_dprintf(mod, "failed to set %s property: %s\n",
227 TOPO_PGROUP_DATALINK_LINK_STATUS, topo_strerror(err));
228 return (topo_mod_seterrno(mod, err));
229 }
230
231 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK,
232 TOPO_PGROUP_DATALINK_LINK_NAME, TOPO_PROP_IMMUTABLE, dlname,
233 &err) != 0) {
234 topo_mod_dprintf(mod, "failed to set %s propery: %s\n",
235 TOPO_PGROUP_DATALINK_LINK_NAME, topo_strerror(err));
236 return (topo_mod_seterrno(mod, err));
237 }
238
239 if (media_str != NULL && topo_prop_set_string(port,
240 TOPO_PGROUP_DATALINK, TOPO_PGROUP_DATALINK_LINK_MEDIA,
241 TOPO_PROP_IMMUTABLE, media_str, &err) != 0) {
242 topo_mod_dprintf(mod, "failed to set %s propery: %s\n",
243 TOPO_PGROUP_DATALINK_LINK_MEDIA, topo_strerror(err));
244 return (topo_mod_seterrno(mod, err));
245 }
246
247 if (mac.npm_valid) {
248 if (topo_prop_set_string(port, TOPO_PGROUP_DATALINK,
249 TOPO_PGROUP_DATALINK_PMAC, TOPO_PROP_IMMUTABLE,
250 mac.npm_mac, &err) != 0) {
251 topo_mod_dprintf(mod, "failed to set %s propery: %s\n",
252 TOPO_PGROUP_DATALINK_PMAC, topo_strerror(err));
253 return (topo_mod_seterrno(mod, err));
254 }
255 }
256
257
258 return (0);
259 }
260
261 /*
262 * Create an instance of a transceiver with the specified id. We must create
263 * both its port and the transceiver node.
264 */
265 static int
nic_create_transceiver(topo_mod_t * mod,tnode_t * pnode,dladm_handle_t handle,datalink_id_t linkid,topo_instance_t inst,uint_t tranid,nic_port_type_t port_type)266 nic_create_transceiver(topo_mod_t *mod, tnode_t *pnode, dladm_handle_t handle,
267 datalink_id_t linkid, topo_instance_t inst, uint_t tranid,
268 nic_port_type_t port_type)
269 {
270 int ret;
271 tnode_t *port;
272 dld_ioc_gettran_t dgt;
273 dld_ioc_tranio_t dti;
274 uint8_t buf[256];
275 char ouibuf[16];
276 char *vendor = NULL, *part = NULL, *rev = NULL, *serial = NULL;
277 nvlist_t *nvl = NULL;
278
279 switch (port_type) {
280 case NIC_PORT_UNKNOWN:
281 ret = port_create_unknown(mod, pnode, inst, &port);
282 break;
283 case NIC_PORT_SFF:
284 ret = port_create_sff(mod, pnode, inst, &port);
285 break;
286 default:
287 return (-1);
288 }
289
290 if ((ret = nic_port_datalink_props(mod, port, handle, linkid)) != 0)
291 return (ret);
292
293 if (port_type != NIC_PORT_SFF)
294 return (0);
295
296 bzero(&dgt, sizeof (dgt));
297 dgt.dgt_linkid = linkid;
298 dgt.dgt_tran_id = tranid;
299
300 if (ioctl(dladm_dld_fd(handle), DLDIOC_GETTRAN, &dgt) != 0) {
301 if (errno == ENOTSUP)
302 return (0);
303 return (-1);
304 }
305
306 if (dgt.dgt_present == 0)
307 return (0);
308
309 bzero(&dti, sizeof (dti));
310 dti.dti_linkid = linkid;
311 dti.dti_tran_id = tranid;
312 dti.dti_page = 0xa0;
313 dti.dti_nbytes = sizeof (buf);
314 dti.dti_buf = (uintptr_t)buf;
315
316 if (ioctl(dladm_dld_fd(handle), DLDIOC_READTRAN, &dti) == 0) {
317 uchar_t *oui;
318 uint_t nbyte;
319
320 if (libsff_parse(buf, dti.dti_nbytes, dti.dti_page,
321 &nvl) == 0) {
322 if ((ret = nvlist_lookup_string(nvl, LIBSFF_KEY_VENDOR,
323 &vendor)) != 0 && nvlist_lookup_byte_array(nvl,
324 LIBSFF_KEY_OUI, &oui, &nbyte) == 0 && nbyte == 3) {
325 if (snprintf(ouibuf, sizeof (ouibuf),
326 "%02x:%02x:%02x", oui[0], oui[1], oui[2]) <
327 sizeof (ouibuf)) {
328 vendor = ouibuf;
329 }
330 } else if (ret != 0) {
331 vendor = NULL;
332 }
333
334 if (nvlist_lookup_string(nvl, LIBSFF_KEY_PART,
335 &part) != 0) {
336 part = NULL;
337 }
338
339 if (nvlist_lookup_string(nvl, LIBSFF_KEY_REVISION,
340 &rev) != 0) {
341 rev = NULL;
342 }
343
344 if (nvlist_lookup_string(nvl, LIBSFF_KEY_SERIAL,
345 &serial) != 0) {
346 serial = NULL;
347 }
348 }
349 }
350
351 if (transceiver_range_create(mod, port, 0, 0) != 0) {
352 nvlist_free(nvl);
353 return (-1);
354 }
355
356 if (transceiver_create_sff(mod, port, 0, dgt.dgt_usable, vendor, part,
357 rev, serial, NULL) != 0) {
358 nvlist_free(nvl);
359 return (-1);
360 }
361
362 nvlist_free(nvl);
363 return (0);
364 }
365
366 static boolean_t
nic_enum_link_ntrans(dladm_handle_t handle,datalink_id_t linkid,uint_t * ntran,nic_port_type_t * pt)367 nic_enum_link_ntrans(dladm_handle_t handle, datalink_id_t linkid, uint_t *ntran,
368 nic_port_type_t *pt)
369 {
370 dld_ioc_gettran_t dgt;
371
372 memset(&dgt, 0, sizeof (dgt));
373 dgt.dgt_linkid = linkid;
374 dgt.dgt_tran_id = DLDIOC_GETTRAN_GETNTRAN;
375
376 if (ioctl(dladm_dld_fd(handle), DLDIOC_GETTRAN, &dgt) != 0) {
377 if (errno != ENOTSUP) {
378 return (B_FALSE);
379 }
380 *pt = NIC_PORT_UNKNOWN;
381 *ntran = 1;
382 } else {
383 *ntran = dgt.dgt_tran_id;
384 *pt = NIC_PORT_SFF;
385 }
386
387 return (B_TRUE);
388 }
389
390 static boolean_t
nic_enum_devinfo_linkid(dladm_handle_t handle,di_node_t din,datalink_id_t * linkidp)391 nic_enum_devinfo_linkid(dladm_handle_t handle, di_node_t din,
392 datalink_id_t *linkidp)
393 {
394 char dname[MAXNAMELEN];
395
396 if (snprintf(dname, sizeof (dname), "%s%d", di_driver_name(din),
397 di_instance(din)) >= sizeof (dname)) {
398 return (B_FALSE);
399 }
400
401 if (dladm_dev2linkid(handle, dname, linkidp) != DLADM_STATUS_OK)
402 return (B_FALSE);
403
404 return (B_TRUE);
405 }
406
407 /*
408 * When we encounter a nexus driver we need to walk each of its children to
409 * actually get at the dladm handles and devices that we can use for this.
410 */
411 static int
nic_enum_nexus(topo_mod_t * mod,tnode_t * pnode,dladm_handle_t handle,di_node_t din)412 nic_enum_nexus(topo_mod_t *mod, tnode_t *pnode, dladm_handle_t handle,
413 di_node_t din)
414 {
415 uint_t total_ports = 0;
416 nic_port_type_t pt;
417 di_node_t child;
418
419 /*
420 * We have to iterate child nodes in two passes. The first pass is used
421 * to determine the number of children to create. FM requires that we
422 * create all the children nodes at once currently.
423 */
424 for (child = di_child_node(din); child != DI_NODE_NIL;
425 child = di_sibling_node(child)) {
426 datalink_id_t linkid;
427 uint_t ntrans;
428
429 if (!nic_enum_devinfo_linkid(handle, child, &linkid))
430 return (-1);
431 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt))
432 return (-1);
433
434 total_ports += ntrans;
435 }
436
437 if (total_ports == 0)
438 return (0);
439
440 if (port_range_create(mod, pnode, 0, total_ports - 1) != 0)
441 return (-1);
442
443 total_ports = 0;
444 for (child = di_child_node(din); child != DI_NODE_NIL;
445 child = di_sibling_node(child)) {
446 datalink_id_t linkid;
447 uint_t i, ntrans;
448
449 if (!nic_enum_devinfo_linkid(handle, child, &linkid))
450 return (-1);
451 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt))
452 return (-1);
453
454 for (i = 0; i < ntrans; i++) {
455 if (nic_create_transceiver(mod, pnode, handle, linkid,
456 total_ports + i, i, pt) != 0) {
457 return (-1);
458 }
459 }
460
461 total_ports += ntrans;
462 }
463
464 return (0);
465 }
466
467 /* ARGSUSED */
468 static int
nic_enum(topo_mod_t * mod,tnode_t * pnode,const char * name,topo_instance_t min,topo_instance_t max,void * modarg,void * data)469 nic_enum(topo_mod_t *mod, tnode_t *pnode, const char *name,
470 topo_instance_t min, topo_instance_t max, void *modarg, void *data)
471 {
472 di_node_t din = data;
473 datalink_id_t linkid;
474 dladm_handle_t handle;
475 uint_t ntrans, i;
476 nic_port_type_t pt;
477 const char *drv;
478
479 if (strcmp(name, NIC) != 0) {
480 topo_mod_dprintf(mod, "nic_enum: asked to enumerate unknown "
481 "component: %s\n", name);
482 return (-1);
483 }
484
485 if (din == NULL) {
486 topo_mod_dprintf(mod, "nic_enum: missing data argument\n");
487 return (-1);
488 }
489
490 if ((handle = topo_mod_getspecific(mod)) == NULL) {
491 topo_mod_dprintf(mod, "nic_enum: failed to get nic module "
492 "specific data\n");
493 return (-1);
494 }
495
496 /*
497 * No driver attached, just skip it.
498 */
499 if ((drv = di_driver_name(din)) == NULL) {
500 return (0);
501 }
502
503 for (i = 0; nic_nexuses[i] != NULL; i++) {
504 if (strcmp(drv, nic_nexuses[i]) == 0) {
505 return (nic_enum_nexus(mod, pnode, handle, din));
506 }
507 }
508
509 if (!nic_enum_devinfo_linkid(handle, din, &linkid))
510 return (-1);
511
512 if (!nic_enum_link_ntrans(handle, linkid, &ntrans, &pt))
513 return (-1);
514
515 if (ntrans == 0)
516 return (0);
517
518 if (port_range_create(mod, pnode, 0, ntrans - 1) != 0)
519 return (-1);
520
521 for (i = 0; i < ntrans; i++) {
522 if (nic_create_transceiver(mod, pnode, handle, linkid, i, i,
523 pt) != 0) {
524 return (-1);
525 }
526 }
527
528 return (0);
529 }
530
531 static const topo_modops_t nic_ops = {
532 nic_enum, NULL
533 };
534
535 static topo_modinfo_t nic_mod = {
536 NIC, FM_FMRI_SCHEME_HC, NIC_VERSION, &nic_ops
537 };
538
539 int
_topo_init(topo_mod_t * mod,topo_version_t version)540 _topo_init(topo_mod_t *mod, topo_version_t version)
541 {
542 dladm_handle_t handle;
543
544 if (getenv("TOPONICDEBUG") != NULL)
545 topo_mod_setdebug(mod);
546
547 topo_mod_dprintf(mod, "_mod_init: "
548 "initializing %s enumerator\n", NIC);
549
550 if (version != NIC_VERSION) {
551 return (-1);
552 }
553
554 if (dladm_open(&handle) != 0)
555 return (-1);
556
557 if (topo_mod_register(mod, &nic_mod, TOPO_VERSION) != 0) {
558 dladm_close(handle);
559 return (-1);
560 }
561
562 topo_mod_setspecific(mod, handle);
563
564 return (0);
565 }
566
567 void
_topo_fini(topo_mod_t * mod)568 _topo_fini(topo_mod_t *mod)
569 {
570 dladm_handle_t handle;
571
572 if ((handle = topo_mod_getspecific(mod)) == NULL)
573 return;
574
575 dladm_close(handle);
576 topo_mod_setspecific(mod, NULL);
577 }
578