1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2026 Oxide Computer Company
14 */
15
16 /*
17 * Implement logic related to the state of PCIe links. These operate on a PCIe
18 * bridge (e.g. a root port) by way of the private ioctls implemented by the
19 * 'pcieb' driver.
20 */
21
22 #include <err.h>
23 #include <errno.h>
24 #include <fcntl.h>
25 #include <inttypes.h>
26 #include <limits.h>
27 #include <ofmt.h>
28 #include <stdbool.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <time.h>
33 #include <strings.h>
34 #include <unistd.h>
35 #include <sys/stat.h>
36 #include <sys/time.h>
37 #include <sys/types.h>
38 #include <sys/sysmacros.h>
39 #include <sys/debug.h>
40
41 #include <pcieb_ioctl.h>
42
43 #include "pcieadm.h"
44
45 /*
46 * Go through and initialize the common logic for most of the link related
47 * commands that need to use the pcieb ioctls. This includes:
48 *
49 * - Verifying it makes sense
50 * - Setting privileges
51 * - Opening the actual devctl file
52 * - Finding the dip
53 */
54 static int
pcieadm_link_pcieb_open(pcieadm_t * pcip,const char * device,bool write)55 pcieadm_link_pcieb_open(pcieadm_t *pcip, const char *device, bool write)
56 {
57 const char *drv;
58
59 /*
60 * We retain FILE_DAC_READ and FILE_DAC_SEARCH to find and open the
61 * device file and SYS_DEVICES for the ioctls themselves. To open it for
62 * writing (some bridge ioctls, such as retrain, mutate link state) we
63 * also need FILE_DAC_WRITE to override the node's permissions, and
64 * FILE_WRITE because, unlike FILE_READ, it is stripped from our minimal
65 * privilege set.
66 */
67 VERIFY0(priv_addset(pcip->pia_priv_eff, PRIV_SYS_DEVICES));
68 VERIFY0(priv_addset(pcip->pia_priv_eff, PRIV_FILE_DAC_READ));
69 VERIFY0(priv_addset(pcip->pia_priv_eff, PRIV_FILE_DAC_SEARCH));
70 if (write) {
71 VERIFY0(priv_addset(pcip->pia_priv_eff, PRIV_FILE_WRITE));
72 VERIFY0(priv_addset(pcip->pia_priv_eff, PRIV_FILE_DAC_WRITE));
73 }
74 pcieadm_init_privs(pcip);
75
76 pcieadm_find_dip(pcip, device);
77 drv = di_driver_name(pcip->pia_devi);
78 if (drv == NULL || strcmp(drv, "pcieb") != 0) {
79 errx(EXIT_FAILURE, "device %s is not a PCIe bridge: found "
80 "driver %s, but expected pcieb", device,
81 drv == NULL ? "<none>" : drv);
82 }
83
84 for (di_minor_t m = di_minor_next(pcip->pia_devi, DI_MINOR_NIL);
85 m != NULL; m = di_minor_next(pcip->pia_devi, m)) {
86 if (strcmp(di_minor_name(m), "devctl") == 0) {
87 char buf[PATH_MAX], *mp;
88 int fd;
89
90 mp = di_devfs_minor_path(m);
91 if (mp == NULL) {
92 err(EXIT_FAILURE, "failed to get devfs path "
93 "for %s devctl minor", device);
94 }
95
96 if (snprintf(buf, sizeof (buf), "/devices%s", mp) >=
97 sizeof (buf)) {
98 errx(EXIT_FAILURE, "failed to construct devfs "
99 "minor path for %s devctl minor: internal "
100 "path buffer would have overflowed",
101 device);
102 }
103 di_devfs_path_free(mp);
104
105 if (setppriv(PRIV_SET, PRIV_EFFECTIVE,
106 pcip->pia_priv_eff) != 0) {
107 err(EXIT_FAILURE, "failed to raise privileges");
108 }
109
110 fd = open(buf, write ? O_RDWR : O_RDONLY);
111 if (fd < 0) {
112 err(EXIT_FAILURE,
113 "failed to open %s devctl minor %s",
114 device, buf);
115 }
116
117 if (setppriv(PRIV_SET, PRIV_EFFECTIVE,
118 pcip->pia_priv_min) != 0) {
119 err(EXIT_FAILURE,
120 "failed to reduce privileges");
121 }
122
123 return (fd);
124 }
125 }
126
127 errx(EXIT_FAILURE, "failed to find devctl minor for %s", device);
128 }
129
130 static void
pcieadm_link_limit_usage(FILE * f)131 pcieadm_link_limit_usage(FILE *f)
132 {
133 (void) fprintf(f, "\tlink limit\t[-s speed] -d device\n");
134 }
135
136 static void
pcieadm_link_limit_help(const char * fmt,...)137 pcieadm_link_limit_help(const char *fmt, ...)
138 {
139 if (fmt != NULL) {
140 va_list ap;
141
142 va_start(ap, fmt);
143 vwarnx(fmt, ap);
144 va_end(ap);
145 (void) fprintf(stderr, "\n");
146 }
147
148 (void) fprintf(stderr, "Usage: %s link limit [-s speed] -d device\n",
149 pcieadm_progname);
150 (void) fprintf(stderr, "Print or set the administrative limit on the "
151 "corresponding PCIe link\n\n"
152 "\t-d device\tthe PCIe bridge to operate on (driver instance,"
153 "\n\t\t\t/devices path, or b/d/f)\n"
154 "\t-s speed\tlimit the device to the specified PCIe gen/speed "
155 "(e.g.\n\t\t\t2.5, 32, gen2, gen3, etc.)\n");
156 }
157
158 typedef struct {
159 uint32_t pls_speed;
160 const char *pls_gts;
161 const char *pls_gen;
162 const char *pls_alt;
163 } pcieadm_link_speed_t;
164
165 static const pcieadm_link_speed_t pcieadm_link_speeds[] = {
166 { PCIEB_LINK_SPEED_GEN1, "2.5", "gen1", NULL },
167 { PCIEB_LINK_SPEED_GEN2, "5.0", "gen2", "5" },
168 { PCIEB_LINK_SPEED_GEN3, "8.0", "gen3", "8" },
169 { PCIEB_LINK_SPEED_GEN4, "16.0", "gen4", "16" },
170 { PCIEB_LINK_SPEED_GEN5, "32.0", "gen5", "32" },
171 { PCIEB_LINK_SPEED_GEN6, "64.0", "gen6", "64" }
172 };
173
174 static uint32_t
pcieadm_parse_pcieb_speed(const char * str)175 pcieadm_parse_pcieb_speed(const char *str)
176 {
177 for (uint_t i = 0; i < ARRAY_SIZE(pcieadm_link_speeds); i++) {
178 const pcieadm_link_speed_t *pls = &pcieadm_link_speeds[i];
179
180 if (strcasecmp(str, pls->pls_gts) == 0 ||
181 strcasecmp(str, pls->pls_gen) == 0 ||
182 (pls->pls_alt != NULL &&
183 strcasecmp(str, pls->pls_alt) == 0)) {
184 return (pls->pls_speed);
185 }
186 }
187
188 errx(EXIT_FAILURE, "failed to parse speed: %s", str);
189 }
190
191 static const pcieadm_link_speed_t *
pcieadm_link_speed_lookup(uint32_t speed)192 pcieadm_link_speed_lookup(uint32_t speed)
193 {
194 for (uint_t i = 0; i < ARRAY_SIZE(pcieadm_link_speeds); i++) {
195 if (pcieadm_link_speeds[i].pls_speed == speed)
196 return (&pcieadm_link_speeds[i]);
197 }
198
199 return (NULL);
200 }
201
202 static int
pcieadm_link_limit(pcieadm_t * pcip,int argc,char * argv[])203 pcieadm_link_limit(pcieadm_t *pcip, int argc, char *argv[])
204 {
205 int c, ret = EXIT_SUCCESS;
206 const char *device = NULL, *speed = NULL;
207 pcieb_ioctl_target_speed_t pits;
208
209 while ((c = getopt(argc, argv, ":d:s:")) != -1) {
210 switch (c) {
211 case 'd':
212 device = optarg;
213 break;
214 case 's':
215 speed = optarg;
216 break;
217 case ':':
218 pcieadm_link_limit_help("Option -%c requires an "
219 "argument", optopt);
220 exit(EXIT_USAGE);
221 case '?':
222 default:
223 pcieadm_link_limit_help("unknown option: -%c",
224 optopt);
225 exit(EXIT_USAGE);
226 }
227
228 }
229
230 if (device == NULL) {
231 pcieadm_link_limit_help("missing required device argument "
232 "(-d)");
233 exit(EXIT_USAGE);
234 }
235
236 argc -= optind;
237 argv += optind;
238 if (argc != 0) {
239 errx(EXIT_USAGE, "encountered extraneous arguments starting "
240 "with %s", argv[0]);
241 }
242
243 int fd = pcieadm_link_pcieb_open(pcip, device, speed != NULL);
244 bzero(&pits, sizeof (pits));
245
246 if (speed != NULL) {
247 pits.pits_speed = pcieadm_parse_pcieb_speed(speed);
248
249 if (setppriv(PRIV_SET, PRIV_EFFECTIVE, pcip->pia_priv_eff) != 0)
250 err(EXIT_FAILURE, "failed to raise privileges");
251
252 if (ioctl(fd, PCIEB_IOCTL_SET_TARGET_SPEED, &pits) != 0) {
253 err(EXIT_FAILURE, "failed to set %s target speed",
254 device);
255 }
256
257 if (setppriv(PRIV_SET, PRIV_EFFECTIVE, pcip->pia_priv_min) != 0)
258 err(EXIT_FAILURE, "failed to reduce privileges");
259 } else {
260 const pcieadm_link_speed_t *pls;
261
262 if (setppriv(PRIV_SET, PRIV_EFFECTIVE, pcip->pia_priv_eff) != 0)
263 err(EXIT_FAILURE, "failed to raise privileges");
264
265 if (ioctl(fd, PCIEB_IOCTL_GET_TARGET_SPEED, &pits) != 0) {
266 err(EXIT_FAILURE, "failed to get %s target speed",
267 device);
268 }
269
270 if (setppriv(PRIV_SET, PRIV_EFFECTIVE, pcip->pia_priv_min) != 0)
271 err(EXIT_FAILURE, "failed to reduce privileges");
272
273 pls = pcieadm_link_speed_lookup(pits.pits_speed);
274 if (pls != NULL) {
275 (void) printf("Target speed: %s GT/s (%s)\n",
276 pls->pls_gts, pls->pls_gen);
277 } else {
278 (void) printf("Target speed: unknown speed value: "
279 "0x%x\n", pits.pits_speed);
280 }
281
282 if ((pits.pits_flags & ~PCIEB_FLAGS_ADMIN_SET) != 0) {
283 (void) printf("Unknown flags: 0x%x\n", pits.pits_flags);
284 } else if ((pits.pits_flags & PCIEB_FLAGS_ADMIN_SET) != 0) {
285 (void) printf("Flags: Admin Set Speed\n");
286 }
287 }
288
289 VERIFY0(close(fd));
290 return (ret);
291 }
292
293 static void
pcieadm_link_retrain_usage(FILE * f)294 pcieadm_link_retrain_usage(FILE *f)
295 {
296 (void) fprintf(f, "\tlink retrain\t-d device\n");
297 }
298
299 static void
pcieadm_link_retrain_help(const char * fmt,...)300 pcieadm_link_retrain_help(const char *fmt, ...)
301 {
302 if (fmt != NULL) {
303 va_list ap;
304
305 va_start(ap, fmt);
306 vwarnx(fmt, ap);
307 va_end(ap);
308 (void) fprintf(stderr, "\n");
309 }
310
311 (void) fprintf(stderr, "Usage: %s link retrain -d device\n",
312 pcieadm_progname);
313 (void) fprintf(stderr,
314 "Retrain the link on the specified upstream port by requesting\n"
315 "it via the PCIe Link Control register.\n\n"
316 "\t-d device\tthe PCIe bridge to operate on (driver instance,\n"
317 "\t\t\t/devices path, or b/d/f)\n");
318 }
319
320 static int
pcieadm_link_retrain(pcieadm_t * pcip,int argc,char * argv[])321 pcieadm_link_retrain(pcieadm_t *pcip, int argc, char *argv[])
322 {
323 int c, ret = EXIT_SUCCESS;
324 const char *device = NULL;
325
326 while ((c = getopt(argc, argv, ":d:")) != -1) {
327 switch (c) {
328 case 'd':
329 device = optarg;
330 break;
331 case ':':
332 pcieadm_link_retrain_help("Option -%c requires an "
333 "argument", optopt);
334 exit(EXIT_USAGE);
335 case '?':
336 default:
337 pcieadm_link_retrain_help("unknown option: -%c",
338 optopt);
339 exit(EXIT_USAGE);
340 }
341 }
342
343 if (device == NULL) {
344 pcieadm_link_retrain_help("missing required device argument "
345 "(-d)");
346 exit(EXIT_USAGE);
347 }
348
349 argc -= optind;
350 argv += optind;
351 if (argc != 0) {
352 errx(EXIT_USAGE, "encountered extraneous arguments starting "
353 "with %s", argv[0]);
354 }
355
356 int fd = pcieadm_link_pcieb_open(pcip, device, true);
357
358 if (setppriv(PRIV_SET, PRIV_EFFECTIVE, pcip->pia_priv_eff) != 0)
359 err(EXIT_FAILURE, "failed to raise privileges");
360
361 if (ioctl(fd, PCIEB_IOCTL_RETRAIN) != 0) {
362 err(EXIT_FAILURE, "failed to retrain link %s", device);
363 }
364
365 if (setppriv(PRIV_SET, PRIV_EFFECTIVE, pcip->pia_priv_min) != 0)
366 err(EXIT_FAILURE, "failed to reduce privileges");
367
368 VERIFY0(close(fd));
369 return (ret);
370 }
371
372 static const pcieadm_cmdtab_t pcieadm_cmds_link[] = {
373 { "limit", pcieadm_link_limit, pcieadm_link_limit_usage },
374 { "retrain", pcieadm_link_retrain, pcieadm_link_retrain_usage },
375 { NULL }
376 };
377
378 int
pcieadm_link(pcieadm_t * pcip,int argc,char * argv[])379 pcieadm_link(pcieadm_t *pcip, int argc, char *argv[])
380 {
381 return (pcieadm_walk_tab(pcip, pcieadm_cmds_link, argc, argv));
382 }
383
384 void
pcieadm_link_usage(FILE * f)385 pcieadm_link_usage(FILE *f)
386 {
387 pcieadm_walk_usage(pcieadm_cmds_link, f);
388 }
389