1 /*- 2 * Copyright (c) 2013-2015 Sandvine Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/param.h> 28 #include <sys/iov.h> 29 #include <sys/dnv.h> 30 #include <sys/nv.h> 31 32 #include <err.h> 33 #include <errno.h> 34 #include <fcntl.h> 35 #include <regex.h> 36 #include <stdio.h> 37 #include <stdlib.h> 38 #include <string.h> 39 #include <unistd.h> 40 41 #include "iovctl.h" 42 43 static void config_action(const char *filename, int dryrun); 44 static void delete_action(const char *device, int dryrun); 45 static void print_schema(const char *device); 46 static void print_status(const char *device); 47 48 /* 49 * Fetch the config schema from the kernel via ioctl. This function has to 50 * call the ioctl twice: the first returns the amount of memory that we need 51 * to allocate for the schema, and the second actually fetches the schema. 52 */ 53 static nvlist_t * 54 get_schema(int fd) 55 { 56 struct pci_iov_schema arg; 57 nvlist_t *schema; 58 int error; 59 60 /* Do the ioctl() once to fetch the size of the schema. */ 61 arg.schema = NULL; 62 arg.len = 0; 63 arg.error = 0; 64 error = ioctl(fd, IOV_GET_SCHEMA, &arg); 65 if (error != 0) 66 err(1, "Could not fetch size of config schema"); 67 68 arg.schema = malloc(arg.len); 69 if (arg.schema == NULL) 70 err(1, "Could not allocate %zu bytes for schema", 71 arg.len); 72 73 /* Now do the ioctl() for real to get the schema. */ 74 error = ioctl(fd, IOV_GET_SCHEMA, &arg); 75 if (error != 0 || arg.error != 0) { 76 if (arg.error != 0) 77 errno = arg.error; 78 err(1, "Could not fetch config schema"); 79 } 80 81 schema = nvlist_unpack(arg.schema, arg.len, NV_FLAG_IGNORE_CASE); 82 if (schema == NULL) 83 err(1, "Could not unpack schema"); 84 85 free(arg.schema); 86 return (schema); 87 } 88 89 /* Fetch and unpack the current PCI SR-IOV status. */ 90 static nvlist_t * 91 get_status(int fd) 92 { 93 struct pci_iov_status arg; 94 nvlist_t *status; 95 void *buf, *newbuf; 96 size_t buflen; 97 int error; 98 99 buf = NULL; 100 buflen = 0; 101 for (;;) { 102 memset(&arg, 0, sizeof(arg)); 103 arg.status = buf; 104 arg.len = buflen; 105 error = ioctl(fd, IOV_GET_STATUS, &arg); 106 if (error != 0) 107 err(1, "Could not fetch SR-IOV status"); 108 if (arg.error == 0) 109 break; 110 if (arg.error != EMSGSIZE || arg.len <= buflen) { 111 errno = arg.error; 112 err(1, "Could not fetch SR-IOV status"); 113 } 114 newbuf = realloc(buf, arg.len); 115 if (newbuf == NULL) 116 err(1, "Could not allocate %zu bytes for SR-IOV status", 117 arg.len); 118 buf = newbuf; 119 buflen = arg.len; 120 } 121 if (arg.len == 0 || arg.len > buflen) 122 errx(1, "Kernel returned an invalid SR-IOV status length"); 123 124 status = nvlist_unpack(buf, arg.len, 0); 125 if (status == NULL) 126 err(1, "Could not unpack SR-IOV status"); 127 free(buf); 128 return (status); 129 } 130 131 /* 132 * Call the ioctl that activates SR-IOV and creates the VFs. 133 */ 134 static void 135 config_iov(int fd, const char *dev_name, const nvlist_t *config, int dryrun) 136 { 137 struct pci_iov_arg arg; 138 int error; 139 140 arg.config = nvlist_pack(config, &arg.len); 141 if (arg.config == NULL) 142 err(1, "Could not pack configuration"); 143 144 if (dryrun) { 145 printf("Would enable SR-IOV on device '%s'.\n", dev_name); 146 printf( 147 "The following configuration parameters would be used:\n"); 148 nvlist_fdump(config, stdout); 149 printf( 150 "The configuration parameters consume %zu bytes when packed.\n", 151 arg.len); 152 } else { 153 error = ioctl(fd, IOV_CONFIG, &arg); 154 if (error != 0) 155 err(1, "Failed to configure SR-IOV"); 156 } 157 158 free(arg.config); 159 } 160 161 static int 162 open_device(const char *dev_name) 163 { 164 char *dev; 165 int fd; 166 size_t copied, size; 167 long path_max; 168 169 path_max = pathconf("/dev", _PC_PATH_MAX); 170 if (path_max < 0) 171 err(1, "Could not get maximum path length"); 172 173 size = path_max; 174 dev = malloc(size); 175 if (dev == NULL) 176 err(1, "Could not allocate memory for device path"); 177 178 if (dev_name[0] == '/') 179 copied = strlcpy(dev, dev_name, size); 180 else 181 copied = snprintf(dev, size, "/dev/iov/%s", dev_name); 182 183 /* >= to account for null terminator. */ 184 if (copied >= size) 185 errx(1, "Provided file name too long"); 186 187 fd = open(dev, O_RDWR); 188 if (fd < 0) 189 err(1, "Could not open device '%s'", dev); 190 191 free(dev); 192 return (fd); 193 } 194 195 static void 196 usage(void) 197 { 198 199 warnx("Usage: iovctl -C -f <config file> [-n]"); 200 warnx(" iovctl -D [-d <PF device> | -f <config file>] [-n]"); 201 warnx(" iovctl -L [-d <PF device> | -f <config file>]"); 202 warnx(" iovctl -S [-d <PF device> | -f <config file>]"); 203 exit(1); 204 205 } 206 207 enum main_action { 208 NONE, 209 CONFIG, 210 DELETE, 211 PRINT_STATUS, 212 PRINT_SCHEMA, 213 }; 214 215 int 216 main(int argc, char **argv) 217 { 218 char *device; 219 const char *filename; 220 int ch, dryrun; 221 enum main_action action; 222 223 device = NULL; 224 filename = NULL; 225 dryrun = 0; 226 action = NONE; 227 228 while ((ch = getopt(argc, argv, "Cd:Df:LnS")) != -1) { 229 switch (ch) { 230 case 'C': 231 if (action != NONE) { 232 warnx("Only one action may be specified"); 233 usage(); 234 } 235 action = CONFIG; 236 break; 237 case 'd': 238 device = strdup(optarg); 239 break; 240 case 'D': 241 if (action != NONE) { 242 warnx("Only one action may be specified"); 243 usage(); 244 } 245 action = DELETE; 246 break; 247 case 'f': 248 filename = optarg; 249 break; 250 case 'n': 251 dryrun = 1; 252 break; 253 case 'L': 254 if (action != NONE) { 255 warnx("Only one action may be specified"); 256 usage(); 257 } 258 action = PRINT_STATUS; 259 break; 260 case 'S': 261 if (action != NONE) { 262 warnx("Only one action may be specified"); 263 usage(); 264 } 265 action = PRINT_SCHEMA; 266 break; 267 case '?': 268 warnx("Unrecognized argument '-%c'\n", optopt); 269 usage(); 270 break; 271 } 272 } 273 274 if (device != NULL && filename != NULL) { 275 warnx("Only one of the -d and -f flags may be specified"); 276 usage(); 277 } 278 279 if (device == NULL && filename == NULL && action != CONFIG) { 280 warnx("Either the -d or -f flag must be specified"); 281 usage(); 282 } 283 284 switch (action) { 285 case CONFIG: 286 if (device != NULL) { 287 warnx("-d flag cannot be used with the -C flag"); 288 usage(); 289 } 290 if (filename == NULL) { 291 warnx("The -f flag must be specified"); 292 usage(); 293 } 294 config_action(filename, dryrun); 295 break; 296 case DELETE: 297 if (device == NULL) 298 device = find_device(filename); 299 delete_action(device, dryrun); 300 free(device); 301 break; 302 case PRINT_SCHEMA: 303 if (dryrun) { 304 warnx("-n flag cannot be used with the -S flag"); 305 usage(); 306 } 307 if (device == NULL) 308 device = find_device(filename); 309 print_schema(device); 310 free(device); 311 break; 312 case PRINT_STATUS: 313 if (dryrun) { 314 warnx("-n flag cannot be used with the -L flag"); 315 usage(); 316 } 317 if (device == NULL) 318 device = find_device(filename); 319 print_status(device); 320 free(device); 321 break; 322 default: 323 usage(); 324 break; 325 } 326 327 exit(0); 328 } 329 330 static void 331 config_action(const char *filename, int dryrun) 332 { 333 char *dev; 334 nvlist_t *schema, *config; 335 int fd; 336 337 dev = find_device(filename); 338 fd = open(dev, O_RDWR); 339 if (fd < 0) 340 err(1, "Could not open device '%s'", dev); 341 342 schema = get_schema(fd); 343 config = parse_config_file(filename, schema); 344 if (config == NULL) 345 errx(1, "Could not parse config"); 346 347 config_iov(fd, dev, config, dryrun); 348 349 nvlist_destroy(config); 350 nvlist_destroy(schema); 351 free(dev); 352 close(fd); 353 } 354 355 static void 356 delete_action(const char *dev_name, int dryrun) 357 { 358 int fd, error; 359 360 fd = open_device(dev_name); 361 362 if (dryrun) 363 printf("Would attempt to delete all VF children of '%s'\n", 364 dev_name); 365 else { 366 error = ioctl(fd, IOV_DELETE); 367 if (error != 0) 368 err(1, "Failed to delete VFs"); 369 } 370 371 close(fd); 372 } 373 374 static void 375 validate_status(const nvlist_t *status) 376 { 377 const nvlist_t *pf; 378 const nvlist_t * const *vfs; 379 size_t i, num_vfs; 380 uint64_t configured_vfs, index, total_vfs, version; 381 382 if (!nvlist_exists_number(status, IOV_STATUS_VERSION_NAME) || 383 !nvlist_exists_nvlist(status, IOV_STATUS_PF_NAME)) 384 errx(1, "Kernel returned an invalid SR-IOV status"); 385 version = nvlist_get_number(status, IOV_STATUS_VERSION_NAME); 386 if (version != IOV_STATUS_VERSION) 387 errx(1, "Unsupported SR-IOV status version %ju", 388 (uintmax_t)version); 389 390 pf = nvlist_get_nvlist(status, IOV_STATUS_PF_NAME); 391 if (!nvlist_exists_string(pf, IOV_STATUS_DEVICE_NAME) || 392 !nvlist_exists_string(pf, IOV_STATUS_PCI_LOCATION_NAME) || 393 !nvlist_exists_bool(pf, IOV_STATUS_ENABLED_NAME) || 394 !nvlist_exists_number(pf, IOV_STATUS_NUM_VFS_NAME) || 395 !nvlist_exists_number(pf, IOV_STATUS_TOTAL_VFS_NAME)) 396 errx(1, "Kernel returned an invalid SR-IOV PF status"); 397 configured_vfs = nvlist_get_number(pf, IOV_STATUS_NUM_VFS_NAME); 398 total_vfs = nvlist_get_number(pf, IOV_STATUS_TOTAL_VFS_NAME); 399 if (configured_vfs > total_vfs) 400 errx(1, "Kernel returned inconsistent SR-IOV VF counts"); 401 if (configured_vfs == 0) { 402 if (nvlist_exists(status, IOV_STATUS_VFS_NAME)) 403 errx(1, "Kernel returned inconsistent SR-IOV VF counts"); 404 return; 405 } 406 if (!nvlist_exists_nvlist_array(status, IOV_STATUS_VFS_NAME)) 407 errx(1, "Kernel returned an invalid SR-IOV status"); 408 vfs = nvlist_get_nvlist_array(status, IOV_STATUS_VFS_NAME, &num_vfs); 409 if (configured_vfs != num_vfs) 410 errx(1, "Kernel returned inconsistent SR-IOV VF counts"); 411 for (i = 0; i < num_vfs; i++) { 412 if (!nvlist_exists_number(vfs[i], IOV_STATUS_VF_INDEX_NAME) || 413 !nvlist_exists_string(vfs[i], 414 IOV_STATUS_PCI_LOCATION_NAME) || 415 !nvlist_exists_bool(vfs[i], IOV_STATUS_ATTACHED_NAME) || 416 !nvlist_exists_bool(vfs[i], IOV_STATUS_PASSTHROUGH_NAME) || 417 (nvlist_exists(vfs[i], IOV_STATUS_BOUND_DRIVER_NAME) && 418 !nvlist_exists_string(vfs[i], 419 IOV_STATUS_BOUND_DRIVER_NAME))) 420 errx(1, "Kernel returned an invalid SR-IOV VF status"); 421 index = nvlist_get_number(vfs[i], IOV_STATUS_VF_INDEX_NAME); 422 if (index >= configured_vfs) 423 errx(1, "Kernel returned an invalid SR-IOV VF index"); 424 } 425 } 426 427 static void 428 print_status(const char *dev_name) 429 { 430 const nvlist_t *pf, *vf; 431 const nvlist_t * const *vfs; 432 nvlist_t *status; 433 size_t i, num_vfs; 434 int fd; 435 436 fd = open_device(dev_name); 437 status = get_status(fd); 438 validate_status(status); 439 pf = nvlist_get_nvlist(status, IOV_STATUS_PF_NAME); 440 vfs = NULL; 441 num_vfs = 0; 442 if (nvlist_exists_nvlist_array(status, IOV_STATUS_VFS_NAME)) 443 vfs = nvlist_get_nvlist_array(status, IOV_STATUS_VFS_NAME, 444 &num_vfs); 445 446 printf("%s:\n", nvlist_get_string(pf, IOV_STATUS_DEVICE_NAME)); 447 printf("\tidentity: pci-location=%s\n", 448 nvlist_get_string(pf, IOV_STATUS_PCI_LOCATION_NAME)); 449 printf("\tsriov: enabled=%s vfs=%ju/%ju\n", 450 nvlist_get_bool(pf, IOV_STATUS_ENABLED_NAME) ? "yes" : "no", 451 (uintmax_t)nvlist_get_number(pf, IOV_STATUS_NUM_VFS_NAME), 452 (uintmax_t)nvlist_get_number(pf, IOV_STATUS_TOTAL_VFS_NAME)); 453 for (i = 0; i < num_vfs; i++) { 454 vf = vfs[i]; 455 printf("\t\tvf %3ju:\n", (uintmax_t)nvlist_get_number(vf, 456 IOV_STATUS_VF_INDEX_NAME)); 457 printf("\t\t\tidentity: pci-location=%s\n", 458 nvlist_get_string(vf, IOV_STATUS_PCI_LOCATION_NAME)); 459 printf("\t\t\thost: attached=%s", 460 nvlist_get_bool(vf, IOV_STATUS_ATTACHED_NAME) ? "yes" : 461 "no"); 462 if (nvlist_exists_string(vf, IOV_STATUS_BOUND_DRIVER_NAME)) 463 printf(" driver=%s", nvlist_get_string(vf, 464 IOV_STATUS_BOUND_DRIVER_NAME)); 465 printf(" passthrough=%s\n", 466 nvlist_get_bool(vf, IOV_STATUS_PASSTHROUGH_NAME) ? "yes" : 467 "no"); 468 } 469 470 nvlist_destroy(status); 471 close(fd); 472 } 473 474 static void 475 print_default_value(const nvlist_t *parameter, const char *type) 476 { 477 const uint8_t *mac; 478 size_t size; 479 480 if (strcasecmp(type, "bool") == 0) 481 printf(" (default = %s)", 482 nvlist_get_bool(parameter, DEFAULT_SCHEMA_NAME) ? "true" : 483 "false"); 484 else if (strcasecmp(type, "string") == 0) 485 printf(" (default = %s)", 486 nvlist_get_string(parameter, DEFAULT_SCHEMA_NAME)); 487 else if (strcasecmp(type, "uint8_t") == 0) 488 printf(" (default = %ju)", 489 (uintmax_t)nvlist_get_number(parameter, 490 DEFAULT_SCHEMA_NAME)); 491 else if (strcasecmp(type, "uint16_t") == 0) 492 printf(" (default = %ju)", 493 (uintmax_t)nvlist_get_number(parameter, 494 DEFAULT_SCHEMA_NAME)); 495 else if (strcasecmp(type, "uint32_t") == 0) 496 printf(" (default = %ju)", 497 (uintmax_t)nvlist_get_number(parameter, 498 DEFAULT_SCHEMA_NAME)); 499 else if (strcasecmp(type, "uint64_t") == 0) 500 printf(" (default = %ju)", 501 (uintmax_t)nvlist_get_number(parameter, 502 DEFAULT_SCHEMA_NAME)); 503 else if (strcasecmp(type, "unicast-mac") == 0) { 504 mac = nvlist_get_binary(parameter, DEFAULT_SCHEMA_NAME, &size); 505 printf(" (default = %02x:%02x:%02x:%02x:%02x:%02x)", mac[0], 506 mac[1], mac[2], mac[3], mac[4], mac[5]); 507 } else if (strcasecmp(type, "vlan") == 0) { 508 uint16_t vlan = nvlist_get_number(parameter, DEFAULT_SCHEMA_NAME); 509 if (vlan == VF_VLAN_TRUNK) 510 printf(" (default = trunk)"); 511 else 512 printf(" (default = %d)", vlan); 513 } else 514 errx(1, "Unexpected type in schema: '%s'", type); 515 } 516 517 static void 518 print_subsystem_schema(const nvlist_t * subsystem_schema) 519 { 520 const char *name, *type; 521 const nvlist_t *parameter; 522 void *it; 523 int nvtype; 524 525 it = NULL; 526 while ((name = nvlist_next(subsystem_schema, &nvtype, &it)) != NULL) { 527 parameter = nvlist_get_nvlist(subsystem_schema, name); 528 type = nvlist_get_string(parameter, TYPE_SCHEMA_NAME); 529 530 printf("\t%s : %s", name, type); 531 if (dnvlist_get_bool(parameter, REQUIRED_SCHEMA_NAME, false)) 532 printf(" (required)"); 533 else if (nvlist_exists(parameter, DEFAULT_SCHEMA_NAME)) 534 print_default_value(parameter, type); 535 else 536 printf(" (optional)"); 537 printf("\n"); 538 } 539 } 540 541 static void 542 print_schema(const char *dev_name) 543 { 544 nvlist_t *schema; 545 const nvlist_t *iov_schema, *driver_schema, *pf_schema, *vf_schema; 546 int fd; 547 548 fd = open_device(dev_name); 549 schema = get_schema(fd); 550 551 pf_schema = nvlist_get_nvlist(schema, PF_CONFIG_NAME); 552 iov_schema = nvlist_get_nvlist(pf_schema, IOV_CONFIG_NAME); 553 driver_schema = nvlist_get_nvlist(pf_schema, DRIVER_CONFIG_NAME); 554 printf( 555 "The following configuration parameters may be configured on the PF:\n"); 556 print_subsystem_schema(iov_schema); 557 print_subsystem_schema(driver_schema); 558 559 vf_schema = nvlist_get_nvlist(schema, VF_SCHEMA_NAME); 560 iov_schema = nvlist_get_nvlist(vf_schema, IOV_CONFIG_NAME); 561 driver_schema = nvlist_get_nvlist(vf_schema, DRIVER_CONFIG_NAME); 562 printf( 563 "\nThe following configuration parameters may be configured on a VF:\n"); 564 print_subsystem_schema(iov_schema); 565 print_subsystem_schema(driver_schema); 566 567 nvlist_destroy(schema); 568 close(fd); 569 } 570