1 /* 2 * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved. 3 * Copyright (c) 2014 Intel Corporation. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 * 33 */ 34 35 #include <config.h> 36 37 #include <sys/poll.h> 38 #include <unistd.h> 39 #include <string.h> 40 #include <stdio.h> 41 #include <errno.h> 42 #include <sys/types.h> 43 #include <sys/stat.h> 44 #include <fcntl.h> 45 #include <sys/ioctl.h> 46 #include <dirent.h> 47 #include <ctype.h> 48 #include <inttypes.h> 49 #include <assert.h> 50 51 #include <infiniband/umad.h> 52 53 #define IB_OPENIB_OUI (0x001405) 54 55 #include "sysfs.h" 56 57 typedef struct ib_user_mad_reg_req { 58 uint32_t id; 59 uint32_t method_mask[4]; 60 uint8_t qpn; 61 uint8_t mgmt_class; 62 uint8_t mgmt_class_version; 63 uint8_t oui[3]; 64 uint8_t rmpp_version; 65 } ib_user_mad_reg_req_t; 66 67 static_assert(sizeof(struct ib_user_mad_reg_req) == IOCPARM_LEN(IB_USER_MAD_REGISTER_AGENT), 68 "Invalid structure size"); 69 70 struct ib_user_mad_reg_req2 { 71 uint32_t id; 72 uint32_t qpn; 73 uint8_t mgmt_class; 74 uint8_t mgmt_class_version; 75 uint16_t res; 76 uint32_t flags; 77 uint64_t method_mask[2]; 78 uint32_t oui; 79 uint8_t rmpp_version; 80 uint8_t reserved[3]; 81 }; 82 83 static_assert(sizeof(struct ib_user_mad_reg_req2) == IOCPARM_LEN(IB_USER_MAD_REGISTER_AGENT2), 84 "Invalid structure size"); 85 86 #define IBWARN(fmt, args...) fprintf(stderr, "ibwarn: [%d] %s: " fmt "\n", getpid(), __func__, ## args) 87 88 #define TRACE if (umaddebug) IBWARN 89 #define DEBUG if (umaddebug) IBWARN 90 91 static int umaddebug = 0; 92 93 #define UMAD_DEV_FILE_SZ 256 94 95 static const char *def_ca_name = "mthca0"; 96 static int def_ca_port = 1; 97 98 static unsigned abi_version; 99 static unsigned new_user_mad_api; 100 101 /************************************* 102 * Port 103 */ 104 static int find_cached_ca(const char *ca_name, umad_ca_t * ca) 105 { 106 return 0; /* caching not implemented yet */ 107 } 108 109 static int put_ca(umad_ca_t * ca) 110 { 111 return 0; /* caching not implemented yet */ 112 } 113 114 static int release_port(umad_port_t * port) 115 { 116 free(port->pkeys); 117 port->pkeys = NULL; 118 port->pkeys_size = 0; 119 return 0; 120 } 121 122 static int check_for_digit_name(const struct dirent *dent) 123 { 124 const char *p = dent->d_name; 125 while (*p && isdigit(*p)) 126 p++; 127 return *p ? 0 : 1; 128 } 129 130 static int get_port(const char *ca_name, const char *dir, int portnum, umad_port_t * port) 131 { 132 char port_dir[256]; 133 union umad_gid gid; 134 struct dirent **namelist = NULL; 135 int i, len, num_pkeys = 0; 136 uint32_t capmask; 137 138 strncpy(port->ca_name, ca_name, sizeof port->ca_name - 1); 139 port->portnum = portnum; 140 port->pkeys = NULL; 141 142 len = snprintf(port_dir, sizeof(port_dir), "%s/%d", dir, portnum); 143 if (len < 0 || len > sizeof(port_dir)) 144 goto clean; 145 146 if (sys_read_uint(port_dir, SYS_PORT_LMC, &port->lmc) < 0) 147 goto clean; 148 if (sys_read_uint(port_dir, SYS_PORT_SMLID, &port->sm_lid) < 0) 149 goto clean; 150 if (sys_read_uint(port_dir, SYS_PORT_SMSL, &port->sm_sl) < 0) 151 goto clean; 152 if (sys_read_uint(port_dir, SYS_PORT_LID, &port->base_lid) < 0) 153 goto clean; 154 if (sys_read_uint(port_dir, SYS_PORT_STATE, &port->state) < 0) 155 goto clean; 156 if (sys_read_uint(port_dir, SYS_PORT_PHY_STATE, &port->phys_state) < 0) 157 goto clean; 158 if (sys_read_uint(port_dir, SYS_PORT_RATE, &port->rate) < 0) 159 goto clean; 160 if (sys_read_uint(port_dir, SYS_PORT_CAPMASK, &capmask) < 0) 161 goto clean; 162 163 if (sys_read_string(port_dir, SYS_PORT_LINK_LAYER, 164 port->link_layer, UMAD_CA_NAME_LEN) < 0) 165 /* assume IB by default */ 166 sprintf(port->link_layer, "IB"); 167 168 port->capmask = htobe32(capmask); 169 170 if (sys_read_gid(port_dir, SYS_PORT_GID, &gid) < 0) 171 goto clean; 172 173 port->gid_prefix = gid.global.subnet_prefix; 174 port->port_guid = gid.global.interface_id; 175 176 snprintf(port_dir + len, sizeof(port_dir) - len, "/pkeys"); 177 num_pkeys = sys_scandir(port_dir, &namelist, check_for_digit_name, NULL); 178 if (num_pkeys <= 0) { 179 IBWARN("no pkeys found for %s:%u (at dir %s)...", 180 port->ca_name, port->portnum, port_dir); 181 goto clean; 182 } 183 port->pkeys = calloc(num_pkeys, sizeof(port->pkeys[0])); 184 if (!port->pkeys) { 185 IBWARN("get_port: calloc failed: %s", strerror(errno)); 186 goto clean; 187 } 188 for (i = 0; i < num_pkeys; i++) { 189 unsigned idx, val; 190 idx = strtoul(namelist[i]->d_name, NULL, 0); 191 sys_read_uint(port_dir, namelist[i]->d_name, &val); 192 port->pkeys[idx] = val; 193 free(namelist[i]); 194 } 195 port->pkeys_size = num_pkeys; 196 free(namelist); 197 namelist = NULL; 198 port_dir[len] = '\0'; 199 200 /* FIXME: handle gids */ 201 202 return 0; 203 204 clean: 205 if (namelist) { 206 for (i = 0; i < num_pkeys; i++) 207 free(namelist[i]); 208 free(namelist); 209 } 210 if (port->pkeys) 211 free(port->pkeys); 212 return -EIO; 213 } 214 215 static int release_ca(umad_ca_t * ca) 216 { 217 int i; 218 219 for (i = 0; i <= ca->numports; i++) { 220 if (!ca->ports[i]) 221 continue; 222 release_port(ca->ports[i]); 223 free(ca->ports[i]); 224 ca->ports[i] = NULL; 225 } 226 return 0; 227 } 228 229 /* 230 * if *port > 0, check ca[port] state. Otherwise set *port to 231 * the first port that is active, and if such is not found, to 232 * the first port that is link up and if none are linkup, then 233 * the first port that is not disabled. Otherwise return -1. 234 */ 235 static int resolve_ca_port(const char *ca_name, int *port) 236 { 237 umad_ca_t ca; 238 int active = -1, up = -1; 239 int i, ret = 0; 240 241 TRACE("checking ca '%s'", ca_name); 242 243 if (umad_get_ca(ca_name, &ca) < 0) 244 return -1; 245 246 if (ca.node_type == 2) { 247 *port = 0; /* switch sma port 0 */ 248 ret = 1; 249 goto Exit; 250 } 251 252 if (*port > 0) { /* check only the port the user wants */ 253 if (*port > ca.numports) { 254 ret = -1; 255 goto Exit; 256 } 257 if (!ca.ports[*port]) { 258 ret = -1; 259 goto Exit; 260 } 261 if (strcmp(ca.ports[*port]->link_layer, "InfiniBand") && 262 strcmp(ca.ports[*port]->link_layer, "IB")) { 263 ret = -1; 264 goto Exit; 265 } 266 if (ca.ports[*port]->state == 4) { 267 ret = 1; 268 goto Exit; 269 } 270 if (ca.ports[*port]->phys_state != 3) 271 goto Exit; 272 ret = -1; 273 goto Exit; 274 } 275 276 for (i = 0; i <= ca.numports; i++) { 277 DEBUG("checking port %d", i); 278 if (!ca.ports[i]) 279 continue; 280 if (strcmp(ca.ports[i]->link_layer, "InfiniBand") && 281 strcmp(ca.ports[i]->link_layer, "IB")) 282 continue; 283 if (up < 0 && ca.ports[i]->phys_state == 5) 284 up = *port = i; 285 if (ca.ports[i]->state == 4) { 286 active = *port = i; 287 DEBUG("found active port %d", i); 288 break; 289 } 290 } 291 292 if (active == -1 && up == -1) { /* no active or linkup port found */ 293 for (i = 0; i <= ca.numports; i++) { 294 DEBUG("checking port %d", i); 295 if (!ca.ports[i]) 296 continue; 297 if (ca.ports[i]->phys_state != 3) { 298 up = *port = i; 299 break; 300 } 301 } 302 } 303 304 if (active >= 0) { 305 ret = 1; 306 goto Exit; 307 } 308 if (up >= 0) { 309 ret = 0; 310 goto Exit; 311 } 312 ret = -1; 313 Exit: 314 release_ca(&ca); 315 return ret; 316 } 317 318 static const char *resolve_ca_name(const char *ca_name, int *best_port) 319 { 320 static char names[UMAD_MAX_DEVICES][UMAD_CA_NAME_LEN]; 321 int phys_found = -1, port_found = 0, port, port_type; 322 int caidx, n; 323 324 if (ca_name && (!best_port || *best_port)) 325 return ca_name; 326 327 if (ca_name) { 328 if (resolve_ca_port(ca_name, best_port) < 0) 329 return NULL; 330 return ca_name; 331 } 332 333 /* Get the list of CA names */ 334 if ((n = umad_get_cas_names((void *)names, UMAD_MAX_DEVICES)) < 0) 335 return NULL; 336 337 /* Find the first existing CA with an active port */ 338 for (caidx = 0; caidx < n; caidx++) { 339 TRACE("checking ca '%s'", names[caidx]); 340 341 port = best_port ? *best_port : 0; 342 if ((port_type = resolve_ca_port(names[caidx], &port)) < 0) 343 continue; 344 345 DEBUG("found ca %s with port %d type %d", 346 names[caidx], port, port_type); 347 348 if (port_type > 0) { 349 if (best_port) 350 *best_port = port; 351 DEBUG("found ca %s with active port %d", 352 names[caidx], port); 353 return (char *)(names + caidx); 354 } 355 356 if (phys_found == -1) { 357 phys_found = caidx; 358 port_found = port; 359 } 360 } 361 362 DEBUG("phys found %d on %s port %d", 363 phys_found, phys_found >= 0 ? names[phys_found] : NULL, 364 port_found); 365 if (phys_found >= 0) { 366 if (best_port) 367 *best_port = port_found; 368 return names[phys_found]; 369 } 370 371 if (best_port) 372 *best_port = def_ca_port; 373 return def_ca_name; 374 } 375 376 static int get_ca(const char *ca_name, umad_ca_t * ca) 377 { 378 char dir_name[256]; 379 struct dirent **namelist; 380 int r, i, ret; 381 int portnum; 382 383 ca->numports = 0; 384 memset(ca->ports, 0, sizeof ca->ports); 385 strncpy(ca->ca_name, ca_name, sizeof(ca->ca_name) - 1); 386 387 snprintf(dir_name, sizeof(dir_name), "%s/%s", SYS_INFINIBAND, 388 ca->ca_name); 389 390 if ((r = sys_read_uint(dir_name, SYS_NODE_TYPE, &ca->node_type)) < 0) 391 return r; 392 if (sys_read_string(dir_name, SYS_CA_FW_VERS, ca->fw_ver, 393 sizeof ca->fw_ver) < 0) 394 ca->fw_ver[0] = '\0'; 395 if (sys_read_string(dir_name, SYS_CA_HW_VERS, ca->hw_ver, 396 sizeof ca->hw_ver) < 0) 397 ca->hw_ver[0] = '\0'; 398 if ((r = sys_read_string(dir_name, SYS_CA_TYPE, ca->ca_type, 399 sizeof ca->ca_type)) < 0) 400 ca->ca_type[0] = '\0'; 401 if ((r = sys_read_guid(dir_name, SYS_CA_NODE_GUID, &ca->node_guid)) < 0) 402 return r; 403 if ((r = 404 sys_read_guid(dir_name, SYS_CA_SYS_GUID, &ca->system_guid)) < 0) 405 return r; 406 407 snprintf(dir_name, sizeof(dir_name), "%s/%s/%s", 408 SYS_INFINIBAND, ca->ca_name, SYS_CA_PORTS_DIR); 409 410 if ((r = sys_scandir(dir_name, &namelist, NULL, alphasort)) < 0) { 411 ret = errno < 0 ? errno : -EIO; 412 goto error; 413 } 414 415 ret = 0; 416 for (i = 0; i < r; i++) { 417 portnum = 0; 418 if (!strcmp(".", namelist[i]->d_name) || 419 !strcmp("..", namelist[i]->d_name)) 420 continue; 421 if (strcmp("0", namelist[i]->d_name) && 422 ((portnum = atoi(namelist[i]->d_name)) <= 0 || 423 portnum >= UMAD_CA_MAX_PORTS)) { 424 ret = -EIO; 425 goto clean; 426 } 427 if (!(ca->ports[portnum] = 428 calloc(1, sizeof(*ca->ports[portnum])))) { 429 ret = -ENOMEM; 430 goto clean; 431 } 432 if (get_port(ca_name, dir_name, portnum, ca->ports[portnum]) < 433 0) { 434 free(ca->ports[portnum]); 435 ca->ports[portnum] = NULL; 436 ret = -EIO; 437 goto clean; 438 } 439 if (ca->numports < portnum) 440 ca->numports = portnum; 441 } 442 443 for (i = 0; i < r; i++) 444 free(namelist[i]); 445 free(namelist); 446 447 put_ca(ca); 448 return 0; 449 450 clean: 451 for (i = 0; i < r; i++) 452 free(namelist[i]); 453 free(namelist); 454 error: 455 release_ca(ca); 456 457 return ret; 458 } 459 460 static int umad_id_to_dev(int umad_id, char *dev, unsigned *port) 461 { 462 char path[256]; 463 int r; 464 465 snprintf(path, sizeof(path), SYS_INFINIBAND_MAD "/umad%d", umad_id); 466 467 if ((r = 468 sys_read_string(path, SYS_IB_MAD_DEV, dev, UMAD_CA_NAME_LEN)) < 0) 469 return r; 470 471 if ((r = sys_read_uint(path, SYS_IB_MAD_PORT, port)) < 0) 472 return r; 473 474 return 0; 475 } 476 477 static int dev_to_umad_id(const char *dev, unsigned port) 478 { 479 char umad_dev[UMAD_CA_NAME_LEN]; 480 unsigned umad_port; 481 int id; 482 483 for (id = 0; id < UMAD_MAX_PORTS; id++) { 484 if (umad_id_to_dev(id, umad_dev, &umad_port) < 0) 485 continue; 486 if (strncmp(dev, umad_dev, UMAD_CA_NAME_LEN)) 487 continue; 488 if (port != umad_port) 489 continue; 490 491 DEBUG("mapped %s %d to %d", dev, port, id); 492 return id; 493 } 494 495 return -1; /* not found */ 496 } 497 498 /******************************* 499 * Public interface 500 */ 501 502 int umad_init(void) 503 { 504 TRACE("umad_init"); 505 if (sys_read_uint(IB_UMAD_ABI_DIR, IB_UMAD_ABI_FILE, &abi_version) < 0) { 506 IBWARN 507 ("can't read ABI version from %s/%s (%m): is ib_umad module loaded?", 508 IB_UMAD_ABI_DIR, IB_UMAD_ABI_FILE); 509 return -1; 510 } 511 if (abi_version < IB_UMAD_ABI_VERSION) { 512 IBWARN 513 ("wrong ABI version: %s/%s is %d but library minimal ABI is %d", 514 IB_UMAD_ABI_DIR, IB_UMAD_ABI_FILE, abi_version, 515 IB_UMAD_ABI_VERSION); 516 return -1; 517 } 518 return 0; 519 } 520 521 int umad_done(void) 522 { 523 TRACE("umad_done"); 524 /* FIXME - verify that all ports are closed */ 525 return 0; 526 } 527 528 static unsigned is_ib_type(const char *ca_name) 529 { 530 char dir_name[256]; 531 unsigned type; 532 533 snprintf(dir_name, sizeof(dir_name), "%s/%s", SYS_INFINIBAND, ca_name); 534 535 if (sys_read_uint(dir_name, SYS_NODE_TYPE, &type) < 0) 536 return 0; 537 538 return type >= 1 && type <= 3 ? 1 : 0; 539 } 540 541 int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max) 542 { 543 struct dirent **namelist; 544 int n, i, j = 0; 545 546 TRACE("max %d", max); 547 548 n = sys_scandir(SYS_INFINIBAND, &namelist, NULL, alphasort); 549 if (n > 0) { 550 for (i = 0; i < n; i++) { 551 if (strcmp(namelist[i]->d_name, ".") && 552 strcmp(namelist[i]->d_name, "..")) { 553 if (j < max && is_ib_type(namelist[i]->d_name)) 554 strncpy(cas[j++], namelist[i]->d_name, 555 UMAD_CA_NAME_LEN); 556 } 557 free(namelist[i]); 558 } 559 DEBUG("return %d cas", j); 560 } else { 561 /* Is this still needed ? */ 562 strncpy((char *)cas, def_ca_name, UMAD_CA_NAME_LEN); 563 DEBUG("return 1 ca"); 564 j = 1; 565 } 566 if (n >= 0) 567 free(namelist); 568 return j; 569 } 570 571 int umad_get_ca_portguids(const char *ca_name, __be64 *portguids, int max) 572 { 573 umad_ca_t ca; 574 int ports = 0, i; 575 576 TRACE("ca name %s max port guids %d", ca_name, max); 577 if (!(ca_name = resolve_ca_name(ca_name, NULL))) 578 return -ENODEV; 579 580 if (umad_get_ca(ca_name, &ca) < 0) 581 return -1; 582 583 if (portguids) { 584 if (ca.numports + 1 > max) { 585 release_ca(&ca); 586 return -ENOMEM; 587 } 588 589 for (i = 0; i <= ca.numports; i++) 590 portguids[ports++] = ca.ports[i] ? 591 ca.ports[i]->port_guid : htobe64(0); 592 } 593 594 release_ca(&ca); 595 DEBUG("%s: %d ports", ca_name, ports); 596 597 return ports; 598 } 599 600 int umad_get_issm_path(const char *ca_name, int portnum, char path[], int max) 601 { 602 int umad_id; 603 604 TRACE("ca %s port %d", ca_name, portnum); 605 606 if (!(ca_name = resolve_ca_name(ca_name, &portnum))) 607 return -ENODEV; 608 609 if ((umad_id = dev_to_umad_id(ca_name, portnum)) < 0) 610 return -EINVAL; 611 612 snprintf(path, max, "%s/issm%u", UMAD_DEV_DIR, umad_id); 613 614 return 0; 615 } 616 617 int umad_open_port(const char *ca_name, int portnum) 618 { 619 char dev_file[UMAD_DEV_FILE_SZ]; 620 int umad_id, fd; 621 622 TRACE("ca %s port %d", ca_name, portnum); 623 624 if (!(ca_name = resolve_ca_name(ca_name, &portnum))) 625 return -ENODEV; 626 627 DEBUG("opening %s port %d", ca_name, portnum); 628 629 if ((umad_id = dev_to_umad_id(ca_name, portnum)) < 0) 630 return -EINVAL; 631 632 snprintf(dev_file, sizeof(dev_file), "%s/umad%d", 633 UMAD_DEV_DIR, umad_id); 634 635 if ((fd = open(dev_file, O_RDWR | O_NONBLOCK)) < 0) { 636 DEBUG("open %s failed: %s", dev_file, strerror(errno)); 637 return -EIO; 638 } 639 640 if (abi_version > 5 || !ioctl(fd, IB_USER_MAD_ENABLE_PKEY, NULL)) 641 new_user_mad_api = 1; 642 else 643 new_user_mad_api = 0; 644 645 DEBUG("opened %s fd %d portid %d", dev_file, fd, umad_id); 646 return fd; 647 } 648 649 int umad_get_ca(const char *ca_name, umad_ca_t * ca) 650 { 651 int r; 652 653 TRACE("ca_name %s", ca_name); 654 if (!(ca_name = resolve_ca_name(ca_name, NULL))) 655 return -ENODEV; 656 657 if (find_cached_ca(ca_name, ca) > 0) 658 return 0; 659 660 if ((r = get_ca(ca_name, ca)) < 0) 661 return r; 662 663 DEBUG("opened %s", ca_name); 664 return 0; 665 } 666 667 int umad_release_ca(umad_ca_t * ca) 668 { 669 int r; 670 671 TRACE("ca_name %s", ca->ca_name); 672 if (!ca) 673 return -ENODEV; 674 675 if ((r = release_ca(ca)) < 0) 676 return r; 677 678 DEBUG("releasing %s", ca->ca_name); 679 return 0; 680 } 681 682 int umad_get_port(const char *ca_name, int portnum, umad_port_t * port) 683 { 684 char dir_name[256]; 685 686 TRACE("ca_name %s portnum %d", ca_name, portnum); 687 688 if (!(ca_name = resolve_ca_name(ca_name, &portnum))) 689 return -ENODEV; 690 691 snprintf(dir_name, sizeof(dir_name), "%s/%s/%s", 692 SYS_INFINIBAND, ca_name, SYS_CA_PORTS_DIR); 693 694 return get_port(ca_name, dir_name, portnum, port); 695 } 696 697 int umad_release_port(umad_port_t * port) 698 { 699 int r; 700 701 TRACE("port %s:%d", port->ca_name, port->portnum); 702 if (!port) 703 return -ENODEV; 704 705 if ((r = release_port(port)) < 0) 706 return r; 707 708 DEBUG("releasing %s:%d", port->ca_name, port->portnum); 709 return 0; 710 } 711 712 int umad_close_port(int fd) 713 { 714 close(fd); 715 DEBUG("closed fd %d", fd); 716 return 0; 717 } 718 719 void *umad_get_mad(void *umad) 720 { 721 return new_user_mad_api ? ((struct ib_user_mad *)umad)->data : 722 (void *)&((struct ib_user_mad *)umad)->addr.pkey_index; 723 } 724 725 size_t umad_size(void) 726 { 727 return new_user_mad_api ? sizeof(struct ib_user_mad) : 728 sizeof(struct ib_user_mad) - 8; 729 } 730 731 int umad_set_grh(void *umad, void *mad_addr) 732 { 733 struct ib_user_mad *mad = umad; 734 struct ib_mad_addr *addr = mad_addr; 735 736 if (mad_addr) { 737 mad->addr.grh_present = 1; 738 mad->addr.ib_gid = addr->ib_gid; 739 /* The definition for umad_set_grh requires that the input be 740 * in host order */ 741 mad->addr.flow_label = htobe32((uint32_t)addr->flow_label); 742 mad->addr.hop_limit = addr->hop_limit; 743 mad->addr.traffic_class = addr->traffic_class; 744 } else 745 mad->addr.grh_present = 0; 746 return 0; 747 } 748 749 int umad_set_pkey(void *umad, int pkey_index) 750 { 751 struct ib_user_mad *mad = umad; 752 753 if (new_user_mad_api) 754 mad->addr.pkey_index = pkey_index; 755 756 return 0; 757 } 758 759 int umad_get_pkey(void *umad) 760 { 761 struct ib_user_mad *mad = umad; 762 763 if (new_user_mad_api) 764 return mad->addr.pkey_index; 765 766 return 0; 767 } 768 769 int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey) 770 { 771 struct ib_user_mad *mad = umad; 772 773 TRACE("umad %p dlid %u dqp %d sl %d, qkey %x", 774 umad, dlid, dqp, sl, qkey); 775 mad->addr.qpn = htobe32(dqp); 776 mad->addr.lid = htobe16(dlid); 777 mad->addr.qkey = htobe32(qkey); 778 mad->addr.sl = sl; 779 780 return 0; 781 } 782 783 int umad_set_addr_net(void *umad, __be16 dlid, __be32 dqp, int sl, __be32 qkey) 784 { 785 struct ib_user_mad *mad = umad; 786 787 TRACE("umad %p dlid %u dqp %d sl %d qkey %x", 788 umad, be16toh(dlid), be32toh(dqp), sl, be32toh(qkey)); 789 mad->addr.qpn = dqp; 790 mad->addr.lid = dlid; 791 mad->addr.qkey = qkey; 792 mad->addr.sl = sl; 793 794 return 0; 795 } 796 797 int umad_send(int fd, int agentid, void *umad, int length, 798 int timeout_ms, int retries) 799 { 800 struct ib_user_mad *mad = umad; 801 int n; 802 803 TRACE("fd %d agentid %d umad %p timeout %u", 804 fd, agentid, umad, timeout_ms); 805 errno = 0; 806 807 mad->timeout_ms = timeout_ms; 808 mad->retries = retries; 809 mad->agent_id = agentid; 810 811 if (umaddebug > 1) 812 umad_dump(mad); 813 814 n = write(fd, mad, length + umad_size()); 815 if (n == length + umad_size()) 816 return 0; 817 818 DEBUG("write returned %d != sizeof umad %zu + length %d (%m)", 819 n, umad_size(), length); 820 if (!errno) 821 errno = EIO; 822 return -EIO; 823 } 824 825 static int dev_poll(int fd, int timeout_ms) 826 { 827 struct pollfd ufds; 828 int n; 829 830 ufds.fd = fd; 831 ufds.events = POLLIN; 832 833 if ((n = poll(&ufds, 1, timeout_ms)) == 1) 834 return 0; 835 836 if (n == 0) 837 return -ETIMEDOUT; 838 839 return -EIO; 840 } 841 842 int umad_recv(int fd, void *umad, int *length, int timeout_ms) 843 { 844 struct ib_user_mad *mad = umad; 845 int n; 846 847 errno = 0; 848 TRACE("fd %d umad %p timeout %u", fd, umad, timeout_ms); 849 850 if (!umad || !length) { 851 errno = EINVAL; 852 return -EINVAL; 853 } 854 855 if (timeout_ms && (n = dev_poll(fd, timeout_ms)) < 0) { 856 if (!errno) 857 errno = -n; 858 return n; 859 } 860 861 n = read(fd, umad, umad_size() + *length); 862 863 VALGRIND_MAKE_MEM_DEFINED(umad, umad_size() + *length); 864 865 if ((n >= 0) && (n <= umad_size() + *length)) { 866 DEBUG("mad received by agent %d length %d", mad->agent_id, n); 867 if (n > umad_size()) 868 *length = n - umad_size(); 869 else 870 *length = 0; 871 return mad->agent_id; 872 } 873 874 if (n == -EWOULDBLOCK) { 875 if (!errno) 876 errno = EWOULDBLOCK; 877 return n; 878 } 879 880 DEBUG("read returned %zu > sizeof umad %zu + length %d (%m)", 881 mad->length - umad_size(), umad_size(), *length); 882 883 *length = mad->length - umad_size(); 884 if (!errno) 885 errno = EIO; 886 return -errno; 887 } 888 889 int umad_poll(int fd, int timeout_ms) 890 { 891 TRACE("fd %d timeout %u", fd, timeout_ms); 892 return dev_poll(fd, timeout_ms); 893 } 894 895 int umad_get_fd(int fd) 896 { 897 TRACE("fd %d", fd); 898 return fd; 899 } 900 901 int umad_register_oui(int fd, int mgmt_class, uint8_t rmpp_version, 902 uint8_t oui[3], long method_mask[]) 903 { 904 struct ib_user_mad_reg_req req; 905 906 TRACE("fd %d mgmt_class %u rmpp_version %d oui 0x%x%x%x method_mask %p", 907 fd, mgmt_class, (int)rmpp_version, (int)oui[0], (int)oui[1], 908 (int)oui[2], method_mask); 909 910 if (mgmt_class < 0x30 || mgmt_class > 0x4f) { 911 DEBUG("mgmt class %d not in vendor range 2", mgmt_class); 912 return -EINVAL; 913 } 914 915 req.qpn = 1; 916 req.mgmt_class = mgmt_class; 917 req.mgmt_class_version = 1; 918 memcpy(req.oui, oui, sizeof req.oui); 919 req.rmpp_version = rmpp_version; 920 921 if (method_mask) 922 memcpy(req.method_mask, method_mask, sizeof req.method_mask); 923 else 924 memset(req.method_mask, 0, sizeof req.method_mask); 925 926 VALGRIND_MAKE_MEM_DEFINED(&req, sizeof req); 927 928 if (!ioctl(fd, IB_USER_MAD_REGISTER_AGENT, (void *)&req)) { 929 DEBUG 930 ("fd %d registered to use agent %d qp %d class 0x%x oui %p", 931 fd, req.id, req.qpn, req.mgmt_class, oui); 932 return req.id; /* return agentid */ 933 } 934 935 DEBUG("fd %d registering qp %d class 0x%x version %d oui %p failed: %m", 936 fd, req.qpn, req.mgmt_class, req.mgmt_class_version, oui); 937 return -EPERM; 938 } 939 940 int umad_register(int fd, int mgmt_class, int mgmt_version, 941 uint8_t rmpp_version, long method_mask[]) 942 { 943 struct ib_user_mad_reg_req req; 944 __be32 oui = htobe32(IB_OPENIB_OUI); 945 int qp; 946 947 TRACE 948 ("fd %d mgmt_class %u mgmt_version %u rmpp_version %d method_mask %p", 949 fd, mgmt_class, mgmt_version, rmpp_version, method_mask); 950 951 req.qpn = qp = (mgmt_class == 0x1 || mgmt_class == 0x81) ? 0 : 1; 952 req.mgmt_class = mgmt_class; 953 req.mgmt_class_version = mgmt_version; 954 req.rmpp_version = rmpp_version; 955 956 if (method_mask) 957 memcpy(req.method_mask, method_mask, sizeof req.method_mask); 958 else 959 memset(req.method_mask, 0, sizeof req.method_mask); 960 961 memcpy(&req.oui, (char *)&oui + 1, sizeof req.oui); 962 963 VALGRIND_MAKE_MEM_DEFINED(&req, sizeof req); 964 965 if (!ioctl(fd, IB_USER_MAD_REGISTER_AGENT, (void *)&req)) { 966 DEBUG("fd %d registered to use agent %d qp %d", fd, req.id, qp); 967 return req.id; /* return agentid */ 968 } 969 970 DEBUG("fd %d registering qp %d class 0x%x version %d failed: %m", 971 fd, qp, mgmt_class, mgmt_version); 972 return -EPERM; 973 } 974 975 int umad_register2(int port_fd, struct umad_reg_attr *attr, uint32_t *agent_id) 976 { 977 struct ib_user_mad_reg_req2 req; 978 int rc; 979 980 if (!attr || !agent_id) 981 return EINVAL; 982 983 TRACE("fd %d mgmt_class %u mgmt_class_version %u flags 0x%08x " 984 "method_mask 0x%016" PRIx64 " %016" PRIx64 985 "oui 0x%06x rmpp_version %u ", 986 port_fd, attr->mgmt_class, attr->mgmt_class_version, 987 attr->flags, attr->method_mask[0], attr->method_mask[1], 988 attr->oui, attr->rmpp_version); 989 990 if (attr->mgmt_class >= 0x30 && attr->mgmt_class <= 0x4f && 991 ((attr->oui & 0x00ffffff) == 0 || (attr->oui & 0xff000000) != 0)) { 992 DEBUG("mgmt class %d is in vendor range 2 but oui (0x%08x) is invalid", 993 attr->mgmt_class, attr->oui); 994 return EINVAL; 995 } 996 997 memset(&req, 0, sizeof(req)); 998 999 req.mgmt_class = attr->mgmt_class; 1000 req.mgmt_class_version = attr->mgmt_class_version; 1001 req.qpn = (attr->mgmt_class == 0x1 || attr->mgmt_class == 0x81) ? 0 : 1; 1002 req.flags = attr->flags; 1003 memcpy(req.method_mask, attr->method_mask, sizeof req.method_mask); 1004 req.oui = attr->oui; 1005 req.rmpp_version = attr->rmpp_version; 1006 1007 VALGRIND_MAKE_MEM_DEFINED(&req, sizeof req); 1008 1009 if ((rc = ioctl(port_fd, IB_USER_MAD_REGISTER_AGENT2, (void *)&req)) == 0) { 1010 DEBUG("fd %d registered to use agent %d qp %d class 0x%x oui 0x%06x", 1011 port_fd, req.id, req.qpn, req.mgmt_class, attr->oui); 1012 *agent_id = req.id; 1013 return 0; 1014 } 1015 1016 if (errno == ENOTTY || errno == EINVAL) { 1017 1018 TRACE("no kernel support for registration flags"); 1019 req.flags = 0; 1020 1021 if (attr->flags == 0) { 1022 struct ib_user_mad_reg_req req_v1; 1023 1024 TRACE("attempting original register ioctl"); 1025 1026 memset(&req_v1, 0, sizeof(req_v1)); 1027 req_v1.mgmt_class = req.mgmt_class; 1028 req_v1.mgmt_class_version = req.mgmt_class_version; 1029 req_v1.qpn = req.qpn; 1030 req_v1.rmpp_version = req.rmpp_version; 1031 req_v1.oui[0] = (req.oui & 0xff0000) >> 16; 1032 req_v1.oui[1] = (req.oui & 0x00ff00) >> 8; 1033 req_v1.oui[2] = req.oui & 0x0000ff; 1034 1035 memcpy(req_v1.method_mask, req.method_mask, sizeof req_v1.method_mask); 1036 1037 if ((rc = ioctl(port_fd, IB_USER_MAD_REGISTER_AGENT, 1038 (void *)&req_v1)) == 0) { 1039 DEBUG("fd %d registered to use agent %d qp %d class 0x%x oui 0x%06x", 1040 port_fd, req_v1.id, req_v1.qpn, req_v1.mgmt_class, attr->oui); 1041 *agent_id = req_v1.id; 1042 return 0; 1043 } 1044 } 1045 } 1046 1047 rc = errno; 1048 attr->flags = req.flags; 1049 1050 DEBUG("fd %d registering qp %d class 0x%x version %d " 1051 "oui 0x%06x failed flags returned 0x%x : %m", 1052 port_fd, req.qpn, req.mgmt_class, req.mgmt_class_version, 1053 attr->oui, req.flags); 1054 1055 return rc; 1056 } 1057 1058 int umad_unregister(int fd, int agentid) 1059 { 1060 TRACE("fd %d unregistering agent %d", fd, agentid); 1061 return ioctl(fd, IB_USER_MAD_UNREGISTER_AGENT, &agentid); 1062 } 1063 1064 int umad_status(void *umad) 1065 { 1066 struct ib_user_mad *mad = umad; 1067 1068 return mad->status; 1069 } 1070 1071 ib_mad_addr_t *umad_get_mad_addr(void *umad) 1072 { 1073 struct ib_user_mad *mad = umad; 1074 1075 return &mad->addr; 1076 } 1077 1078 int umad_debug(int level) 1079 { 1080 if (level >= 0) 1081 umaddebug = level; 1082 return umaddebug; 1083 } 1084 1085 void umad_addr_dump(ib_mad_addr_t * addr) 1086 { 1087 #define HEX(x) ((x) < 10 ? '0' + (x) : 'a' + ((x) -10)) 1088 char gid_str[64]; 1089 int i; 1090 1091 for (i = 0; i < sizeof addr->gid; i++) { 1092 gid_str[i * 2] = HEX(addr->gid[i] >> 4); 1093 gid_str[i * 2 + 1] = HEX(addr->gid[i] & 0xf); 1094 } 1095 gid_str[i * 2] = 0; 1096 IBWARN("qpn %d qkey 0x%x lid %u sl %d\n" 1097 "grh_present %d gid_index %d hop_limit %d traffic_class %d flow_label 0x%x pkey_index 0x%x\n" 1098 "Gid 0x%s", 1099 be32toh(addr->qpn), be32toh(addr->qkey), be16toh(addr->lid), addr->sl, 1100 addr->grh_present, (int)addr->gid_index, (int)addr->hop_limit, 1101 (int)addr->traffic_class, addr->flow_label, addr->pkey_index, 1102 gid_str); 1103 } 1104 1105 void umad_dump(void *umad) 1106 { 1107 struct ib_user_mad *mad = umad; 1108 1109 IBWARN("agent id %d status %x timeout %d", 1110 mad->agent_id, mad->status, mad->timeout_ms); 1111 umad_addr_dump(&mad->addr); 1112 } 1113