1 /* 2 * Copyright (c) 2012 Mellanox Technologies LTD. All rights reserved. 3 * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 * 33 */ 34 35 #if HAVE_CONFIG_H 36 # include <config.h> 37 #endif /* HAVE_CONFIG_H */ 38 39 #include <stdio.h> 40 #include <stdlib.h> 41 #include <unistd.h> 42 #include <getopt.h> 43 #include <netinet/in.h> 44 45 #include <infiniband/umad.h> 46 #include <infiniband/mad.h> 47 48 #include "ibdiag_common.h" 49 50 #define IS3_DEVICE_ID 47396 51 52 #define IB_MLX_VENDOR_CLASS 10 53 /* Vendor specific Attribute IDs */ 54 #define IB_MLX_IS3_GENERAL_INFO 0x17 55 #define IB_MLX_IS3_CONFIG_SPACE_ACCESS 0x50 56 #define IB_MLX_IS4_COUNTER_GROUP_INFO 0x90 57 #define IB_MLX_IS4_CONFIG_COUNTER_GROUP 0x91 58 /* Config space addresses */ 59 #define IB_MLX_IS3_PORT_XMIT_WAIT 0x10013C 60 61 62 struct ibmad_port *srcport; 63 64 static ibmad_gid_t dgid; 65 static int with_grh; 66 67 typedef struct { 68 uint16_t hw_revision; 69 uint16_t device_id; 70 uint8_t reserved[24]; 71 uint32_t uptime; 72 } is3_hw_info_t; 73 74 typedef struct { 75 uint8_t resv1; 76 uint8_t major; 77 uint8_t minor; 78 uint8_t sub_minor; 79 uint32_t build_id; 80 uint8_t month; 81 uint8_t day; 82 uint16_t year; 83 uint16_t resv2; 84 uint16_t hour; 85 uint8_t psid[16]; 86 uint32_t ini_file_version; 87 } is3_fw_info_t; 88 89 typedef struct { 90 uint32_t ext_major; 91 uint32_t ext_minor; 92 uint32_t ext_sub_minor; 93 uint32_t reserved[4]; 94 } is4_fw_ext_info_t; 95 96 typedef struct { 97 uint8_t resv1; 98 uint8_t major; 99 uint8_t minor; 100 uint8_t sub_minor; 101 uint8_t resv2[28]; 102 } is3_sw_info_t; 103 104 typedef struct { 105 uint8_t reserved[8]; 106 is3_hw_info_t hw_info; 107 is3_fw_info_t fw_info; 108 is3_sw_info_t sw_info; 109 } is3_general_info_t; 110 111 typedef struct { 112 uint8_t reserved[8]; 113 is3_hw_info_t hw_info; 114 is3_fw_info_t fw_info; 115 is4_fw_ext_info_t ext_fw_info; 116 is3_sw_info_t sw_info; 117 } is4_general_info_t; 118 119 typedef struct { 120 uint8_t reserved[8]; 121 struct is3_record { 122 uint32_t address; 123 uint32_t data; 124 uint32_t mask; 125 } record[18]; 126 } is3_config_space_t; 127 128 #define COUNTER_GROUPS_NUM 2 129 130 typedef struct { 131 uint8_t reserved1[8]; 132 uint8_t reserved[3]; 133 uint8_t num_of_counter_groups; 134 uint32_t group_masks[COUNTER_GROUPS_NUM]; 135 } is4_counter_group_info_t; 136 137 typedef struct { 138 uint8_t reserved[3]; 139 uint8_t group_select; 140 } is4_group_select_t; 141 142 typedef struct { 143 uint8_t reserved1[8]; 144 uint8_t reserved[4]; 145 is4_group_select_t group_selects[COUNTER_GROUPS_NUM]; 146 } is4_config_counter_groups_t; 147 148 static uint16_t ext_fw_info_device[][2] = { 149 {0x0245, 0x0245}, /* Switch-X */ 150 {0xc738, 0xc73b}, /* Switch-X */ 151 {0xcb20, 0xcb20}, /* Switch-IB */ 152 {0xcf08, 0xcf08}, /* Switch-IB2*/ 153 {0x01b3, 0x01b3}, /* IS-4 */ 154 {0x1003, 0x1017}, /* Connect-X */ 155 {0x1b02, 0x1b02}, /* Bull SwitchX */ 156 {0x1b50, 0x1b50}, /* Bull SwitchX */ 157 {0x1ba0, 0x1ba0}, /* Bull SwitchIB */ 158 {0x1bd0, 0x1bd5}, /* Bull SwitchIB and SwitchIB2 */ 159 {0x1b33, 0x1b33}, /* Bull ConnectX3 */ 160 {0x1b73, 0x1b73}, /* Bull ConnectX3 */ 161 {0x1b40, 0x1b41}, /* Bull ConnectX3 */ 162 {0x1b60, 0x1b61}, /* Bull ConnectX3 */ 163 {0x1b83, 0x1b83}, /* Bull ConnectIB */ 164 {0x1b93, 0x1b94}, /* Bull ConnectIB */ 165 {0x1bb4, 0x1bb5}, /* Bull ConnectX4 */ 166 {0x1bc4, 0x1bc4}, /* Bull ConnectX4 */ 167 {0x0000, 0x0000}}; 168 169 static int is_ext_fw_info_supported(uint16_t device_id) { 170 int i; 171 for (i = 0; ext_fw_info_device[i][0]; i++) 172 if (ext_fw_info_device[i][0] <= device_id && 173 device_id <= ext_fw_info_device[i][1]) 174 return 1; 175 return 0; 176 } 177 178 static int do_vendor(ib_portid_t *portid, struct ibmad_port *srcport, 179 uint8_t class, uint8_t method, uint16_t attr_id, 180 uint32_t attr_mod, void *data) 181 { 182 ib_vendor_call_t call; 183 184 memset(&call, 0, sizeof(call)); 185 call.mgmt_class = class; 186 call.method = method; 187 call.timeout = ibd_timeout; 188 call.attrid = attr_id; 189 call.mod = attr_mod; 190 191 if (!ib_vendor_call_via(data, portid, &call, srcport)) { 192 fprintf(stderr,"vendstat: method %u, attribute %u failure\n", method, attr_id); 193 return -1; 194 } 195 return 0; 196 } 197 198 static int do_config_space_records(ib_portid_t *portid, unsigned set, 199 is3_config_space_t *cs, unsigned records) 200 { 201 unsigned i; 202 203 if (records > 18) 204 records = 18; 205 for (i = 0; i < records; i++) { 206 cs->record[i].address = htonl(cs->record[i].address); 207 cs->record[i].data = htonl(cs->record[i].data); 208 cs->record[i].mask = htonl(cs->record[i].mask); 209 } 210 211 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS, 212 set ? IB_MAD_METHOD_SET : IB_MAD_METHOD_GET, 213 IB_MLX_IS3_CONFIG_SPACE_ACCESS, 2 << 22 | records << 16, 214 cs)) { 215 fprintf(stderr,"cannot %s config space records\n", set ? "set" : "get"); 216 return -1; 217 } 218 for (i = 0; i < records; i++) { 219 printf("Config space record at 0x%x: 0x%x\n", 220 ntohl(cs->record[i].address), 221 ntohl(cs->record[i].data & cs->record[i].mask)); 222 } 223 return 0; 224 } 225 226 static int counter_groups_info(ib_portid_t * portid, int port) 227 { 228 char buf[1024]; 229 is4_counter_group_info_t *cg_info; 230 int i, num_cg; 231 232 /* Counter Group Info */ 233 memset(&buf, 0, sizeof(buf)); 234 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET, 235 IB_MLX_IS4_COUNTER_GROUP_INFO, port, buf)) { 236 fprintf(stderr,"counter group info query failure\n"); 237 return -1; 238 } 239 cg_info = (is4_counter_group_info_t *) & buf; 240 num_cg = cg_info->num_of_counter_groups; 241 printf("counter_group_info:\n"); 242 printf("%d counter groups\n", num_cg); 243 for (i = 0; i < num_cg; i++) 244 printf("group%d mask %#x\n", i, ntohl(cg_info->group_masks[i])); 245 return 0; 246 } 247 248 /* Group0 counter config values */ 249 #define IS4_G0_PortXmtDataSL_0_7 0 250 #define IS4_G0_PortXmtDataSL_8_15 1 251 #define IS4_G0_PortRcvDataSL_0_7 2 252 253 /* Group1 counter config values */ 254 #define IS4_G1_PortXmtDataSL_8_15 1 255 #define IS4_G1_PortRcvDataSL_0_7 2 256 #define IS4_G1_PortRcvDataSL_8_15 8 257 258 static int cg0, cg1; 259 260 static int config_counter_groups(ib_portid_t * portid, int port) 261 { 262 char buf[1024]; 263 is4_config_counter_groups_t *cg_config; 264 265 /* configure counter groups for groups 0 and 1 */ 266 memset(&buf, 0, sizeof(buf)); 267 cg_config = (is4_config_counter_groups_t *) & buf; 268 269 printf("counter_groups_config: configuring group0 %d group1 %d\n", cg0, 270 cg1); 271 cg_config->group_selects[0].group_select = (uint8_t) cg0; 272 cg_config->group_selects[1].group_select = (uint8_t) cg1; 273 274 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_SET, 275 IB_MLX_IS4_CONFIG_COUNTER_GROUP, port, buf)) { 276 fprintf(stderr, "config counter group set failure\n"); 277 return -1; 278 } 279 /* get config counter groups */ 280 memset(&buf, 0, sizeof(buf)); 281 282 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET, 283 IB_MLX_IS4_CONFIG_COUNTER_GROUP, port, buf)) { 284 fprintf(stderr, "config counter group query failure\n"); 285 return -1; 286 } 287 return 0; 288 } 289 290 static int general_info, xmit_wait, counter_group_info, config_counter_group; 291 static is3_config_space_t write_cs, read_cs; 292 static unsigned write_cs_records, read_cs_records; 293 294 295 static int process_opt(void *context, int ch, char *optarg) 296 { 297 int ret; 298 switch (ch) { 299 case 'N': 300 general_info = 1; 301 break; 302 case 'w': 303 xmit_wait = 1; 304 break; 305 case 'i': 306 counter_group_info = 1; 307 break; 308 case 'c': 309 config_counter_group = 1; 310 ret = sscanf(optarg, "%d,%d", &cg0, &cg1); 311 if (ret != 2) 312 return -1; 313 break; 314 case 'R': 315 if (read_cs_records >= 18) 316 break; 317 ret = sscanf(optarg, "%x,%x", 318 &read_cs.record[read_cs_records].address, 319 &read_cs.record[read_cs_records].mask); 320 if (ret < 1) 321 return -1; 322 else if (ret == 1) 323 read_cs.record[read_cs_records].mask = 0xffffffff; 324 read_cs_records++; 325 break; 326 case 'W': 327 if (write_cs_records >= 18) 328 break; 329 ret = sscanf(optarg, "%x,%x,%x", 330 &write_cs.record[write_cs_records].address, 331 &write_cs.record[write_cs_records].data, 332 &write_cs.record[write_cs_records].mask); 333 if (ret < 2) 334 return -1; 335 else if (ret == 2) 336 write_cs.record[write_cs_records].mask = 0xffffffff; 337 write_cs_records++; 338 break; 339 case 25: 340 if (!inet_pton(AF_INET6, optarg, &dgid)) { 341 fprintf(stderr, "dgid format is wrong!\n"); 342 ibdiag_show_usage(); 343 return 1; 344 } 345 with_grh = 1; 346 break; 347 default: 348 return -1; 349 } 350 return 0; 351 } 352 353 int main(int argc, char **argv) 354 { 355 int mgmt_classes[2] = { IB_SA_CLASS, IB_MLX_VENDOR_CLASS }; 356 ib_portid_t portid = { 0 }; 357 int port = 0; 358 char buf[1024]; 359 uint32_t fw_ver_major = 0; 360 uint32_t fw_ver_minor = 0; 361 uint32_t fw_ver_sub_minor = 0; 362 uint8_t sw_ver_major = 0, sw_ver_minor = 0, sw_ver_sub_minor = 0; 363 is3_general_info_t *gi_is3; 364 is4_general_info_t *gi_is4; 365 const struct ibdiag_opt opts[] = { 366 {"N", 'N', 0, NULL, "show IS3 or IS4 general information"}, 367 {"w", 'w', 0, NULL, "show IS3 port xmit wait counters"}, 368 {"i", 'i', 0, NULL, "show IS4 counter group info"}, 369 {"c", 'c', 1, "<num,num>", "configure IS4 counter groups"}, 370 {"Read", 'R', 1, "<addr,mask>", "Read configuration space record at addr"}, 371 {"Write", 'W', 1, "<addr,val,mask>", "Write configuration space record at addr"}, 372 {"dgid", 25, 1, NULL, "remote gid (IPv6 format)"}, 373 {0} 374 }; 375 376 char usage_args[] = "<lid|guid> [port]"; 377 const char *usage_examples[] = { 378 "-N 6\t\t# read IS3 or IS4 general information", 379 "-w 6\t\t# read IS3 port xmit wait counters", 380 "-i 6 12\t# read IS4 port 12 counter group info", 381 "-c 0,1 6 12\t# configure IS4 port 12 counter groups for PortXmitDataSL", 382 "-c 2,8 6 12\t# configure IS4 port 12 counter groups for PortRcvDataSL", 383 NULL 384 }; 385 386 ibdiag_process_opts(argc, argv, NULL, "DKy", opts, process_opt, 387 usage_args, usage_examples); 388 389 argc -= optind; 390 argv += optind; 391 392 if (argc > 1) 393 port = strtoul(argv[1], 0, 0); 394 395 srcport = mad_rpc_open_port(ibd_ca, ibd_ca_port, mgmt_classes, 2); 396 if (!srcport) 397 IBEXIT("Failed to open '%s' port '%d'", ibd_ca, ibd_ca_port); 398 399 if (argc) { 400 if (with_grh && ibd_dest_type != IB_DEST_LID) { 401 mad_rpc_close_port(srcport); 402 IBEXIT("When using GRH, LID should be provided"); 403 } 404 if (resolve_portid_str(ibd_ca, ibd_ca_port, &portid, argv[0], 405 ibd_dest_type, ibd_sm_id, srcport) < 0) { 406 mad_rpc_close_port(srcport); 407 IBEXIT("can't resolve destination port %s", argv[0]); 408 } 409 if (with_grh) { 410 portid.grh_present = 1; 411 memcpy(&portid.gid, &dgid, sizeof(portid.gid)); 412 } 413 } else { 414 if (resolve_self(ibd_ca, ibd_ca_port, &portid, &port, 0) < 0) { 415 mad_rpc_close_port(srcport); 416 IBEXIT("can't resolve self port %s", argv[0]); 417 } 418 } 419 420 if (counter_group_info) { 421 counter_groups_info(&portid, port); 422 mad_rpc_close_port(srcport); 423 exit(0); 424 } 425 426 if (config_counter_group) { 427 config_counter_groups(&portid, port); 428 mad_rpc_close_port(srcport); 429 exit(0); 430 } 431 432 if (read_cs_records || write_cs_records) { 433 if (read_cs_records) 434 do_config_space_records(&portid, 0, &read_cs, 435 read_cs_records); 436 if (write_cs_records) 437 do_config_space_records(&portid, 1, &write_cs, 438 write_cs_records); 439 mad_rpc_close_port(srcport); 440 exit(0); 441 } 442 443 /* These are Mellanox specific vendor MADs */ 444 /* but vendors change the VendorId so how know for sure ? */ 445 /* Only General Info and Port Xmit Wait Counters */ 446 /* queries are currently supported */ 447 if (!general_info && !xmit_wait) { 448 mad_rpc_close_port(srcport); 449 IBEXIT("at least one of -N and -w must be specified"); 450 } 451 /* Would need a list of these and it might not be complete */ 452 /* so for right now, punt on this */ 453 454 /* vendor ClassPortInfo is required attribute if class supported */ 455 memset(&buf, 0, sizeof(buf)); 456 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET, 457 CLASS_PORT_INFO, 0, buf)) { 458 mad_rpc_close_port(srcport); 459 IBEXIT("classportinfo query"); 460 } 461 memset(&buf, 0, sizeof(buf)); 462 gi_is3 = (is3_general_info_t *) &buf; 463 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET, 464 IB_MLX_IS3_GENERAL_INFO, 0, gi_is3)) { 465 mad_rpc_close_port(srcport); 466 IBEXIT("generalinfo query"); 467 } 468 469 if (is_ext_fw_info_supported(ntohs(gi_is3->hw_info.device_id))) { 470 gi_is4 = (is4_general_info_t *) &buf; 471 fw_ver_major = ntohl(gi_is4->ext_fw_info.ext_major); 472 fw_ver_minor = ntohl(gi_is4->ext_fw_info.ext_minor); 473 fw_ver_sub_minor = ntohl(gi_is4->ext_fw_info.ext_sub_minor); 474 sw_ver_major = gi_is4->sw_info.major; 475 sw_ver_minor = gi_is4->sw_info.minor; 476 sw_ver_sub_minor = gi_is4->sw_info.sub_minor; 477 } else { 478 fw_ver_major = gi_is3->fw_info.major; 479 fw_ver_minor = gi_is3->fw_info.minor; 480 fw_ver_sub_minor = gi_is3->fw_info.sub_minor; 481 sw_ver_major = gi_is3->sw_info.major; 482 sw_ver_minor = gi_is3->sw_info.minor; 483 sw_ver_sub_minor = gi_is3->sw_info.sub_minor; 484 } 485 486 if (general_info) { 487 /* dump IS3 or IS4 general info here */ 488 printf("hw_dev_rev: 0x%04x\n", ntohs(gi_is3->hw_info.hw_revision)); 489 printf("hw_dev_id: 0x%04x\n", ntohs(gi_is3->hw_info.device_id)); 490 printf("hw_uptime: 0x%08x\n", ntohl(gi_is3->hw_info.uptime)); 491 printf("fw_version: %02d.%02d.%02d\n", 492 fw_ver_major, fw_ver_minor, fw_ver_sub_minor); 493 printf("fw_build_id: 0x%04x\n", ntohl(gi_is3->fw_info.build_id)); 494 printf("fw_date: %02x/%02x/%04x\n", 495 gi_is3->fw_info.month, gi_is3->fw_info.day, 496 ntohs(gi_is3->fw_info.year)); 497 printf("fw_psid: '%s'\n", gi_is3->fw_info.psid); 498 printf("fw_ini_ver: %d\n", 499 ntohl(gi_is3->fw_info.ini_file_version)); 500 printf("sw_version: %02d.%02d.%02d\n", sw_ver_major, 501 sw_ver_minor, sw_ver_sub_minor); 502 } 503 504 if (xmit_wait) { 505 is3_config_space_t *cs; 506 unsigned i; 507 508 if (ntohs(gi_is3->hw_info.device_id) != IS3_DEVICE_ID) { 509 mad_rpc_close_port(srcport); 510 IBEXIT("Unsupported device ID 0x%x", 511 ntohs(gi_is3->hw_info.device_id)); 512 } 513 memset(&buf, 0, sizeof(buf)); 514 /* Set record addresses for each port */ 515 cs = (is3_config_space_t *) & buf; 516 for (i = 0; i < 16; i++) 517 cs->record[i].address = 518 htonl(IB_MLX_IS3_PORT_XMIT_WAIT + ((i + 1) << 12)); 519 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS, 520 IB_MAD_METHOD_GET, IB_MLX_IS3_CONFIG_SPACE_ACCESS, 521 2 << 22 | 16 << 16, cs)) { 522 mad_rpc_close_port(srcport); 523 IBEXIT("vendstat"); 524 } 525 for (i = 0; i < 16; i++) 526 if (cs->record[i].data) /* PortXmitWait is 32 bit counter */ 527 printf("Port %d: PortXmitWait 0x%x\n", i + 4, ntohl(cs->record[i].data)); /* port 4 is first port */ 528 529 /* Last 8 ports is another query */ 530 memset(&buf, 0, sizeof(buf)); 531 /* Set record addresses for each port */ 532 cs = (is3_config_space_t *) & buf; 533 for (i = 0; i < 8; i++) 534 cs->record[i].address = 535 htonl(IB_MLX_IS3_PORT_XMIT_WAIT + ((i + 17) << 12)); 536 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS, 537 IB_MAD_METHOD_GET, IB_MLX_IS3_CONFIG_SPACE_ACCESS, 538 2 << 22 | 8 << 16, cs)) { 539 mad_rpc_close_port(srcport); 540 IBEXIT("vendstat"); 541 } 542 543 for (i = 0; i < 8; i++) 544 if (cs->record[i].data) /* PortXmitWait is 32 bit counter */ 545 printf("Port %d: PortXmitWait 0x%x\n", 546 i < 4 ? i + 21 : i - 3, 547 ntohl(cs->record[i].data)); 548 } 549 550 mad_rpc_close_port(srcport); 551 exit(0); 552 } 553