1 /*
2 * Copyright (c) 2012 Mellanox Technologies LTD. All rights reserved.
3 * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 */
34
35 #if HAVE_CONFIG_H
36 # include <config.h>
37 #endif /* HAVE_CONFIG_H */
38
39 #include <stdio.h>
40 #include <stdlib.h>
41 #include <unistd.h>
42 #include <getopt.h>
43 #include <netinet/in.h>
44
45 #include <infiniband/umad.h>
46 #include <infiniband/mad.h>
47
48 #include "ibdiag_common.h"
49
50 #define IS3_DEVICE_ID 47396
51
52 #define IB_MLX_VENDOR_CLASS 10
53 /* Vendor specific Attribute IDs */
54 #define IB_MLX_IS3_GENERAL_INFO 0x17
55 #define IB_MLX_IS3_CONFIG_SPACE_ACCESS 0x50
56 #define IB_MLX_IS4_COUNTER_GROUP_INFO 0x90
57 #define IB_MLX_IS4_CONFIG_COUNTER_GROUP 0x91
58 /* Config space addresses */
59 #define IB_MLX_IS3_PORT_XMIT_WAIT 0x10013C
60
61
62 struct ibmad_port *srcport;
63
64 static ibmad_gid_t dgid;
65 static int with_grh;
66
67 typedef struct {
68 uint16_t hw_revision;
69 uint16_t device_id;
70 uint8_t reserved[24];
71 uint32_t uptime;
72 } is3_hw_info_t;
73
74 typedef struct {
75 uint8_t resv1;
76 uint8_t major;
77 uint8_t minor;
78 uint8_t sub_minor;
79 uint32_t build_id;
80 uint8_t month;
81 uint8_t day;
82 uint16_t year;
83 uint16_t resv2;
84 uint16_t hour;
85 uint8_t psid[16];
86 uint32_t ini_file_version;
87 } is3_fw_info_t;
88
89 typedef struct {
90 uint32_t ext_major;
91 uint32_t ext_minor;
92 uint32_t ext_sub_minor;
93 uint32_t reserved[4];
94 } is4_fw_ext_info_t;
95
96 typedef struct {
97 uint8_t resv1;
98 uint8_t major;
99 uint8_t minor;
100 uint8_t sub_minor;
101 uint8_t resv2[28];
102 } is3_sw_info_t;
103
104 typedef struct {
105 uint8_t reserved[8];
106 is3_hw_info_t hw_info;
107 is3_fw_info_t fw_info;
108 is3_sw_info_t sw_info;
109 } is3_general_info_t;
110
111 typedef struct {
112 uint8_t reserved[8];
113 is3_hw_info_t hw_info;
114 is3_fw_info_t fw_info;
115 is4_fw_ext_info_t ext_fw_info;
116 is3_sw_info_t sw_info;
117 } is4_general_info_t;
118
119 typedef struct {
120 uint8_t reserved[8];
121 struct is3_record {
122 uint32_t address;
123 uint32_t data;
124 uint32_t mask;
125 } record[18];
126 } is3_config_space_t;
127
128 #define COUNTER_GROUPS_NUM 2
129
130 typedef struct {
131 uint8_t reserved1[8];
132 uint8_t reserved[3];
133 uint8_t num_of_counter_groups;
134 uint32_t group_masks[COUNTER_GROUPS_NUM];
135 } is4_counter_group_info_t;
136
137 typedef struct {
138 uint8_t reserved[3];
139 uint8_t group_select;
140 } is4_group_select_t;
141
142 typedef struct {
143 uint8_t reserved1[8];
144 uint8_t reserved[4];
145 is4_group_select_t group_selects[COUNTER_GROUPS_NUM];
146 } is4_config_counter_groups_t;
147
148 static uint16_t ext_fw_info_device[][2] = {
149 {0x0245, 0x0245}, /* Switch-X */
150 {0xc738, 0xc73b}, /* Switch-X */
151 {0xcb20, 0xcb20}, /* Switch-IB */
152 {0xcf08, 0xcf08}, /* Switch-IB2*/
153 {0x01b3, 0x01b3}, /* IS-4 */
154 {0x1003, 0x1017}, /* Connect-X */
155 {0x1b02, 0x1b02}, /* Bull SwitchX */
156 {0x1b50, 0x1b50}, /* Bull SwitchX */
157 {0x1ba0, 0x1ba0}, /* Bull SwitchIB */
158 {0x1bd0, 0x1bd5}, /* Bull SwitchIB and SwitchIB2 */
159 {0x1b33, 0x1b33}, /* Bull ConnectX3 */
160 {0x1b73, 0x1b73}, /* Bull ConnectX3 */
161 {0x1b40, 0x1b41}, /* Bull ConnectX3 */
162 {0x1b60, 0x1b61}, /* Bull ConnectX3 */
163 {0x1b83, 0x1b83}, /* Bull ConnectIB */
164 {0x1b93, 0x1b94}, /* Bull ConnectIB */
165 {0x1bb4, 0x1bb5}, /* Bull ConnectX4 */
166 {0x1bc4, 0x1bc4}, /* Bull ConnectX4 */
167 {0x0000, 0x0000}};
168
is_ext_fw_info_supported(uint16_t device_id)169 static int is_ext_fw_info_supported(uint16_t device_id) {
170 int i;
171 for (i = 0; ext_fw_info_device[i][0]; i++)
172 if (ext_fw_info_device[i][0] <= device_id &&
173 device_id <= ext_fw_info_device[i][1])
174 return 1;
175 return 0;
176 }
177
do_vendor(ib_portid_t * portid,struct ibmad_port * srcport,uint8_t class,uint8_t method,uint16_t attr_id,uint32_t attr_mod,void * data)178 static int do_vendor(ib_portid_t *portid, struct ibmad_port *srcport,
179 uint8_t class, uint8_t method, uint16_t attr_id,
180 uint32_t attr_mod, void *data)
181 {
182 ib_vendor_call_t call;
183
184 memset(&call, 0, sizeof(call));
185 call.mgmt_class = class;
186 call.method = method;
187 call.timeout = ibd_timeout;
188 call.attrid = attr_id;
189 call.mod = attr_mod;
190
191 if (!ib_vendor_call_via(data, portid, &call, srcport)) {
192 fprintf(stderr,"vendstat: method %u, attribute %u failure\n", method, attr_id);
193 return -1;
194 }
195 return 0;
196 }
197
do_config_space_records(ib_portid_t * portid,unsigned set,is3_config_space_t * cs,unsigned records)198 static int do_config_space_records(ib_portid_t *portid, unsigned set,
199 is3_config_space_t *cs, unsigned records)
200 {
201 unsigned i;
202
203 if (records > 18)
204 records = 18;
205 for (i = 0; i < records; i++) {
206 cs->record[i].address = htonl(cs->record[i].address);
207 cs->record[i].data = htonl(cs->record[i].data);
208 cs->record[i].mask = htonl(cs->record[i].mask);
209 }
210
211 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS,
212 set ? IB_MAD_METHOD_SET : IB_MAD_METHOD_GET,
213 IB_MLX_IS3_CONFIG_SPACE_ACCESS, 2 << 22 | records << 16,
214 cs)) {
215 fprintf(stderr,"cannot %s config space records\n", set ? "set" : "get");
216 return -1;
217 }
218 for (i = 0; i < records; i++) {
219 printf("Config space record at 0x%x: 0x%x\n",
220 ntohl(cs->record[i].address),
221 ntohl(cs->record[i].data & cs->record[i].mask));
222 }
223 return 0;
224 }
225
counter_groups_info(ib_portid_t * portid,int port)226 static int counter_groups_info(ib_portid_t * portid, int port)
227 {
228 char buf[1024];
229 is4_counter_group_info_t *cg_info;
230 int i, num_cg;
231
232 /* Counter Group Info */
233 memset(&buf, 0, sizeof(buf));
234 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET,
235 IB_MLX_IS4_COUNTER_GROUP_INFO, port, buf)) {
236 fprintf(stderr,"counter group info query failure\n");
237 return -1;
238 }
239 cg_info = (is4_counter_group_info_t *) & buf;
240 num_cg = cg_info->num_of_counter_groups;
241 printf("counter_group_info:\n");
242 printf("%d counter groups\n", num_cg);
243 for (i = 0; i < num_cg; i++)
244 printf("group%d mask %#x\n", i, ntohl(cg_info->group_masks[i]));
245 return 0;
246 }
247
248 /* Group0 counter config values */
249 #define IS4_G0_PortXmtDataSL_0_7 0
250 #define IS4_G0_PortXmtDataSL_8_15 1
251 #define IS4_G0_PortRcvDataSL_0_7 2
252
253 /* Group1 counter config values */
254 #define IS4_G1_PortXmtDataSL_8_15 1
255 #define IS4_G1_PortRcvDataSL_0_7 2
256 #define IS4_G1_PortRcvDataSL_8_15 8
257
258 static int cg0, cg1;
259
config_counter_groups(ib_portid_t * portid,int port)260 static int config_counter_groups(ib_portid_t * portid, int port)
261 {
262 char buf[1024];
263 is4_config_counter_groups_t *cg_config;
264
265 /* configure counter groups for groups 0 and 1 */
266 memset(&buf, 0, sizeof(buf));
267 cg_config = (is4_config_counter_groups_t *) & buf;
268
269 printf("counter_groups_config: configuring group0 %d group1 %d\n", cg0,
270 cg1);
271 cg_config->group_selects[0].group_select = (uint8_t) cg0;
272 cg_config->group_selects[1].group_select = (uint8_t) cg1;
273
274 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_SET,
275 IB_MLX_IS4_CONFIG_COUNTER_GROUP, port, buf)) {
276 fprintf(stderr, "config counter group set failure\n");
277 return -1;
278 }
279 /* get config counter groups */
280 memset(&buf, 0, sizeof(buf));
281
282 if (do_vendor(portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET,
283 IB_MLX_IS4_CONFIG_COUNTER_GROUP, port, buf)) {
284 fprintf(stderr, "config counter group query failure\n");
285 return -1;
286 }
287 return 0;
288 }
289
290 static int general_info, xmit_wait, counter_group_info, config_counter_group;
291 static is3_config_space_t write_cs, read_cs;
292 static unsigned write_cs_records, read_cs_records;
293
294
process_opt(void * context,int ch,char * optarg)295 static int process_opt(void *context, int ch, char *optarg)
296 {
297 int ret;
298 switch (ch) {
299 case 'N':
300 general_info = 1;
301 break;
302 case 'w':
303 xmit_wait = 1;
304 break;
305 case 'i':
306 counter_group_info = 1;
307 break;
308 case 'c':
309 config_counter_group = 1;
310 ret = sscanf(optarg, "%d,%d", &cg0, &cg1);
311 if (ret != 2)
312 return -1;
313 break;
314 case 'R':
315 if (read_cs_records >= 18)
316 break;
317 ret = sscanf(optarg, "%x,%x",
318 &read_cs.record[read_cs_records].address,
319 &read_cs.record[read_cs_records].mask);
320 if (ret < 1)
321 return -1;
322 else if (ret == 1)
323 read_cs.record[read_cs_records].mask = 0xffffffff;
324 read_cs_records++;
325 break;
326 case 'W':
327 if (write_cs_records >= 18)
328 break;
329 ret = sscanf(optarg, "%x,%x,%x",
330 &write_cs.record[write_cs_records].address,
331 &write_cs.record[write_cs_records].data,
332 &write_cs.record[write_cs_records].mask);
333 if (ret < 2)
334 return -1;
335 else if (ret == 2)
336 write_cs.record[write_cs_records].mask = 0xffffffff;
337 write_cs_records++;
338 break;
339 case 25:
340 if (!inet_pton(AF_INET6, optarg, &dgid)) {
341 fprintf(stderr, "dgid format is wrong!\n");
342 ibdiag_show_usage();
343 return 1;
344 }
345 with_grh = 1;
346 break;
347 default:
348 return -1;
349 }
350 return 0;
351 }
352
main(int argc,char ** argv)353 int main(int argc, char **argv)
354 {
355 int mgmt_classes[2] = { IB_SA_CLASS, IB_MLX_VENDOR_CLASS };
356 ib_portid_t portid = { 0 };
357 int port = 0;
358 char buf[1024];
359 uint32_t fw_ver_major = 0;
360 uint32_t fw_ver_minor = 0;
361 uint32_t fw_ver_sub_minor = 0;
362 uint8_t sw_ver_major = 0, sw_ver_minor = 0, sw_ver_sub_minor = 0;
363 is3_general_info_t *gi_is3;
364 is4_general_info_t *gi_is4;
365 const struct ibdiag_opt opts[] = {
366 {"N", 'N', 0, NULL, "show IS3 or IS4 general information"},
367 {"w", 'w', 0, NULL, "show IS3 port xmit wait counters"},
368 {"i", 'i', 0, NULL, "show IS4 counter group info"},
369 {"c", 'c', 1, "<num,num>", "configure IS4 counter groups"},
370 {"Read", 'R', 1, "<addr,mask>", "Read configuration space record at addr"},
371 {"Write", 'W', 1, "<addr,val,mask>", "Write configuration space record at addr"},
372 {"dgid", 25, 1, NULL, "remote gid (IPv6 format)"},
373 {0}
374 };
375
376 char usage_args[] = "<lid|guid> [port]";
377 const char *usage_examples[] = {
378 "-N 6\t\t# read IS3 or IS4 general information",
379 "-w 6\t\t# read IS3 port xmit wait counters",
380 "-i 6 12\t# read IS4 port 12 counter group info",
381 "-c 0,1 6 12\t# configure IS4 port 12 counter groups for PortXmitDataSL",
382 "-c 2,8 6 12\t# configure IS4 port 12 counter groups for PortRcvDataSL",
383 NULL
384 };
385
386 ibdiag_process_opts(argc, argv, NULL, "DKy", opts, process_opt,
387 usage_args, usage_examples);
388
389 argc -= optind;
390 argv += optind;
391
392 if (argc > 1)
393 port = strtoul(argv[1], 0, 0);
394
395 srcport = mad_rpc_open_port(ibd_ca, ibd_ca_port, mgmt_classes, 2);
396 if (!srcport)
397 IBEXIT("Failed to open '%s' port '%d'", ibd_ca, ibd_ca_port);
398
399 if (argc) {
400 if (with_grh && ibd_dest_type != IB_DEST_LID) {
401 mad_rpc_close_port(srcport);
402 IBEXIT("When using GRH, LID should be provided");
403 }
404 if (resolve_portid_str(ibd_ca, ibd_ca_port, &portid, argv[0],
405 ibd_dest_type, ibd_sm_id, srcport) < 0) {
406 mad_rpc_close_port(srcport);
407 IBEXIT("can't resolve destination port %s", argv[0]);
408 }
409 if (with_grh) {
410 portid.grh_present = 1;
411 memcpy(&portid.gid, &dgid, sizeof(portid.gid));
412 }
413 } else {
414 if (resolve_self(ibd_ca, ibd_ca_port, &portid, &port, 0) < 0) {
415 mad_rpc_close_port(srcport);
416 IBEXIT("can't resolve self port %s", argv[0]);
417 }
418 }
419
420 if (counter_group_info) {
421 counter_groups_info(&portid, port);
422 mad_rpc_close_port(srcport);
423 exit(0);
424 }
425
426 if (config_counter_group) {
427 config_counter_groups(&portid, port);
428 mad_rpc_close_port(srcport);
429 exit(0);
430 }
431
432 if (read_cs_records || write_cs_records) {
433 if (read_cs_records)
434 do_config_space_records(&portid, 0, &read_cs,
435 read_cs_records);
436 if (write_cs_records)
437 do_config_space_records(&portid, 1, &write_cs,
438 write_cs_records);
439 mad_rpc_close_port(srcport);
440 exit(0);
441 }
442
443 /* These are Mellanox specific vendor MADs */
444 /* but vendors change the VendorId so how know for sure ? */
445 /* Only General Info and Port Xmit Wait Counters */
446 /* queries are currently supported */
447 if (!general_info && !xmit_wait) {
448 mad_rpc_close_port(srcport);
449 IBEXIT("at least one of -N and -w must be specified");
450 }
451 /* Would need a list of these and it might not be complete */
452 /* so for right now, punt on this */
453
454 /* vendor ClassPortInfo is required attribute if class supported */
455 memset(&buf, 0, sizeof(buf));
456 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET,
457 CLASS_PORT_INFO, 0, buf)) {
458 mad_rpc_close_port(srcport);
459 IBEXIT("classportinfo query");
460 }
461 memset(&buf, 0, sizeof(buf));
462 gi_is3 = (is3_general_info_t *) &buf;
463 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS, IB_MAD_METHOD_GET,
464 IB_MLX_IS3_GENERAL_INFO, 0, gi_is3)) {
465 mad_rpc_close_port(srcport);
466 IBEXIT("generalinfo query");
467 }
468
469 if (is_ext_fw_info_supported(ntohs(gi_is3->hw_info.device_id))) {
470 gi_is4 = (is4_general_info_t *) &buf;
471 fw_ver_major = ntohl(gi_is4->ext_fw_info.ext_major);
472 fw_ver_minor = ntohl(gi_is4->ext_fw_info.ext_minor);
473 fw_ver_sub_minor = ntohl(gi_is4->ext_fw_info.ext_sub_minor);
474 sw_ver_major = gi_is4->sw_info.major;
475 sw_ver_minor = gi_is4->sw_info.minor;
476 sw_ver_sub_minor = gi_is4->sw_info.sub_minor;
477 } else {
478 fw_ver_major = gi_is3->fw_info.major;
479 fw_ver_minor = gi_is3->fw_info.minor;
480 fw_ver_sub_minor = gi_is3->fw_info.sub_minor;
481 sw_ver_major = gi_is3->sw_info.major;
482 sw_ver_minor = gi_is3->sw_info.minor;
483 sw_ver_sub_minor = gi_is3->sw_info.sub_minor;
484 }
485
486 if (general_info) {
487 /* dump IS3 or IS4 general info here */
488 printf("hw_dev_rev: 0x%04x\n", ntohs(gi_is3->hw_info.hw_revision));
489 printf("hw_dev_id: 0x%04x\n", ntohs(gi_is3->hw_info.device_id));
490 printf("hw_uptime: 0x%08x\n", ntohl(gi_is3->hw_info.uptime));
491 printf("fw_version: %02d.%02d.%02d\n",
492 fw_ver_major, fw_ver_minor, fw_ver_sub_minor);
493 printf("fw_build_id: 0x%04x\n", ntohl(gi_is3->fw_info.build_id));
494 printf("fw_date: %02x/%02x/%04x\n",
495 gi_is3->fw_info.month, gi_is3->fw_info.day,
496 ntohs(gi_is3->fw_info.year));
497 printf("fw_psid: '%s'\n", gi_is3->fw_info.psid);
498 printf("fw_ini_ver: %d\n",
499 ntohl(gi_is3->fw_info.ini_file_version));
500 printf("sw_version: %02d.%02d.%02d\n", sw_ver_major,
501 sw_ver_minor, sw_ver_sub_minor);
502 }
503
504 if (xmit_wait) {
505 is3_config_space_t *cs;
506 unsigned i;
507
508 if (ntohs(gi_is3->hw_info.device_id) != IS3_DEVICE_ID) {
509 mad_rpc_close_port(srcport);
510 IBEXIT("Unsupported device ID 0x%x",
511 ntohs(gi_is3->hw_info.device_id));
512 }
513 memset(&buf, 0, sizeof(buf));
514 /* Set record addresses for each port */
515 cs = (is3_config_space_t *) & buf;
516 for (i = 0; i < 16; i++)
517 cs->record[i].address =
518 htonl(IB_MLX_IS3_PORT_XMIT_WAIT + ((i + 1) << 12));
519 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS,
520 IB_MAD_METHOD_GET, IB_MLX_IS3_CONFIG_SPACE_ACCESS,
521 2 << 22 | 16 << 16, cs)) {
522 mad_rpc_close_port(srcport);
523 IBEXIT("vendstat");
524 }
525 for (i = 0; i < 16; i++)
526 if (cs->record[i].data) /* PortXmitWait is 32 bit counter */
527 printf("Port %d: PortXmitWait 0x%x\n", i + 4, ntohl(cs->record[i].data)); /* port 4 is first port */
528
529 /* Last 8 ports is another query */
530 memset(&buf, 0, sizeof(buf));
531 /* Set record addresses for each port */
532 cs = (is3_config_space_t *) & buf;
533 for (i = 0; i < 8; i++)
534 cs->record[i].address =
535 htonl(IB_MLX_IS3_PORT_XMIT_WAIT + ((i + 17) << 12));
536 if (do_vendor(&portid, srcport, IB_MLX_VENDOR_CLASS,
537 IB_MAD_METHOD_GET, IB_MLX_IS3_CONFIG_SPACE_ACCESS,
538 2 << 22 | 8 << 16, cs)) {
539 mad_rpc_close_port(srcport);
540 IBEXIT("vendstat");
541 }
542
543 for (i = 0; i < 8; i++)
544 if (cs->record[i].data) /* PortXmitWait is 32 bit counter */
545 printf("Port %d: PortXmitWait 0x%x\n",
546 i < 4 ? i + 21 : i - 3,
547 ntohl(cs->record[i].data));
548 }
549
550 mad_rpc_close_port(srcport);
551 exit(0);
552 }
553