1 /* 2 * Copyright (c) 2006-2009 Voltaire, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #if HAVE_CONFIG_H 35 # include <config.h> 36 #endif /* HAVE_CONFIG_H */ 37 38 #include <stdio.h> 39 #include <string.h> 40 #include <errno.h> 41 #include <inttypes.h> 42 43 #include <infiniband/umad.h> 44 #include <infiniband/mad.h> 45 #include <infiniband/iba/ib_types.h> 46 47 #include "ibdiag_common.h" 48 49 #define info(fmt, ...) fprintf(stderr, "INFO: " fmt, ## __VA_ARGS__ ) 50 #define err(fmt, ...) fprintf(stderr, "ERR: " fmt, ## __VA_ARGS__ ) 51 #ifdef NOISY_DEBUG 52 #define dbg(fmt, ...) fprintf(stderr, "DBG: " fmt, ## __VA_ARGS__ ) 53 #else 54 #define dbg(fmt, ...) 55 #endif 56 57 #define TMO 100 58 59 static ibmad_gid_t mgid_ipoib = { 60 0xff, 0x12, 0x40, 0x1b, 0xff, 0xff, 0x00, 0x00, 61 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff 62 }; 63 64 struct ibmad_port *srcport; 65 66 uint64_t build_mcm_rec(uint8_t * data, ibmad_gid_t mgid, ibmad_gid_t port_gid) 67 { 68 memset(data, 0, IB_SA_DATA_SIZE); 69 mad_set_array(data, 0, IB_SA_MCM_MGID_F, mgid); 70 mad_set_array(data, 0, IB_SA_MCM_PORTGID_F, port_gid); 71 mad_set_field(data, 0, IB_SA_MCM_JOIN_STATE_F, 1); 72 73 return IB_MCR_COMPMASK_MGID | IB_MCR_COMPMASK_PORT_GID | 74 IB_MCR_COMPMASK_JOIN_STATE; 75 } 76 77 static void build_mcm_rec_umad(void *umad, ib_portid_t * dport, int method, 78 uint64_t comp_mask, uint8_t * data) 79 { 80 ib_rpc_t rpc; 81 82 memset(&rpc, 0, sizeof(rpc)); 83 rpc.mgtclass = IB_SA_CLASS; 84 rpc.method = method; 85 rpc.attr.id = IB_SA_ATTR_MCRECORD; 86 rpc.attr.mod = 0; // ??? 87 rpc.mask = comp_mask; 88 rpc.datasz = IB_SA_DATA_SIZE; 89 rpc.dataoffs = IB_SA_DATA_OFFS; 90 91 mad_build_pkt(umad, &rpc, dport, NULL, data); 92 } 93 94 static int rereg_send(int port, int agent, ib_portid_t * dport, 95 uint8_t * umad, int len, int method, ibmad_gid_t port_gid) 96 { 97 uint8_t data[IB_SA_DATA_SIZE]; 98 uint64_t comp_mask; 99 100 comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid); 101 102 build_mcm_rec_umad(umad, dport, method, comp_mask, data); 103 if (umad_send(port, agent, umad, len, TMO, 0) < 0) { 104 err("umad_send %s failed: %s\n", 105 (method == IB_MAD_METHOD_GET) ? "query" : "non query", 106 strerror(errno)); 107 return -1; 108 } 109 dbg("umad_send %d: tid = 0x%016" PRIx64 "\n", method, 110 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F)); 111 112 return 0; 113 } 114 115 static int rereg_port_gid(int port, int agent, ib_portid_t * dport, 116 uint8_t * umad, int len, ibmad_gid_t port_gid) 117 { 118 uint8_t data[IB_SA_DATA_SIZE]; 119 uint64_t comp_mask; 120 121 comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid); 122 123 build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_DELETE, comp_mask, data); 124 if (umad_send(port, agent, umad, len, TMO, 0) < 0) { 125 err("umad_send leave failed: %s\n", strerror(errno)); 126 return -1; 127 } 128 dbg("umad_send leave: tid = 0x%016" PRIx64 "\n", 129 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F)); 130 131 build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_SET, comp_mask, data); 132 if (umad_send(port, agent, umad, len, TMO, 0) < 0) { 133 err("umad_send join failed: %s\n", strerror(errno)); 134 return -1; 135 } 136 dbg("umad_send join: tid = 0x%016" PRIx64 "\n", 137 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F)); 138 139 return 0; 140 } 141 142 struct guid_trid { 143 ibmad_gid_t gid; 144 uint64_t guid; 145 uint64_t trid; 146 }; 147 148 static int rereg_send_all(int port, int agent, ib_portid_t * dport, 149 struct guid_trid *list, unsigned cnt) 150 { 151 uint8_t *umad; 152 int len = umad_size() + 256; 153 unsigned i; 154 int ret; 155 156 info("rereg_send_all... cnt = %u\n", cnt); 157 158 umad = calloc(1, len); 159 if (!umad) { 160 err("cannot alloc mem for umad: %s\n", strerror(errno)); 161 return -1; 162 } 163 164 for (i = 0; i < cnt; i++) { 165 ret = 166 rereg_port_gid(port, agent, dport, umad, len, list[i].gid); 167 if (ret < 0) { 168 err("rereg_send_all: rereg_port_gid 0x%016" PRIx64 169 " failed\n", ntohll(list[i].guid)); 170 continue; 171 } 172 list[i].trid = mad_get_field64(umad_get_mad(umad), 0, 173 IB_MAD_TRID_F); 174 } 175 176 info("rereg_send_all: sent %u requests\n", cnt * 2); 177 178 free(umad); 179 180 return 0; 181 } 182 183 static int rereg_recv(int port, int agent, ib_portid_t * dport, 184 uint8_t * umad, int length, int tmo) 185 { 186 int ret, retry = 0; 187 int len = length; 188 189 while ((ret = umad_recv(port, umad, &len, tmo)) < 0 && 190 errno == ETIMEDOUT) { 191 if (retry++ > 3) 192 return 0; 193 } 194 if (ret < 0) { 195 err("umad_recv %d failed: %s\n", ret, strerror(errno)); 196 return -1; 197 } 198 dbg("umad_recv (retries %d), tid = 0x%016" PRIx64 199 ": len = %d, status = %d\n", retry, 200 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F), len, 201 umad_status(umad)); 202 203 return 1; 204 } 205 206 static int rereg_recv_all(int port, int agent, ib_portid_t * dport, 207 struct guid_trid *list, unsigned cnt) 208 { 209 uint8_t *umad, *mad; 210 int len = umad_size() + 256; 211 uint64_t trid; 212 unsigned n, method, status; 213 unsigned i; 214 215 info("rereg_recv_all...\n"); 216 217 umad = calloc(1, len); 218 if (!umad) { 219 err("cannot alloc mem for umad: %s\n", strerror(errno)); 220 return -1; 221 } 222 223 n = 0; 224 while (rereg_recv(port, agent, dport, umad, len, TMO) > 0) { 225 dbg("rereg_recv_all: done %d\n", n); 226 n++; 227 mad = umad_get_mad(umad); 228 229 method = mad_get_field(mad, 0, IB_MAD_METHOD_F); 230 status = mad_get_field(mad, 0, IB_MAD_STATUS_F); 231 232 if (status) 233 dbg("MAD status %x, method %x\n", status, method); 234 235 if (status && 236 (method & 0x7f) == (IB_MAD_METHOD_GET_RESPONSE & 0x7f)) { 237 trid = mad_get_field64(mad, 0, IB_MAD_TRID_F); 238 for (i = 0; i < cnt; i++) 239 if (trid == list[i].trid) 240 break; 241 if (i == cnt) { 242 err("cannot find trid 0x%016" PRIx64 "\n", 243 trid); 244 continue; 245 } 246 info("guid 0x%016" PRIx64 247 ": method = %x status = %x. Resending\n", 248 ntohll(list[i].guid), method, status); 249 rereg_port_gid(port, agent, dport, umad, len, 250 list[i].gid); 251 list[i].trid = 252 mad_get_field64(umad_get_mad(umad), 0, 253 IB_MAD_TRID_F); 254 } 255 } 256 257 info("rereg_recv_all: got %u responses\n", n); 258 259 free(umad); 260 return 0; 261 } 262 263 static int rereg_query_all(int port, int agent, ib_portid_t * dport, 264 struct guid_trid *list, unsigned cnt) 265 { 266 uint8_t *umad, *mad; 267 int len = umad_size() + 256; 268 unsigned method, status; 269 unsigned i; 270 int ret; 271 272 info("rereg_query_all...\n"); 273 274 umad = calloc(1, len); 275 if (!umad) { 276 err("cannot alloc mem for umad: %s\n", strerror(errno)); 277 return -1; 278 } 279 280 for (i = 0; i < cnt; i++) { 281 ret = rereg_send(port, agent, dport, umad, len, 282 IB_MAD_METHOD_GET, list[i].gid); 283 if (ret < 0) { 284 err("query_all: rereg_send failed.\n"); 285 continue; 286 } 287 288 ret = rereg_recv(port, agent, dport, umad, len, TMO); 289 if (ret < 0) { 290 err("query_all: rereg_recv failed.\n"); 291 continue; 292 } 293 294 mad = umad_get_mad(umad); 295 296 method = mad_get_field(mad, 0, IB_MAD_METHOD_F); 297 status = mad_get_field(mad, 0, IB_MAD_STATUS_F); 298 299 if (status) 300 info("guid 0x%016" PRIx64 ": status %x, method %x\n", 301 ntohll(list[i].guid), status, method); 302 } 303 304 info("rereg_query_all: %u queried.\n", cnt); 305 306 free(umad); 307 return 0; 308 } 309 310 #define MAX_CLIENTS 50 311 312 static int rereg_and_test_port(char *guid_file, int port, int agent, 313 ib_portid_t * dport, int timeout) 314 { 315 char line[256]; 316 FILE *f; 317 ibmad_gid_t port_gid; 318 uint64_t prefix = htonll(0xfe80000000000000ull); 319 uint64_t guid = htonll(0x0002c90200223825ull); 320 struct guid_trid *list; 321 int i = 0; 322 323 list = calloc(MAX_CLIENTS, sizeof(*list)); 324 if (!list) { 325 err("cannot alloc mem for guid/trid list: %s\n", 326 strerror(errno)); 327 return -1; 328 } 329 330 f = fopen(guid_file, "r"); 331 if (!f) { 332 err("cannot open %s: %s\n", guid_file, strerror(errno)); 333 return -1; 334 } 335 336 while (fgets(line, sizeof(line), f)) { 337 guid = strtoull(line, NULL, 0); 338 guid = htonll(guid); 339 memcpy(&port_gid[0], &prefix, 8); 340 memcpy(&port_gid[8], &guid, 8); 341 342 list[i].guid = guid; 343 memcpy(list[i].gid, port_gid, sizeof(list[i].gid)); 344 list[i].trid = 0; 345 if (++i >= MAX_CLIENTS) 346 break; 347 } 348 fclose(f); 349 350 rereg_send_all(port, agent, dport, list, i); 351 rereg_recv_all(port, agent, dport, list, i); 352 353 rereg_query_all(port, agent, dport, list, i); 354 355 free(list); 356 return 0; 357 } 358 359 int main(int argc, char **argv) 360 { 361 char *guid_file = "port_guids.list"; 362 int mgmt_classes[2] = { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS }; 363 ib_portid_t dport_id; 364 int port, agent; 365 uint8_t *umad, *mad; 366 int len; 367 368 if (argc > 1) 369 guid_file = argv[1]; 370 371 srcport = mad_rpc_open_port(NULL, 0, mgmt_classes, 2); 372 if (!srcport) 373 err("Failed to open port"); 374 375 resolve_sm_portid(NULL, 0, &dport_id); 376 dport_id.qp = 1; 377 if (!dport_id.qkey) 378 dport_id.qkey = IB_DEFAULT_QP1_QKEY; 379 380 len = umad_size() + 256; 381 umad = calloc(1, len); 382 if (!umad) { 383 err("cannot alloc mem for umad: %s\n", strerror(errno)); 384 return -1; 385 } 386 port = mad_rpc_portid(srcport); 387 388 agent = umad_register(port, IB_SA_CLASS, 2, 0, NULL); 389 390 rereg_and_test_port(guid_file, port, agent, &dport_id, TMO); 391 mad = umad_get_mad(umad); 392 393 free(umad); 394 umad_unregister(port, agent); 395 umad_close_port(port); 396 umad_done(); 397 398 return 0; 399 } 400