1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #if HAVE_CONFIG_H 34 # include <config.h> 35 #endif /* HAVE_CONFIG_H */ 36 37 #include <stdio.h> 38 #include <stdlib.h> 39 #include <unistd.h> 40 #include <string.h> 41 #include <sys/types.h> 42 #include <sys/socket.h> 43 #include <sys/time.h> 44 #include <netdb.h> 45 #include <getopt.h> 46 #include <arpa/inet.h> 47 #include <time.h> 48 49 #include "pingpong.h" 50 51 enum { 52 PINGPONG_RECV_WRID = 1, 53 PINGPONG_SEND_WRID = 2, 54 }; 55 56 static int page_size; 57 58 struct pingpong_context { 59 struct ibv_context *context; 60 struct ibv_comp_channel *channel; 61 struct ibv_pd *pd; 62 struct ibv_mr *mr; 63 struct ibv_cq *cq; 64 struct ibv_qp *qp; 65 struct ibv_ah *ah; 66 void *buf; 67 int size; 68 int rx_depth; 69 int pending; 70 struct ibv_port_attr portinfo; 71 }; 72 73 struct pingpong_dest { 74 int lid; 75 int qpn; 76 int psn; 77 union ibv_gid gid; 78 }; 79 80 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 81 int sl, struct pingpong_dest *dest, int sgid_idx) 82 { 83 struct ibv_ah_attr ah_attr = { 84 .is_global = 0, 85 .dlid = dest->lid, 86 .sl = sl, 87 .src_path_bits = 0, 88 .port_num = port 89 }; 90 struct ibv_qp_attr attr = { 91 .qp_state = IBV_QPS_RTR 92 }; 93 94 if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) { 95 fprintf(stderr, "Failed to modify QP to RTR\n"); 96 return 1; 97 } 98 99 attr.qp_state = IBV_QPS_RTS; 100 attr.sq_psn = my_psn; 101 102 if (ibv_modify_qp(ctx->qp, &attr, 103 IBV_QP_STATE | 104 IBV_QP_SQ_PSN)) { 105 fprintf(stderr, "Failed to modify QP to RTS\n"); 106 return 1; 107 } 108 109 if (dest->gid.global.interface_id) { 110 ah_attr.is_global = 1; 111 ah_attr.grh.hop_limit = 1; 112 ah_attr.grh.dgid = dest->gid; 113 ah_attr.grh.sgid_index = sgid_idx; 114 } 115 116 ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); 117 if (!ctx->ah) { 118 fprintf(stderr, "Failed to create AH\n"); 119 return 1; 120 } 121 122 return 0; 123 } 124 125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, 126 const struct pingpong_dest *my_dest) 127 { 128 struct addrinfo *res, *t; 129 struct addrinfo hints = { 130 .ai_family = AF_INET, 131 .ai_socktype = SOCK_STREAM 132 }; 133 char *service; 134 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 135 int n; 136 int sockfd = -1; 137 struct pingpong_dest *rem_dest = NULL; 138 char gid[33]; 139 140 if (asprintf(&service, "%d", port) < 0) 141 return NULL; 142 143 n = getaddrinfo(servername, service, &hints, &res); 144 145 if (n < 0) { 146 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); 147 free(service); 148 return NULL; 149 } 150 151 for (t = res; t; t = t->ai_next) { 152 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 153 if (sockfd >= 0) { 154 if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) 155 break; 156 close(sockfd); 157 sockfd = -1; 158 } 159 } 160 161 freeaddrinfo(res); 162 free(service); 163 164 if (sockfd < 0) { 165 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); 166 return NULL; 167 } 168 169 gid_to_wire_gid(&my_dest->gid, gid); 170 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 171 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 172 fprintf(stderr, "Couldn't send local address\n"); 173 goto out; 174 } 175 176 if (read(sockfd, msg, sizeof msg) != sizeof msg) { 177 perror("client read"); 178 fprintf(stderr, "Couldn't read remote address\n"); 179 goto out; 180 } 181 182 write(sockfd, "done", sizeof "done"); 183 184 rem_dest = malloc(sizeof *rem_dest); 185 if (!rem_dest) 186 goto out; 187 188 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 189 wire_gid_to_gid(gid, &rem_dest->gid); 190 191 out: 192 close(sockfd); 193 return rem_dest; 194 } 195 196 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 197 int ib_port, int port, int sl, 198 const struct pingpong_dest *my_dest, 199 int sgid_idx) 200 { 201 struct addrinfo *res, *t; 202 struct addrinfo hints = { 203 .ai_flags = AI_PASSIVE, 204 .ai_family = AF_INET, 205 .ai_socktype = SOCK_STREAM 206 }; 207 char *service; 208 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 209 int n; 210 int sockfd = -1, connfd; 211 struct pingpong_dest *rem_dest = NULL; 212 char gid[33]; 213 214 if (asprintf(&service, "%d", port) < 0) 215 return NULL; 216 217 n = getaddrinfo(NULL, service, &hints, &res); 218 219 if (n < 0) { 220 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); 221 free(service); 222 return NULL; 223 } 224 225 for (t = res; t; t = t->ai_next) { 226 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 227 if (sockfd >= 0) { 228 n = 1; 229 230 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); 231 232 if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) 233 break; 234 close(sockfd); 235 sockfd = -1; 236 } 237 } 238 239 freeaddrinfo(res); 240 free(service); 241 242 if (sockfd < 0) { 243 fprintf(stderr, "Couldn't listen to port %d\n", port); 244 return NULL; 245 } 246 247 listen(sockfd, 1); 248 connfd = accept(sockfd, NULL, 0); 249 close(sockfd); 250 if (connfd < 0) { 251 fprintf(stderr, "accept() failed\n"); 252 return NULL; 253 } 254 255 n = read(connfd, msg, sizeof msg); 256 if (n != sizeof msg) { 257 perror("server read"); 258 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); 259 goto out; 260 } 261 262 rem_dest = malloc(sizeof *rem_dest); 263 if (!rem_dest) 264 goto out; 265 266 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 267 wire_gid_to_gid(gid, &rem_dest->gid); 268 269 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) { 270 fprintf(stderr, "Couldn't connect to remote QP\n"); 271 free(rem_dest); 272 rem_dest = NULL; 273 goto out; 274 } 275 276 gid_to_wire_gid(&my_dest->gid, gid); 277 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 278 if (write(connfd, msg, sizeof msg) != sizeof msg) { 279 fprintf(stderr, "Couldn't send local address\n"); 280 free(rem_dest); 281 rem_dest = NULL; 282 goto out; 283 } 284 285 read(connfd, msg, sizeof msg); 286 287 out: 288 close(connfd); 289 return rem_dest; 290 } 291 292 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 293 int rx_depth, int port, 294 int use_event) 295 { 296 struct pingpong_context *ctx; 297 298 ctx = malloc(sizeof *ctx); 299 if (!ctx) 300 return NULL; 301 302 ctx->size = size; 303 ctx->rx_depth = rx_depth; 304 305 ctx->buf = malloc(roundup(size + 40, page_size)); 306 if (!ctx->buf) { 307 fprintf(stderr, "Couldn't allocate work buf.\n"); 308 return NULL; 309 } 310 311 memset(ctx->buf, 0, size + 40); 312 313 ctx->context = ibv_open_device(ib_dev); 314 if (!ctx->context) { 315 fprintf(stderr, "Couldn't get context for %s\n", 316 ibv_get_device_name(ib_dev)); 317 return NULL; 318 } 319 320 if (use_event) { 321 ctx->channel = ibv_create_comp_channel(ctx->context); 322 if (!ctx->channel) { 323 fprintf(stderr, "Couldn't create completion channel\n"); 324 return NULL; 325 } 326 } else 327 ctx->channel = NULL; 328 329 ctx->pd = ibv_alloc_pd(ctx->context); 330 if (!ctx->pd) { 331 fprintf(stderr, "Couldn't allocate PD\n"); 332 return NULL; 333 } 334 335 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE); 336 if (!ctx->mr) { 337 fprintf(stderr, "Couldn't register MR\n"); 338 return NULL; 339 } 340 341 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, 342 ctx->channel, 0); 343 if (!ctx->cq) { 344 fprintf(stderr, "Couldn't create CQ\n"); 345 return NULL; 346 } 347 348 { 349 struct ibv_qp_init_attr attr = { 350 .send_cq = ctx->cq, 351 .recv_cq = ctx->cq, 352 .cap = { 353 .max_send_wr = 1, 354 .max_recv_wr = rx_depth, 355 .max_send_sge = 1, 356 .max_recv_sge = 1 357 }, 358 .qp_type = IBV_QPT_UD, 359 }; 360 361 ctx->qp = ibv_create_qp(ctx->pd, &attr); 362 if (!ctx->qp) { 363 fprintf(stderr, "Couldn't create QP\n"); 364 return NULL; 365 } 366 } 367 368 { 369 struct ibv_qp_attr attr = { 370 .qp_state = IBV_QPS_INIT, 371 .pkey_index = 0, 372 .port_num = port, 373 .qkey = 0x11111111 374 }; 375 376 if (ibv_modify_qp(ctx->qp, &attr, 377 IBV_QP_STATE | 378 IBV_QP_PKEY_INDEX | 379 IBV_QP_PORT | 380 IBV_QP_QKEY)) { 381 fprintf(stderr, "Failed to modify QP to INIT\n"); 382 return NULL; 383 } 384 } 385 386 return ctx; 387 } 388 389 int pp_close_ctx(struct pingpong_context *ctx) 390 { 391 if (ibv_destroy_qp(ctx->qp)) { 392 fprintf(stderr, "Couldn't destroy QP\n"); 393 return 1; 394 } 395 396 if (ibv_destroy_cq(ctx->cq)) { 397 fprintf(stderr, "Couldn't destroy CQ\n"); 398 return 1; 399 } 400 401 if (ibv_dereg_mr(ctx->mr)) { 402 fprintf(stderr, "Couldn't deregister MR\n"); 403 return 1; 404 } 405 406 if (ibv_destroy_ah(ctx->ah)) { 407 fprintf(stderr, "Couldn't destroy AH\n"); 408 return 1; 409 } 410 411 if (ibv_dealloc_pd(ctx->pd)) { 412 fprintf(stderr, "Couldn't deallocate PD\n"); 413 return 1; 414 } 415 416 if (ctx->channel) { 417 if (ibv_destroy_comp_channel(ctx->channel)) { 418 fprintf(stderr, "Couldn't destroy completion channel\n"); 419 return 1; 420 } 421 } 422 423 if (ibv_close_device(ctx->context)) { 424 fprintf(stderr, "Couldn't release context\n"); 425 return 1; 426 } 427 428 free(ctx->buf); 429 free(ctx); 430 431 return 0; 432 } 433 434 static int pp_post_recv(struct pingpong_context *ctx, int n) 435 { 436 struct ibv_sge list = { 437 .addr = (uintptr_t) ctx->buf, 438 .length = ctx->size + 40, 439 .lkey = ctx->mr->lkey 440 }; 441 struct ibv_recv_wr wr = { 442 .wr_id = PINGPONG_RECV_WRID, 443 .sg_list = &list, 444 .num_sge = 1, 445 }; 446 struct ibv_recv_wr *bad_wr; 447 int i; 448 449 for (i = 0; i < n; ++i) 450 if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) 451 break; 452 453 return i; 454 } 455 456 static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn) 457 { 458 struct ibv_sge list = { 459 .addr = (uintptr_t) ctx->buf + 40, 460 .length = ctx->size, 461 .lkey = ctx->mr->lkey 462 }; 463 struct ibv_send_wr wr = { 464 .wr_id = PINGPONG_SEND_WRID, 465 .sg_list = &list, 466 .num_sge = 1, 467 .opcode = IBV_WR_SEND, 468 .send_flags = IBV_SEND_SIGNALED, 469 .wr = { 470 .ud = { 471 .ah = ctx->ah, 472 .remote_qpn = qpn, 473 .remote_qkey = 0x11111111 474 } 475 } 476 }; 477 struct ibv_send_wr *bad_wr; 478 479 return ibv_post_send(ctx->qp, &wr, &bad_wr); 480 } 481 482 static void usage(const char *argv0) 483 { 484 printf("Usage:\n"); 485 printf(" %s start a server and wait for connection\n", argv0); 486 printf(" %s <host> connect to server at <host>\n", argv0); 487 printf("\n"); 488 printf("Options:\n"); 489 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 490 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 491 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 492 printf(" -s, --size=<size> size of message to exchange (default 1024)\n"); 493 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 494 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 495 printf(" -e, --events sleep on CQ events (default poll)\n"); 496 printf(" -g, --gid-idx=<gid index> local port gid index\n"); 497 } 498 499 int main(int argc, char *argv[]) 500 { 501 struct ibv_device **dev_list; 502 struct ibv_device *ib_dev; 503 struct pingpong_context *ctx; 504 struct pingpong_dest my_dest; 505 struct pingpong_dest *rem_dest; 506 struct timeval start, end; 507 char *ib_devname = NULL; 508 char *servername = NULL; 509 int port = 18515; 510 int ib_port = 1; 511 int size = 1024; 512 int rx_depth = 500; 513 int iters = 1000; 514 int use_event = 0; 515 int routs; 516 int rcnt, scnt; 517 int num_cq_events = 0; 518 int sl = 0; 519 int gidx = -1; 520 char gid[33]; 521 522 srand48(getpid() * time(NULL)); 523 524 while (1) { 525 int c; 526 527 static struct option long_options[] = { 528 { .name = "port", .has_arg = 1, .val = 'p' }, 529 { .name = "ib-dev", .has_arg = 1, .val = 'd' }, 530 { .name = "ib-port", .has_arg = 1, .val = 'i' }, 531 { .name = "size", .has_arg = 1, .val = 's' }, 532 { .name = "rx-depth", .has_arg = 1, .val = 'r' }, 533 { .name = "iters", .has_arg = 1, .val = 'n' }, 534 { .name = "sl", .has_arg = 1, .val = 'l' }, 535 { .name = "events", .has_arg = 0, .val = 'e' }, 536 { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 537 { 0 } 538 }; 539 540 c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL); 541 if (c == -1) 542 break; 543 544 switch (c) { 545 case 'p': 546 port = strtol(optarg, NULL, 0); 547 if (port < 0 || port > 65535) { 548 usage(argv[0]); 549 return 1; 550 } 551 break; 552 553 case 'd': 554 ib_devname = strdup(optarg); 555 break; 556 557 case 'i': 558 ib_port = strtol(optarg, NULL, 0); 559 if (ib_port < 0) { 560 usage(argv[0]); 561 return 1; 562 } 563 break; 564 565 case 's': 566 size = strtol(optarg, NULL, 0); 567 break; 568 569 case 'r': 570 rx_depth = strtol(optarg, NULL, 0); 571 break; 572 573 case 'n': 574 iters = strtol(optarg, NULL, 0); 575 break; 576 577 case 'l': 578 sl = strtol(optarg, NULL, 0); 579 break; 580 581 case 'e': 582 ++use_event; 583 break; 584 585 case 'g': 586 gidx = strtol(optarg, NULL, 0); 587 break; 588 589 default: 590 usage(argv[0]); 591 return 1; 592 } 593 } 594 595 if (optind == argc - 1) 596 servername = strdup(argv[optind]); 597 else if (optind < argc) { 598 usage(argv[0]); 599 return 1; 600 } 601 602 page_size = sysconf(_SC_PAGESIZE); 603 604 dev_list = ibv_get_device_list(NULL); 605 if (!dev_list) { 606 perror("Failed to get IB devices list"); 607 return 1; 608 } 609 610 if (!ib_devname) { 611 ib_dev = *dev_list; 612 if (!ib_dev) { 613 fprintf(stderr, "No IB devices found\n"); 614 return 1; 615 } 616 } else { 617 int i; 618 for (i = 0; dev_list[i]; ++i) 619 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) 620 break; 621 ib_dev = dev_list[i]; 622 if (!ib_dev) { 623 fprintf(stderr, "IB device %s not found\n", ib_devname); 624 return 1; 625 } 626 } 627 628 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 629 if (!ctx) 630 return 1; 631 632 routs = pp_post_recv(ctx, ctx->rx_depth); 633 if (routs < ctx->rx_depth) { 634 fprintf(stderr, "Couldn't post receive (%d)\n", routs); 635 return 1; 636 } 637 638 if (use_event) 639 if (ibv_req_notify_cq(ctx->cq, 0)) { 640 fprintf(stderr, "Couldn't request CQ notification\n"); 641 return 1; 642 } 643 644 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 645 fprintf(stderr, "Couldn't get port info\n"); 646 return 1; 647 } 648 my_dest.lid = ctx->portinfo.lid; 649 650 my_dest.qpn = ctx->qp->qp_num; 651 my_dest.psn = lrand48() & 0xffffff; 652 653 if (gidx >= 0) { 654 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 655 fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 656 return 1; 657 } 658 } else 659 memset(&my_dest.gid, 0, sizeof my_dest.gid); 660 661 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 662 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", 663 my_dest.lid, my_dest.qpn, my_dest.psn, gid); 664 665 if (servername) 666 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 667 else 668 rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx); 669 670 if (!rem_dest) 671 return 1; 672 673 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 674 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 675 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 676 677 if (servername) 678 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx)) 679 return 1; 680 681 ctx->pending = PINGPONG_RECV_WRID; 682 683 if (servername) { 684 if (pp_post_send(ctx, rem_dest->qpn)) { 685 fprintf(stderr, "Couldn't post send\n"); 686 return 1; 687 } 688 ctx->pending |= PINGPONG_SEND_WRID; 689 } 690 691 if (gettimeofday(&start, NULL)) { 692 perror("gettimeofday"); 693 return 1; 694 } 695 696 rcnt = scnt = 0; 697 while (rcnt < iters || scnt < iters) { 698 if (use_event) { 699 struct ibv_cq *ev_cq; 700 void *ev_ctx; 701 702 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { 703 fprintf(stderr, "Failed to get cq_event\n"); 704 return 1; 705 } 706 707 ++num_cq_events; 708 709 if (ev_cq != ctx->cq) { 710 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); 711 return 1; 712 } 713 714 if (ibv_req_notify_cq(ctx->cq, 0)) { 715 fprintf(stderr, "Couldn't request CQ notification\n"); 716 return 1; 717 } 718 } 719 720 { 721 struct ibv_wc wc[2]; 722 int ne, i; 723 724 do { 725 ne = ibv_poll_cq(ctx->cq, 2, wc); 726 if (ne < 0) { 727 fprintf(stderr, "poll CQ failed %d\n", ne); 728 return 1; 729 } 730 } while (!use_event && ne < 1); 731 732 for (i = 0; i < ne; ++i) { 733 if (wc[i].status != IBV_WC_SUCCESS) { 734 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", 735 ibv_wc_status_str(wc[i].status), 736 wc[i].status, (int) wc[i].wr_id); 737 return 1; 738 } 739 740 switch ((int) wc[i].wr_id) { 741 case PINGPONG_SEND_WRID: 742 ++scnt; 743 break; 744 745 case PINGPONG_RECV_WRID: 746 if (--routs <= 1) { 747 routs += pp_post_recv(ctx, ctx->rx_depth - routs); 748 if (routs < ctx->rx_depth) { 749 fprintf(stderr, 750 "Couldn't post receive (%d)\n", 751 routs); 752 return 1; 753 } 754 } 755 756 ++rcnt; 757 break; 758 759 default: 760 fprintf(stderr, "Completion for unknown wr_id %d\n", 761 (int) wc[i].wr_id); 762 return 1; 763 } 764 765 ctx->pending &= ~(int) wc[i].wr_id; 766 if (scnt < iters && !ctx->pending) { 767 if (pp_post_send(ctx, rem_dest->qpn)) { 768 fprintf(stderr, "Couldn't post send\n"); 769 return 1; 770 } 771 ctx->pending = PINGPONG_RECV_WRID | 772 PINGPONG_SEND_WRID; 773 } 774 } 775 } 776 } 777 778 if (gettimeofday(&end, NULL)) { 779 perror("gettimeofday"); 780 return 1; 781 } 782 783 { 784 float usec = (end.tv_sec - start.tv_sec) * 1000000 + 785 (end.tv_usec - start.tv_usec); 786 long long bytes = (long long) size * iters * 2; 787 788 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", 789 bytes, usec / 1000000., bytes * 8. / usec); 790 printf("%d iters in %.2f seconds = %.2f usec/iter\n", 791 iters, usec / 1000000., usec / iters); 792 } 793 794 ibv_ack_cq_events(ctx->cq, num_cq_events); 795 796 if (pp_close_ctx(ctx)) 797 return 1; 798 799 ibv_free_device_list(dev_list); 800 free(rem_dest); 801 802 return 0; 803 } 804