1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #define _GNU_SOURCE
33 #include <config.h>
34
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <unistd.h>
38 #include <string.h>
39 #include <sys/types.h>
40 #include <sys/socket.h>
41 #include <sys/time.h>
42 #include <netdb.h>
43 #include <stdlib.h>
44 #include <getopt.h>
45 #include <arpa/inet.h>
46 #include <time.h>
47
48 #include "pingpong.h"
49
50 enum {
51 PINGPONG_RECV_WRID = 1,
52 PINGPONG_SEND_WRID = 2,
53 };
54
55 static int page_size;
56
57 struct pingpong_context {
58 struct ibv_context *context;
59 struct ibv_comp_channel *channel;
60 struct ibv_pd *pd;
61 struct ibv_mr *mr;
62 struct ibv_cq *cq;
63 struct ibv_qp *qp;
64 struct ibv_ah *ah;
65 void *buf;
66 int size;
67 int send_flags;
68 int rx_depth;
69 int pending;
70 struct ibv_port_attr portinfo;
71 };
72
73 struct pingpong_dest {
74 int lid;
75 int qpn;
76 int psn;
77 union ibv_gid gid;
78 };
79
pp_connect_ctx(struct pingpong_context * ctx,int port,int my_psn,int sl,struct pingpong_dest * dest,int sgid_idx)80 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
81 int sl, struct pingpong_dest *dest, int sgid_idx)
82 {
83 struct ibv_ah_attr ah_attr = {
84 .is_global = 0,
85 .dlid = dest->lid,
86 .sl = sl,
87 .src_path_bits = 0,
88 .port_num = port
89 };
90 struct ibv_qp_attr attr = {
91 .qp_state = IBV_QPS_RTR
92 };
93
94 if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
95 fprintf(stderr, "Failed to modify QP to RTR\n");
96 return 1;
97 }
98
99 attr.qp_state = IBV_QPS_RTS;
100 attr.sq_psn = my_psn;
101
102 if (ibv_modify_qp(ctx->qp, &attr,
103 IBV_QP_STATE |
104 IBV_QP_SQ_PSN)) {
105 fprintf(stderr, "Failed to modify QP to RTS\n");
106 return 1;
107 }
108
109 if (dest->gid.global.interface_id) {
110 ah_attr.is_global = 1;
111 ah_attr.grh.hop_limit = 1;
112 ah_attr.grh.dgid = dest->gid;
113 ah_attr.grh.sgid_index = sgid_idx;
114 }
115
116 ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
117 if (!ctx->ah) {
118 fprintf(stderr, "Failed to create AH\n");
119 return 1;
120 }
121
122 return 0;
123 }
124
pp_client_exch_dest(const char * servername,int port,const struct pingpong_dest * my_dest)125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
126 const struct pingpong_dest *my_dest)
127 {
128 struct addrinfo *res, *t;
129 struct addrinfo hints = {
130 .ai_family = AF_UNSPEC,
131 .ai_socktype = SOCK_STREAM
132 };
133 char *service;
134 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
135 int n;
136 int sockfd = -1;
137 struct pingpong_dest *rem_dest = NULL;
138 char gid[33];
139
140 if (asprintf(&service, "%d", port) < 0)
141 return NULL;
142
143 n = getaddrinfo(servername, service, &hints, &res);
144
145 if (n < 0) {
146 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
147 free(service);
148 return NULL;
149 }
150
151 for (t = res; t; t = t->ai_next) {
152 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
153 if (sockfd >= 0) {
154 if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
155 break;
156 close(sockfd);
157 sockfd = -1;
158 }
159 }
160
161 freeaddrinfo_null(res);
162 free(service);
163
164 if (sockfd < 0) {
165 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
166 return NULL;
167 }
168
169 gid_to_wire_gid(&my_dest->gid, gid);
170 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
171 my_dest->psn, gid);
172 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
173 fprintf(stderr, "Couldn't send local address\n");
174 goto out;
175 }
176
177 if (read(sockfd, msg, sizeof msg) != sizeof msg ||
178 write(sockfd, "done", sizeof "done") != sizeof "done") {
179 perror("client read/write");
180 fprintf(stderr, "Couldn't read/write remote address\n");
181 goto out;
182 }
183
184 rem_dest = malloc(sizeof *rem_dest);
185 if (!rem_dest)
186 goto out;
187
188 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
189 &rem_dest->psn, gid);
190 wire_gid_to_gid(gid, &rem_dest->gid);
191
192 out:
193 close(sockfd);
194 return rem_dest;
195 }
196
pp_server_exch_dest(struct pingpong_context * ctx,int ib_port,int port,int sl,const struct pingpong_dest * my_dest,int sgid_idx)197 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
198 int ib_port, int port, int sl,
199 const struct pingpong_dest *my_dest,
200 int sgid_idx)
201 {
202 struct addrinfo *res, *t;
203 struct addrinfo hints = {
204 .ai_flags = AI_PASSIVE,
205 .ai_family = AF_UNSPEC,
206 .ai_socktype = SOCK_STREAM
207 };
208 char *service;
209 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
210 int n;
211 int sockfd = -1, connfd;
212 struct pingpong_dest *rem_dest = NULL;
213 char gid[33];
214
215 if (asprintf(&service, "%d", port) < 0)
216 return NULL;
217
218 n = getaddrinfo(NULL, service, &hints, &res);
219
220 if (n < 0) {
221 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
222 free(service);
223 return NULL;
224 }
225
226 for (t = res; t; t = t->ai_next) {
227 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
228 if (sockfd >= 0) {
229 n = 1;
230
231 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
232
233 if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
234 break;
235 close(sockfd);
236 sockfd = -1;
237 }
238 }
239
240 freeaddrinfo_null(res);
241 free(service);
242
243 if (sockfd < 0) {
244 fprintf(stderr, "Couldn't listen to port %d\n", port);
245 return NULL;
246 }
247
248 if (listen(sockfd, 1) < 0) {
249 perror("listen() failed");
250 close(sockfd);
251 return NULL;
252 }
253 connfd = accept(sockfd, NULL, NULL);
254 close(sockfd);
255 if (connfd < 0) {
256 fprintf(stderr, "accept() failed\n");
257 return NULL;
258 }
259
260 n = read(connfd, msg, sizeof msg);
261 if (n != sizeof msg) {
262 perror("server read");
263 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
264 goto out;
265 }
266
267 rem_dest = malloc(sizeof *rem_dest);
268 if (!rem_dest)
269 goto out;
270
271 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
272 &rem_dest->psn, gid);
273 wire_gid_to_gid(gid, &rem_dest->gid);
274
275 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
276 sgid_idx)) {
277 fprintf(stderr, "Couldn't connect to remote QP\n");
278 free(rem_dest);
279 rem_dest = NULL;
280 goto out;
281 }
282
283 gid_to_wire_gid(&my_dest->gid, gid);
284 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
285 my_dest->psn, gid);
286 if (write(connfd, msg, sizeof msg) != sizeof msg ||
287 read(connfd, msg, sizeof msg) != sizeof "done") {
288 fprintf(stderr, "Couldn't send/recv local address\n");
289 free(rem_dest);
290 rem_dest = NULL;
291 goto out;
292 }
293 out:
294 close(connfd);
295 return rem_dest;
296 }
297
pp_init_ctx(struct ibv_device * ib_dev,int size,int rx_depth,int port,int use_event)298 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
299 int rx_depth, int port,
300 int use_event)
301 {
302 struct pingpong_context *ctx;
303
304 ctx = malloc(sizeof *ctx);
305 if (!ctx)
306 return NULL;
307
308 ctx->size = size;
309 ctx->send_flags = IBV_SEND_SIGNALED;
310 ctx->rx_depth = rx_depth;
311
312 ctx->buf = memalign(page_size, size + 40);
313 if (!ctx->buf) {
314 fprintf(stderr, "Couldn't allocate work buf.\n");
315 goto clean_ctx;
316 }
317
318 /* FIXME memset(ctx->buf, 0, size + 40); */
319 memset(ctx->buf, 0x7b, size + 40);
320
321 ctx->context = ibv_open_device(ib_dev);
322 if (!ctx->context) {
323 fprintf(stderr, "Couldn't get context for %s\n",
324 ibv_get_device_name(ib_dev));
325 goto clean_buffer;
326 }
327
328 {
329 struct ibv_port_attr port_info = {};
330 int mtu;
331
332 if (ibv_query_port(ctx->context, port, &port_info)) {
333 fprintf(stderr, "Unable to query port info for port %d\n", port);
334 goto clean_device;
335 }
336 mtu = 1 << (port_info.active_mtu + 7);
337 if (size > mtu) {
338 fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu);
339 goto clean_device;
340 }
341 }
342
343 if (use_event) {
344 ctx->channel = ibv_create_comp_channel(ctx->context);
345 if (!ctx->channel) {
346 fprintf(stderr, "Couldn't create completion channel\n");
347 goto clean_device;
348 }
349 } else
350 ctx->channel = NULL;
351
352 ctx->pd = ibv_alloc_pd(ctx->context);
353 if (!ctx->pd) {
354 fprintf(stderr, "Couldn't allocate PD\n");
355 goto clean_comp_channel;
356 }
357
358 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
359 if (!ctx->mr) {
360 fprintf(stderr, "Couldn't register MR\n");
361 goto clean_pd;
362 }
363
364 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
365 ctx->channel, 0);
366 if (!ctx->cq) {
367 fprintf(stderr, "Couldn't create CQ\n");
368 goto clean_mr;
369 }
370
371 {
372 struct ibv_qp_attr attr;
373 struct ibv_qp_init_attr init_attr = {
374 .send_cq = ctx->cq,
375 .recv_cq = ctx->cq,
376 .cap = {
377 .max_send_wr = 1,
378 .max_recv_wr = rx_depth,
379 .max_send_sge = 1,
380 .max_recv_sge = 1
381 },
382 .qp_type = IBV_QPT_UD,
383 };
384
385 ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
386 if (!ctx->qp) {
387 fprintf(stderr, "Couldn't create QP\n");
388 goto clean_cq;
389 }
390
391 ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
392 if (init_attr.cap.max_inline_data >= size) {
393 ctx->send_flags |= IBV_SEND_INLINE;
394 }
395 }
396
397 {
398 struct ibv_qp_attr attr = {
399 .qp_state = IBV_QPS_INIT,
400 .pkey_index = 0,
401 .port_num = port,
402 .qkey = 0x11111111
403 };
404
405 if (ibv_modify_qp(ctx->qp, &attr,
406 IBV_QP_STATE |
407 IBV_QP_PKEY_INDEX |
408 IBV_QP_PORT |
409 IBV_QP_QKEY)) {
410 fprintf(stderr, "Failed to modify QP to INIT\n");
411 goto clean_qp;
412 }
413 }
414
415 return ctx;
416
417 clean_qp:
418 ibv_destroy_qp(ctx->qp);
419
420 clean_cq:
421 ibv_destroy_cq(ctx->cq);
422
423 clean_mr:
424 ibv_dereg_mr(ctx->mr);
425
426 clean_pd:
427 ibv_dealloc_pd(ctx->pd);
428
429 clean_comp_channel:
430 if (ctx->channel)
431 ibv_destroy_comp_channel(ctx->channel);
432
433 clean_device:
434 ibv_close_device(ctx->context);
435
436 clean_buffer:
437 free(ctx->buf);
438
439 clean_ctx:
440 free(ctx);
441
442 return NULL;
443 }
444
pp_close_ctx(struct pingpong_context * ctx)445 static int pp_close_ctx(struct pingpong_context *ctx)
446 {
447 if (ibv_destroy_qp(ctx->qp)) {
448 fprintf(stderr, "Couldn't destroy QP\n");
449 return 1;
450 }
451
452 if (ibv_destroy_cq(ctx->cq)) {
453 fprintf(stderr, "Couldn't destroy CQ\n");
454 return 1;
455 }
456
457 if (ibv_dereg_mr(ctx->mr)) {
458 fprintf(stderr, "Couldn't deregister MR\n");
459 return 1;
460 }
461
462 if (ibv_destroy_ah(ctx->ah)) {
463 fprintf(stderr, "Couldn't destroy AH\n");
464 return 1;
465 }
466
467 if (ibv_dealloc_pd(ctx->pd)) {
468 fprintf(stderr, "Couldn't deallocate PD\n");
469 return 1;
470 }
471
472 if (ctx->channel) {
473 if (ibv_destroy_comp_channel(ctx->channel)) {
474 fprintf(stderr, "Couldn't destroy completion channel\n");
475 return 1;
476 }
477 }
478
479 if (ibv_close_device(ctx->context)) {
480 fprintf(stderr, "Couldn't release context\n");
481 return 1;
482 }
483
484 free(ctx->buf);
485 free(ctx);
486
487 return 0;
488 }
489
pp_post_recv(struct pingpong_context * ctx,int n)490 static int pp_post_recv(struct pingpong_context *ctx, int n)
491 {
492 struct ibv_sge list = {
493 .addr = (uintptr_t) ctx->buf,
494 .length = ctx->size + 40,
495 .lkey = ctx->mr->lkey
496 };
497 struct ibv_recv_wr wr = {
498 .wr_id = PINGPONG_RECV_WRID,
499 .sg_list = &list,
500 .num_sge = 1,
501 };
502 struct ibv_recv_wr *bad_wr;
503 int i;
504
505 for (i = 0; i < n; ++i)
506 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
507 break;
508
509 return i;
510 }
511
pp_post_send(struct pingpong_context * ctx,uint32_t qpn)512 static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
513 {
514 struct ibv_sge list = {
515 .addr = (uintptr_t) ctx->buf + 40,
516 .length = ctx->size,
517 .lkey = ctx->mr->lkey
518 };
519 struct ibv_send_wr wr = {
520 .wr_id = PINGPONG_SEND_WRID,
521 .sg_list = &list,
522 .num_sge = 1,
523 .opcode = IBV_WR_SEND,
524 .send_flags = ctx->send_flags,
525 .wr = {
526 .ud = {
527 .ah = ctx->ah,
528 .remote_qpn = qpn,
529 .remote_qkey = 0x11111111
530 }
531 }
532 };
533 struct ibv_send_wr *bad_wr;
534
535 return ibv_post_send(ctx->qp, &wr, &bad_wr);
536 }
537
usage(const char * argv0)538 static void usage(const char *argv0)
539 {
540 printf("Usage:\n");
541 printf(" %s start a server and wait for connection\n", argv0);
542 printf(" %s <host> connect to server at <host>\n", argv0);
543 printf("\n");
544 printf("Options:\n");
545 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
546 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
547 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
548 printf(" -s, --size=<size> size of message to exchange (default 2048)\n");
549 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
550 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
551 printf(" -l, --sl=<SL> send messages with service level <SL> (default 0)\n");
552 printf(" -e, --events sleep on CQ events (default poll)\n");
553 printf(" -g, --gid-idx=<gid index> local port gid index\n");
554 }
555
main(int argc,char * argv[])556 int main(int argc, char *argv[])
557 {
558 struct ibv_device **dev_list;
559 struct ibv_device *ib_dev;
560 struct pingpong_context *ctx;
561 struct pingpong_dest my_dest;
562 struct pingpong_dest *rem_dest;
563 struct timeval start, end;
564 char *ib_devname = NULL;
565 char *servername = NULL;
566 unsigned int port = 18515;
567 int ib_port = 1;
568 unsigned int size = 2048;
569 unsigned int rx_depth = 500;
570 unsigned int iters = 1000;
571 int use_event = 0;
572 int routs;
573 int rcnt, scnt;
574 int num_cq_events = 0;
575 int sl = 0;
576 int gidx = -1;
577 char gid[33];
578
579 srand48(getpid() * time(NULL));
580
581 while (1) {
582 int c;
583
584 static struct option long_options[] = {
585 { .name = "port", .has_arg = 1, .val = 'p' },
586 { .name = "ib-dev", .has_arg = 1, .val = 'd' },
587 { .name = "ib-port", .has_arg = 1, .val = 'i' },
588 { .name = "size", .has_arg = 1, .val = 's' },
589 { .name = "rx-depth", .has_arg = 1, .val = 'r' },
590 { .name = "iters", .has_arg = 1, .val = 'n' },
591 { .name = "sl", .has_arg = 1, .val = 'l' },
592 { .name = "events", .has_arg = 0, .val = 'e' },
593 { .name = "gid-idx", .has_arg = 1, .val = 'g' },
594 {}
595 };
596
597 c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:",
598 long_options, NULL);
599 if (c == -1)
600 break;
601
602 switch (c) {
603 case 'p':
604 port = strtol(optarg, NULL, 0);
605 if (port > 65535) {
606 usage(argv[0]);
607 return 1;
608 }
609 break;
610
611 case 'd':
612 ib_devname = strdupa(optarg);
613 break;
614
615 case 'i':
616 ib_port = strtol(optarg, NULL, 0);
617 if (ib_port < 1) {
618 usage(argv[0]);
619 return 1;
620 }
621 break;
622
623 case 's':
624 size = strtoul(optarg, NULL, 0);
625 break;
626
627 case 'r':
628 rx_depth = strtoul(optarg, NULL, 0);
629 break;
630
631 case 'n':
632 iters = strtoul(optarg, NULL, 0);
633 break;
634
635 case 'l':
636 sl = strtol(optarg, NULL, 0);
637 break;
638
639 case 'e':
640 ++use_event;
641 break;
642
643 case 'g':
644 gidx = strtol(optarg, NULL, 0);
645 break;
646
647 default:
648 usage(argv[0]);
649 return 1;
650 }
651 }
652
653 if (optind == argc - 1)
654 servername = strdupa(argv[optind]);
655 else if (optind < argc) {
656 usage(argv[0]);
657 return 1;
658 }
659
660 page_size = sysconf(_SC_PAGESIZE);
661
662 dev_list = ibv_get_device_list(NULL);
663 if (!dev_list) {
664 perror("Failed to get IB devices list");
665 return 1;
666 }
667
668 if (!ib_devname) {
669 ib_dev = *dev_list;
670 if (!ib_dev) {
671 fprintf(stderr, "No IB devices found\n");
672 return 1;
673 }
674 } else {
675 int i;
676 for (i = 0; dev_list[i]; ++i)
677 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
678 break;
679 ib_dev = dev_list[i];
680 if (!ib_dev) {
681 fprintf(stderr, "IB device %s not found\n", ib_devname);
682 return 1;
683 }
684 }
685
686 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
687 if (!ctx)
688 return 1;
689
690 routs = pp_post_recv(ctx, ctx->rx_depth);
691 if (routs < ctx->rx_depth) {
692 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
693 return 1;
694 }
695
696 if (use_event)
697 if (ibv_req_notify_cq(ctx->cq, 0)) {
698 fprintf(stderr, "Couldn't request CQ notification\n");
699 return 1;
700 }
701
702 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
703 fprintf(stderr, "Couldn't get port info\n");
704 return 1;
705 }
706 my_dest.lid = ctx->portinfo.lid;
707
708 my_dest.qpn = ctx->qp->qp_num;
709 my_dest.psn = lrand48() & 0xffffff;
710
711 if (gidx >= 0) {
712 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
713 fprintf(stderr, "Could not get local gid for gid index "
714 "%d\n", gidx);
715 return 1;
716 }
717 } else
718 memset(&my_dest.gid, 0, sizeof my_dest.gid);
719
720 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
721 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
722 my_dest.lid, my_dest.qpn, my_dest.psn, gid);
723
724 if (servername)
725 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
726 else
727 rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl,
728 &my_dest, gidx);
729
730 if (!rem_dest)
731 return 1;
732
733 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
734 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
735 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
736
737 if (servername)
738 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest,
739 gidx))
740 return 1;
741
742 ctx->pending = PINGPONG_RECV_WRID;
743
744 if (servername) {
745 if (pp_post_send(ctx, rem_dest->qpn)) {
746 fprintf(stderr, "Couldn't post send\n");
747 return 1;
748 }
749 ctx->pending |= PINGPONG_SEND_WRID;
750 }
751
752 if (gettimeofday(&start, NULL)) {
753 perror("gettimeofday");
754 return 1;
755 }
756
757 rcnt = scnt = 0;
758 while (rcnt < iters || scnt < iters) {
759 if (use_event) {
760 struct ibv_cq *ev_cq;
761 void *ev_ctx;
762
763 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
764 fprintf(stderr, "Failed to get cq_event\n");
765 return 1;
766 }
767
768 ++num_cq_events;
769
770 if (ev_cq != ctx->cq) {
771 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
772 return 1;
773 }
774
775 if (ibv_req_notify_cq(ctx->cq, 0)) {
776 fprintf(stderr, "Couldn't request CQ notification\n");
777 return 1;
778 }
779 }
780
781 {
782 struct ibv_wc wc[2];
783 int ne, i;
784
785 do {
786 ne = ibv_poll_cq(ctx->cq, 2, wc);
787 if (ne < 0) {
788 fprintf(stderr, "poll CQ failed %d\n", ne);
789 return 1;
790 }
791 } while (!use_event && ne < 1);
792
793 for (i = 0; i < ne; ++i) {
794 if (wc[i].status != IBV_WC_SUCCESS) {
795 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
796 ibv_wc_status_str(wc[i].status),
797 wc[i].status, (int) wc[i].wr_id);
798 return 1;
799 }
800
801 switch ((int) wc[i].wr_id) {
802 case PINGPONG_SEND_WRID:
803 ++scnt;
804 break;
805
806 case PINGPONG_RECV_WRID:
807 if (--routs <= 1) {
808 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
809 if (routs < ctx->rx_depth) {
810 fprintf(stderr,
811 "Couldn't post receive (%d)\n",
812 routs);
813 return 1;
814 }
815 }
816
817 ++rcnt;
818 break;
819
820 default:
821 fprintf(stderr, "Completion for unknown wr_id %d\n",
822 (int) wc[i].wr_id);
823 return 1;
824 }
825
826 ctx->pending &= ~(int) wc[i].wr_id;
827 if (scnt < iters && !ctx->pending) {
828 if (pp_post_send(ctx, rem_dest->qpn)) {
829 fprintf(stderr, "Couldn't post send\n");
830 return 1;
831 }
832 ctx->pending = PINGPONG_RECV_WRID |
833 PINGPONG_SEND_WRID;
834 }
835 }
836 }
837 }
838
839 if (gettimeofday(&end, NULL)) {
840 perror("gettimeofday");
841 return 1;
842 }
843
844 {
845 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
846 (end.tv_usec - start.tv_usec);
847 long long bytes = (long long) size * iters * 2;
848
849 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
850 bytes, usec / 1000000., bytes * 8. / usec);
851 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
852 iters, usec / 1000000., usec / iters);
853 }
854
855 ibv_ack_cq_events(ctx->cq, num_cq_events);
856
857 if (pp_close_ctx(ctx))
858 return 1;
859
860 ibv_free_device_list(dev_list);
861 free(rem_dest);
862
863 return 0;
864 }
865