1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #define _GNU_SOURCE
33 #include <config.h>
34
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <unistd.h>
38 #include <string.h>
39 #include <sys/types.h>
40 #include <sys/socket.h>
41 #include <sys/time.h>
42 #include <netdb.h>
43 #include <stdlib.h>
44 #include <getopt.h>
45 #include <arpa/inet.h>
46 #include <time.h>
47
48 #include "pingpong.h"
49
50 enum {
51 PINGPONG_RECV_WRID = 1,
52 PINGPONG_SEND_WRID = 2,
53 };
54
55 static int page_size;
56
57 struct pingpong_context {
58 struct ibv_context *context;
59 struct ibv_comp_channel *channel;
60 struct ibv_pd *pd;
61 struct ibv_mr *mr;
62 struct ibv_cq *cq;
63 struct ibv_qp *qp;
64 void *buf;
65 int size;
66 int send_flags;
67 int rx_depth;
68 int pending;
69 struct ibv_port_attr portinfo;
70 };
71
72 struct pingpong_dest {
73 int lid;
74 int qpn;
75 int psn;
76 union ibv_gid gid;
77 };
78
pp_connect_ctx(struct pingpong_context * ctx,int port,int my_psn,enum ibv_mtu mtu,int sl,struct pingpong_dest * dest,int sgid_idx)79 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
80 enum ibv_mtu mtu, int sl,
81 struct pingpong_dest *dest, int sgid_idx)
82 {
83 struct ibv_qp_attr attr = {
84 .qp_state = IBV_QPS_RTR,
85 .path_mtu = mtu,
86 .dest_qp_num = dest->qpn,
87 .rq_psn = dest->psn,
88 .ah_attr = {
89 .is_global = 0,
90 .dlid = dest->lid,
91 .sl = sl,
92 .src_path_bits = 0,
93 .port_num = port
94 }
95 };
96
97 if (dest->gid.global.interface_id) {
98 attr.ah_attr.is_global = 1;
99 attr.ah_attr.grh.hop_limit = 1;
100 attr.ah_attr.grh.dgid = dest->gid;
101 attr.ah_attr.grh.sgid_index = sgid_idx;
102 }
103 if (ibv_modify_qp(ctx->qp, &attr,
104 IBV_QP_STATE |
105 IBV_QP_AV |
106 IBV_QP_PATH_MTU |
107 IBV_QP_DEST_QPN |
108 IBV_QP_RQ_PSN)) {
109 fprintf(stderr, "Failed to modify QP to RTR\n");
110 return 1;
111 }
112
113 attr.qp_state = IBV_QPS_RTS;
114 attr.sq_psn = my_psn;
115 if (ibv_modify_qp(ctx->qp, &attr,
116 IBV_QP_STATE |
117 IBV_QP_SQ_PSN)) {
118 fprintf(stderr, "Failed to modify QP to RTS\n");
119 return 1;
120 }
121
122 return 0;
123 }
124
pp_client_exch_dest(const char * servername,int port,const struct pingpong_dest * my_dest)125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
126 const struct pingpong_dest *my_dest)
127 {
128 struct addrinfo *res, *t;
129 struct addrinfo hints = {
130 .ai_family = AF_UNSPEC,
131 .ai_socktype = SOCK_STREAM
132 };
133 char *service;
134 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
135 int n;
136 int sockfd = -1;
137 struct pingpong_dest *rem_dest = NULL;
138 char gid[33];
139
140 if (asprintf(&service, "%d", port) < 0)
141 return NULL;
142
143 n = getaddrinfo(servername, service, &hints, &res);
144
145 if (n < 0) {
146 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
147 free(service);
148 return NULL;
149 }
150
151 for (t = res; t; t = t->ai_next) {
152 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
153 if (sockfd >= 0) {
154 if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
155 break;
156 close(sockfd);
157 sockfd = -1;
158 }
159 }
160
161 freeaddrinfo_null(res);
162 free(service);
163
164 if (sockfd < 0) {
165 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
166 return NULL;
167 }
168
169 gid_to_wire_gid(&my_dest->gid, gid);
170 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
171 my_dest->psn, gid);
172 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
173 fprintf(stderr, "Couldn't send local address\n");
174 goto out;
175 }
176
177 if (read(sockfd, msg, sizeof msg) != sizeof msg ||
178 write(sockfd, "done", sizeof "done") != sizeof "done") {
179 perror("client read/write");
180 fprintf(stderr, "Couldn't read/write remote address\n");
181 goto out;
182 }
183
184
185 rem_dest = malloc(sizeof *rem_dest);
186 if (!rem_dest)
187 goto out;
188
189 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
190 &rem_dest->psn, gid);
191 wire_gid_to_gid(gid, &rem_dest->gid);
192
193 out:
194 close(sockfd);
195 return rem_dest;
196 }
197
pp_server_exch_dest(struct pingpong_context * ctx,int ib_port,enum ibv_mtu mtu,int port,int sl,const struct pingpong_dest * my_dest,int sgid_idx)198 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
199 int ib_port, enum ibv_mtu mtu,
200 int port, int sl,
201 const struct pingpong_dest *my_dest,
202 int sgid_idx)
203 {
204 struct addrinfo *res, *t;
205 struct addrinfo hints = {
206 .ai_flags = AI_PASSIVE,
207 .ai_family = AF_UNSPEC,
208 .ai_socktype = SOCK_STREAM
209 };
210 char *service;
211 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
212 int n;
213 int sockfd = -1, connfd;
214 struct pingpong_dest *rem_dest = NULL;
215 char gid[33];
216
217 if (asprintf(&service, "%d", port) < 0)
218 return NULL;
219
220 n = getaddrinfo(NULL, service, &hints, &res);
221
222 if (n < 0) {
223 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
224 free(service);
225 return NULL;
226 }
227
228 for (t = res; t; t = t->ai_next) {
229 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
230 if (sockfd >= 0) {
231 n = 1;
232
233 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
234
235 if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
236 break;
237 close(sockfd);
238 sockfd = -1;
239 }
240 }
241
242 freeaddrinfo_null(res);
243 free(service);
244
245 if (sockfd < 0) {
246 fprintf(stderr, "Couldn't listen to port %d\n", port);
247 return NULL;
248 }
249
250 if (listen(sockfd, 1) < 0) {
251 perror("listen() failed");
252 close(sockfd);
253 return NULL;
254 }
255 connfd = accept(sockfd, NULL, NULL);
256 close(sockfd);
257 if (connfd < 0) {
258 fprintf(stderr, "accept() failed\n");
259 return NULL;
260 }
261
262 n = read(connfd, msg, sizeof msg);
263 if (n != sizeof msg) {
264 perror("server read");
265 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
266 goto out;
267 }
268
269 rem_dest = malloc(sizeof *rem_dest);
270 if (!rem_dest)
271 goto out;
272
273 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
274 &rem_dest->psn, gid);
275 wire_gid_to_gid(gid, &rem_dest->gid);
276
277 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest,
278 sgid_idx)) {
279 fprintf(stderr, "Couldn't connect to remote QP\n");
280 free(rem_dest);
281 rem_dest = NULL;
282 goto out;
283 }
284
285
286 gid_to_wire_gid(&my_dest->gid, gid);
287 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
288 my_dest->psn, gid);
289 if (write(connfd, msg, sizeof msg) != sizeof msg ||
290 read(connfd, msg, sizeof msg) != sizeof "done") {
291 fprintf(stderr, "Couldn't send/recv local address\n");
292 free(rem_dest);
293 rem_dest = NULL;
294 goto out;
295 }
296
297 out:
298 close(connfd);
299 return rem_dest;
300 }
301
pp_init_ctx(struct ibv_device * ib_dev,int size,int rx_depth,int port,int use_event)302 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
303 int rx_depth, int port,
304 int use_event)
305 {
306 struct pingpong_context *ctx;
307
308 ctx = calloc(1, sizeof *ctx);
309 if (!ctx)
310 return NULL;
311
312 ctx->size = size;
313 ctx->send_flags = IBV_SEND_SIGNALED;
314 ctx->rx_depth = rx_depth;
315
316 ctx->buf = memalign(page_size, size);
317 if (!ctx->buf) {
318 fprintf(stderr, "Couldn't allocate work buf.\n");
319 goto clean_ctx;
320 }
321
322 /* FIXME memset(ctx->buf, 0, size); */
323 memset(ctx->buf, 0x7b, size);
324
325 ctx->context = ibv_open_device(ib_dev);
326 if (!ctx->context) {
327 fprintf(stderr, "Couldn't get context for %s\n",
328 ibv_get_device_name(ib_dev));
329 goto clean_buffer;
330 }
331
332 if (use_event) {
333 ctx->channel = ibv_create_comp_channel(ctx->context);
334 if (!ctx->channel) {
335 fprintf(stderr, "Couldn't create completion channel\n");
336 goto clean_device;
337 }
338 } else
339 ctx->channel = NULL;
340
341 ctx->pd = ibv_alloc_pd(ctx->context);
342 if (!ctx->pd) {
343 fprintf(stderr, "Couldn't allocate PD\n");
344 goto clean_comp_channel;
345 }
346
347 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE);
348 if (!ctx->mr) {
349 fprintf(stderr, "Couldn't register MR\n");
350 goto clean_pd;
351 }
352
353 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
354 ctx->channel, 0);
355 if (!ctx->cq) {
356 fprintf(stderr, "Couldn't create CQ\n");
357 goto clean_mr;
358 }
359
360 {
361 struct ibv_qp_attr attr;
362 struct ibv_qp_init_attr init_attr = {
363 .send_cq = ctx->cq,
364 .recv_cq = ctx->cq,
365 .cap = {
366 .max_send_wr = 1,
367 .max_recv_wr = rx_depth,
368 .max_send_sge = 1,
369 .max_recv_sge = 1
370 },
371 .qp_type = IBV_QPT_UC
372 };
373
374 ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
375 if (!ctx->qp) {
376 fprintf(stderr, "Couldn't create QP\n");
377 goto clean_cq;
378 }
379 ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
380 if (init_attr.cap.max_inline_data >= size) {
381 ctx->send_flags |= IBV_SEND_INLINE;
382 }
383 }
384
385 {
386 struct ibv_qp_attr attr = {
387 .qp_state = IBV_QPS_INIT,
388 .pkey_index = 0,
389 .port_num = port,
390 .qp_access_flags = 0
391 };
392
393 if (ibv_modify_qp(ctx->qp, &attr,
394 IBV_QP_STATE |
395 IBV_QP_PKEY_INDEX |
396 IBV_QP_PORT |
397 IBV_QP_ACCESS_FLAGS)) {
398 fprintf(stderr, "Failed to modify QP to INIT\n");
399 goto clean_qp;
400 }
401 }
402
403 return ctx;
404
405 clean_qp:
406 ibv_destroy_qp(ctx->qp);
407
408 clean_cq:
409 ibv_destroy_cq(ctx->cq);
410
411 clean_mr:
412 ibv_dereg_mr(ctx->mr);
413
414 clean_pd:
415 ibv_dealloc_pd(ctx->pd);
416
417 clean_comp_channel:
418 if (ctx->channel)
419 ibv_destroy_comp_channel(ctx->channel);
420
421 clean_device:
422 ibv_close_device(ctx->context);
423
424 clean_buffer:
425 free(ctx->buf);
426
427 clean_ctx:
428 free(ctx);
429
430 return NULL;
431 }
432
pp_close_ctx(struct pingpong_context * ctx)433 static int pp_close_ctx(struct pingpong_context *ctx)
434 {
435 if (ibv_destroy_qp(ctx->qp)) {
436 fprintf(stderr, "Couldn't destroy QP\n");
437 return 1;
438 }
439
440 if (ibv_destroy_cq(ctx->cq)) {
441 fprintf(stderr, "Couldn't destroy CQ\n");
442 return 1;
443 }
444
445 if (ibv_dereg_mr(ctx->mr)) {
446 fprintf(stderr, "Couldn't deregister MR\n");
447 return 1;
448 }
449
450 if (ibv_dealloc_pd(ctx->pd)) {
451 fprintf(stderr, "Couldn't deallocate PD\n");
452 return 1;
453 }
454
455 if (ctx->channel) {
456 if (ibv_destroy_comp_channel(ctx->channel)) {
457 fprintf(stderr, "Couldn't destroy completion channel\n");
458 return 1;
459 }
460 }
461
462 if (ibv_close_device(ctx->context)) {
463 fprintf(stderr, "Couldn't release context\n");
464 return 1;
465 }
466
467 free(ctx->buf);
468 free(ctx);
469
470 return 0;
471 }
472
pp_post_recv(struct pingpong_context * ctx,int n)473 static int pp_post_recv(struct pingpong_context *ctx, int n)
474 {
475 struct ibv_sge list = {
476 .addr = (uintptr_t) ctx->buf,
477 .length = ctx->size,
478 .lkey = ctx->mr->lkey
479 };
480 struct ibv_recv_wr wr = {
481 .wr_id = PINGPONG_RECV_WRID,
482 .sg_list = &list,
483 .num_sge = 1,
484 };
485 struct ibv_recv_wr *bad_wr;
486 int i;
487
488 for (i = 0; i < n; ++i)
489 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
490 break;
491
492 return i;
493 }
494
pp_post_send(struct pingpong_context * ctx)495 static int pp_post_send(struct pingpong_context *ctx)
496 {
497 struct ibv_sge list = {
498 .addr = (uintptr_t) ctx->buf,
499 .length = ctx->size,
500 .lkey = ctx->mr->lkey
501 };
502 struct ibv_send_wr wr = {
503 .wr_id = PINGPONG_SEND_WRID,
504 .sg_list = &list,
505 .num_sge = 1,
506 .opcode = IBV_WR_SEND,
507 .send_flags = ctx->send_flags,
508 };
509 struct ibv_send_wr *bad_wr;
510
511 return ibv_post_send(ctx->qp, &wr, &bad_wr);
512 }
513
usage(const char * argv0)514 static void usage(const char *argv0)
515 {
516 printf("Usage:\n");
517 printf(" %s start a server and wait for connection\n", argv0);
518 printf(" %s <host> connect to server at <host>\n", argv0);
519 printf("\n");
520 printf("Options:\n");
521 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
522 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
523 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
524 printf(" -s, --size=<size> size of message to exchange (default 4096)\n");
525 printf(" -m, --mtu=<size> path MTU (default 1024)\n");
526 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
527 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
528 printf(" -l, --sl=<sl> service level value\n");
529 printf(" -e, --events sleep on CQ events (default poll)\n");
530 printf(" -g, --gid-idx=<gid index> local port gid index\n");
531 }
532
main(int argc,char * argv[])533 int main(int argc, char *argv[])
534 {
535 struct ibv_device **dev_list;
536 struct ibv_device *ib_dev;
537 struct pingpong_context *ctx;
538 struct pingpong_dest my_dest;
539 struct pingpong_dest *rem_dest;
540 struct timeval start, end;
541 char *ib_devname = NULL;
542 char *servername = NULL;
543 unsigned int port = 18515;
544 int ib_port = 1;
545 unsigned int size = 4096;
546 enum ibv_mtu mtu = IBV_MTU_1024;
547 unsigned int rx_depth = 500;
548 unsigned int iters = 1000;
549 int use_event = 0;
550 int routs;
551 int rcnt, scnt;
552 int num_cq_events = 0;
553 int sl = 0;
554 int gidx = -1;
555 char gid[33];
556
557 srand48(getpid() * time(NULL));
558
559 while (1) {
560 int c;
561
562 static struct option long_options[] = {
563 { .name = "port", .has_arg = 1, .val = 'p' },
564 { .name = "ib-dev", .has_arg = 1, .val = 'd' },
565 { .name = "ib-port", .has_arg = 1, .val = 'i' },
566 { .name = "size", .has_arg = 1, .val = 's' },
567 { .name = "mtu", .has_arg = 1, .val = 'm' },
568 { .name = "rx-depth", .has_arg = 1, .val = 'r' },
569 { .name = "iters", .has_arg = 1, .val = 'n' },
570 { .name = "sl", .has_arg = 1, .val = 'l' },
571 { .name = "events", .has_arg = 0, .val = 'e' },
572 { .name = "gid-idx", .has_arg = 1, .val = 'g' },
573 {}
574 };
575
576 c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:",
577 long_options, NULL);
578 if (c == -1)
579 break;
580
581 switch (c) {
582 case 'p':
583 port = strtoul(optarg, NULL, 0);
584 if (port > 65535) {
585 usage(argv[0]);
586 return 1;
587 }
588 break;
589
590 case 'd':
591 ib_devname = strdupa(optarg);
592 break;
593
594 case 'i':
595 ib_port = strtol(optarg, NULL, 0);
596 if (ib_port < 1) {
597 usage(argv[0]);
598 return 1;
599 }
600 break;
601
602 case 's':
603 size = strtoul(optarg, NULL, 0);
604 break;
605
606 case 'm':
607 mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
608 if (mtu == 0) {
609 usage(argv[0]);
610 return 1;
611 }
612 break;
613
614 case 'r':
615 rx_depth = strtoul(optarg, NULL, 0);
616 break;
617
618 case 'n':
619 iters = strtoul(optarg, NULL, 0);
620 break;
621
622 case 'l':
623 sl = strtol(optarg, NULL, 0);
624 break;
625
626 case 'e':
627 ++use_event;
628 break;
629
630 case 'g':
631 gidx = strtol(optarg, NULL, 0);
632 break;
633
634 default:
635 usage(argv[0]);
636 return 1;
637 }
638 }
639
640 if (optind == argc - 1)
641 servername = strdupa(argv[optind]);
642 else if (optind < argc) {
643 usage(argv[0]);
644 return 1;
645 }
646
647 page_size = sysconf(_SC_PAGESIZE);
648
649 dev_list = ibv_get_device_list(NULL);
650 if (!dev_list) {
651 perror("Failed to get IB devices list");
652 return 1;
653 }
654
655 if (!ib_devname) {
656 ib_dev = *dev_list;
657 if (!ib_dev) {
658 fprintf(stderr, "No IB devices found\n");
659 return 1;
660 }
661 } else {
662 int i;
663 for (i = 0; dev_list[i]; ++i)
664 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
665 break;
666 ib_dev = dev_list[i];
667 if (!ib_dev) {
668 fprintf(stderr, "IB device %s not found\n", ib_devname);
669 return 1;
670 }
671 }
672
673 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
674 if (!ctx)
675 return 1;
676
677 routs = pp_post_recv(ctx, ctx->rx_depth);
678 if (routs < ctx->rx_depth) {
679 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
680 return 1;
681 }
682
683 if (use_event)
684 if (ibv_req_notify_cq(ctx->cq, 0)) {
685 fprintf(stderr, "Couldn't request CQ notification\n");
686 return 1;
687 }
688
689
690 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
691 fprintf(stderr, "Couldn't get port info\n");
692 return 1;
693 }
694
695 my_dest.lid = ctx->portinfo.lid;
696 if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET &&
697 !my_dest.lid) {
698 fprintf(stderr, "Couldn't get local LID\n");
699 return 1;
700 }
701
702 if (gidx >= 0) {
703 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
704 fprintf(stderr, "can't read sgid of index %d\n", gidx);
705 return 1;
706 }
707 } else
708 memset(&my_dest.gid, 0, sizeof my_dest.gid);
709
710 my_dest.qpn = ctx->qp->qp_num;
711 my_dest.psn = lrand48() & 0xffffff;
712 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
713 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
714 my_dest.lid, my_dest.qpn, my_dest.psn, gid);
715
716
717 if (servername)
718 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
719 else
720 rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl,
721 &my_dest, gidx);
722
723 if (!rem_dest)
724 return 1;
725
726 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
727 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
728 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
729
730 if (servername)
731 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest,
732 gidx))
733 return 1;
734
735 ctx->pending = PINGPONG_RECV_WRID;
736
737 if (servername) {
738 if (pp_post_send(ctx)) {
739 fprintf(stderr, "Couldn't post send\n");
740 return 1;
741 }
742 ctx->pending |= PINGPONG_SEND_WRID;
743 }
744
745 if (gettimeofday(&start, NULL)) {
746 perror("gettimeofday");
747 return 1;
748 }
749
750 rcnt = scnt = 0;
751 while (rcnt < iters || scnt < iters) {
752 if (use_event) {
753 struct ibv_cq *ev_cq;
754 void *ev_ctx;
755
756 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
757 fprintf(stderr, "Failed to get cq_event\n");
758 return 1;
759 }
760
761 ++num_cq_events;
762
763 if (ev_cq != ctx->cq) {
764 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
765 return 1;
766 }
767
768 if (ibv_req_notify_cq(ctx->cq, 0)) {
769 fprintf(stderr, "Couldn't request CQ notification\n");
770 return 1;
771 }
772 }
773
774 {
775 struct ibv_wc wc[2];
776 int ne, i;
777
778 do {
779 ne = ibv_poll_cq(ctx->cq, 2, wc);
780 if (ne < 0) {
781 fprintf(stderr, "poll CQ failed %d\n", ne);
782 return 1;
783 }
784
785 } while (!use_event && ne < 1);
786
787 for (i = 0; i < ne; ++i) {
788 if (wc[i].status != IBV_WC_SUCCESS) {
789 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
790 ibv_wc_status_str(wc[i].status),
791 wc[i].status, (int) wc[i].wr_id);
792 return 1;
793 }
794
795 switch ((int) wc[i].wr_id) {
796 case PINGPONG_SEND_WRID:
797 ++scnt;
798 break;
799
800 case PINGPONG_RECV_WRID:
801 if (--routs <= 1) {
802 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
803 if (routs < ctx->rx_depth) {
804 fprintf(stderr,
805 "Couldn't post receive (%d)\n",
806 routs);
807 return 1;
808 }
809 }
810
811 ++rcnt;
812 break;
813
814 default:
815 fprintf(stderr, "Completion for unknown wr_id %d\n",
816 (int) wc[i].wr_id);
817 return 1;
818 }
819
820 ctx->pending &= ~(int) wc[i].wr_id;
821 if (scnt < iters && !ctx->pending) {
822 if (pp_post_send(ctx)) {
823 fprintf(stderr, "Couldn't post send\n");
824 return 1;
825 }
826 ctx->pending = PINGPONG_RECV_WRID |
827 PINGPONG_SEND_WRID;
828 }
829 }
830 }
831 }
832
833 if (gettimeofday(&end, NULL)) {
834 perror("gettimeofday");
835 return 1;
836 }
837
838 {
839 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
840 (end.tv_usec - start.tv_usec);
841 long long bytes = (long long) size * iters * 2;
842
843 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
844 bytes, usec / 1000000., bytes * 8. / usec);
845 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
846 iters, usec / 1000000., usec / iters);
847 }
848
849 ibv_ack_cq_events(ctx->cq, num_cq_events);
850
851 if (pp_close_ctx(ctx))
852 return 1;
853
854 ibv_free_device_list(dev_list);
855 free(rem_dest);
856
857 return 0;
858 }
859