xref: /freebsd/contrib/ofed/libibverbs/examples/ud_pingpong.c (revision f0cfa1b168014f56c02b83e5f28412cc5f78d117)
1 /*
2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #define _GNU_SOURCE
33 #include <config.h>
34 
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <unistd.h>
38 #include <string.h>
39 #include <sys/types.h>
40 #include <sys/socket.h>
41 #include <sys/time.h>
42 #include <netdb.h>
43 #include <stdlib.h>
44 #include <getopt.h>
45 #include <arpa/inet.h>
46 #include <time.h>
47 
48 #include "pingpong.h"
49 
50 enum {
51 	PINGPONG_RECV_WRID = 1,
52 	PINGPONG_SEND_WRID = 2,
53 };
54 
55 static int page_size;
56 
57 struct pingpong_context {
58 	struct ibv_context	*context;
59 	struct ibv_comp_channel *channel;
60 	struct ibv_pd		*pd;
61 	struct ibv_mr		*mr;
62 	struct ibv_cq		*cq;
63 	struct ibv_qp		*qp;
64 	struct ibv_ah		*ah;
65 	void			*buf;
66 	int			 size;
67 	int			 send_flags;
68 	int			 rx_depth;
69 	int			 pending;
70 	struct ibv_port_attr     portinfo;
71 };
72 
73 struct pingpong_dest {
74 	int lid;
75 	int qpn;
76 	int psn;
77 	union ibv_gid gid;
78 };
79 
80 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
81 			  int sl, struct pingpong_dest *dest, int sgid_idx)
82 {
83 	struct ibv_ah_attr ah_attr = {
84 		.is_global     = 0,
85 		.dlid          = dest->lid,
86 		.sl            = sl,
87 		.src_path_bits = 0,
88 		.port_num      = port
89 	};
90 	struct ibv_qp_attr attr = {
91 		.qp_state		= IBV_QPS_RTR
92 	};
93 
94 	if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
95 		fprintf(stderr, "Failed to modify QP to RTR\n");
96 		return 1;
97 	}
98 
99 	attr.qp_state	    = IBV_QPS_RTS;
100 	attr.sq_psn	    = my_psn;
101 
102 	if (ibv_modify_qp(ctx->qp, &attr,
103 			  IBV_QP_STATE              |
104 			  IBV_QP_SQ_PSN)) {
105 		fprintf(stderr, "Failed to modify QP to RTS\n");
106 		return 1;
107 	}
108 
109 	if (dest->gid.global.interface_id) {
110 		ah_attr.is_global = 1;
111 		ah_attr.grh.hop_limit = 1;
112 		ah_attr.grh.dgid = dest->gid;
113 		ah_attr.grh.sgid_index = sgid_idx;
114 	}
115 
116 	ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
117 	if (!ctx->ah) {
118 		fprintf(stderr, "Failed to create AH\n");
119 		return 1;
120 	}
121 
122 	return 0;
123 }
124 
125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
126 						 const struct pingpong_dest *my_dest)
127 {
128 	struct addrinfo *res, *t;
129 	struct addrinfo hints = {
130 		.ai_family   = AF_INET,
131 		.ai_socktype = SOCK_STREAM
132 	};
133 	char *service;
134 	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
135 	int n;
136 	int sockfd = -1;
137 	struct pingpong_dest *rem_dest = NULL;
138 	char gid[33];
139 
140 	if (asprintf(&service, "%d", port) < 0)
141 		return NULL;
142 
143 	n = getaddrinfo(servername, service, &hints, &res);
144 
145 	if (n < 0) {
146 		fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
147 		free(service);
148 		return NULL;
149 	}
150 
151 	for (t = res; t; t = t->ai_next) {
152 		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
153 		if (sockfd >= 0) {
154 			if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
155 				break;
156 			close(sockfd);
157 			sockfd = -1;
158 		}
159 	}
160 
161 	freeaddrinfo_null(res);
162 	free(service);
163 
164 	if (sockfd < 0) {
165 		fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
166 		return NULL;
167 	}
168 
169 	gid_to_wire_gid(&my_dest->gid, gid);
170 	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
171 							my_dest->psn, gid);
172 	if (write(sockfd, msg, sizeof msg) != sizeof msg) {
173 		fprintf(stderr, "Couldn't send local address\n");
174 		goto out;
175 	}
176 
177 	if (read(sockfd, msg, sizeof msg) != sizeof msg ||
178 	    write(sockfd, "done", sizeof "done") != sizeof "done") {
179 		perror("client read/write");
180 		fprintf(stderr, "Couldn't read/write remote address\n");
181 		goto out;
182 	}
183 
184 	rem_dest = malloc(sizeof *rem_dest);
185 	if (!rem_dest)
186 		goto out;
187 
188 	sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
189 							&rem_dest->psn, gid);
190 	wire_gid_to_gid(gid, &rem_dest->gid);
191 
192 out:
193 	close(sockfd);
194 	return rem_dest;
195 }
196 
197 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
198 						 int ib_port, int port, int sl,
199 						 const struct pingpong_dest *my_dest,
200 						 int sgid_idx)
201 {
202 	struct addrinfo *res, *t;
203 	struct addrinfo hints = {
204 		.ai_flags    = AI_PASSIVE,
205 		.ai_family   = AF_INET,
206 		.ai_socktype = SOCK_STREAM
207 	};
208 	char *service;
209 	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
210 	int n;
211 	int sockfd = -1, connfd;
212 	struct pingpong_dest *rem_dest = NULL;
213 	char gid[33];
214 
215 	if (asprintf(&service, "%d", port) < 0)
216 		return NULL;
217 
218 	n = getaddrinfo(NULL, service, &hints, &res);
219 
220 	if (n < 0) {
221 		fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
222 		free(service);
223 		return NULL;
224 	}
225 
226 	for (t = res; t; t = t->ai_next) {
227 		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
228 		if (sockfd >= 0) {
229 			n = 1;
230 
231 			setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
232 
233 			if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
234 				break;
235 			close(sockfd);
236 			sockfd = -1;
237 		}
238 	}
239 
240 	freeaddrinfo_null(res);
241 	free(service);
242 
243 	if (sockfd < 0) {
244 		fprintf(stderr, "Couldn't listen to port %d\n", port);
245 		return NULL;
246 	}
247 
248 	listen(sockfd, 1);
249 	connfd = accept(sockfd, NULL, NULL);
250 	close(sockfd);
251 	if (connfd < 0) {
252 		fprintf(stderr, "accept() failed\n");
253 		return NULL;
254 	}
255 
256 	n = read(connfd, msg, sizeof msg);
257 	if (n != sizeof msg) {
258 		perror("server read");
259 		fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
260 		goto out;
261 	}
262 
263 	rem_dest = malloc(sizeof *rem_dest);
264 	if (!rem_dest)
265 		goto out;
266 
267 	sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
268 							&rem_dest->psn, gid);
269 	wire_gid_to_gid(gid, &rem_dest->gid);
270 
271 	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
272 								sgid_idx)) {
273 		fprintf(stderr, "Couldn't connect to remote QP\n");
274 		free(rem_dest);
275 		rem_dest = NULL;
276 		goto out;
277 	}
278 
279 	gid_to_wire_gid(&my_dest->gid, gid);
280 	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
281 							my_dest->psn, gid);
282 	if (write(connfd, msg, sizeof msg) != sizeof msg ||
283 	    read(connfd, msg, sizeof msg) != sizeof "done") {
284 		fprintf(stderr, "Couldn't send/recv local address\n");
285 		free(rem_dest);
286 		rem_dest = NULL;
287 		goto out;
288 	}
289 out:
290 	close(connfd);
291 	return rem_dest;
292 }
293 
294 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
295 					    int rx_depth, int port,
296 					    int use_event)
297 {
298 	struct pingpong_context *ctx;
299 
300 	ctx = malloc(sizeof *ctx);
301 	if (!ctx)
302 		return NULL;
303 
304 	ctx->size       = size;
305 	ctx->send_flags = IBV_SEND_SIGNALED;
306 	ctx->rx_depth   = rx_depth;
307 
308 	ctx->buf = memalign(page_size, size + 40);
309 	if (!ctx->buf) {
310 		fprintf(stderr, "Couldn't allocate work buf.\n");
311 		goto clean_ctx;
312 	}
313 
314 	/* FIXME memset(ctx->buf, 0, size + 40); */
315 	memset(ctx->buf, 0x7b, size + 40);
316 
317 	ctx->context = ibv_open_device(ib_dev);
318 	if (!ctx->context) {
319 		fprintf(stderr, "Couldn't get context for %s\n",
320 			ibv_get_device_name(ib_dev));
321 		goto clean_buffer;
322 	}
323 
324 	{
325 		struct ibv_port_attr port_info = {};
326 		int mtu;
327 
328 		if (ibv_query_port(ctx->context, port, &port_info)) {
329 			fprintf(stderr, "Unable to query port info for port %d\n", port);
330 			goto clean_device;
331 		}
332 		mtu = 1 << (port_info.active_mtu + 7);
333 		if (size > mtu) {
334 			fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu);
335 			goto clean_device;
336 		}
337 	}
338 
339 	if (use_event) {
340 		ctx->channel = ibv_create_comp_channel(ctx->context);
341 		if (!ctx->channel) {
342 			fprintf(stderr, "Couldn't create completion channel\n");
343 			goto clean_device;
344 		}
345 	} else
346 		ctx->channel = NULL;
347 
348 	ctx->pd = ibv_alloc_pd(ctx->context);
349 	if (!ctx->pd) {
350 		fprintf(stderr, "Couldn't allocate PD\n");
351 		goto clean_comp_channel;
352 	}
353 
354 	ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
355 	if (!ctx->mr) {
356 		fprintf(stderr, "Couldn't register MR\n");
357 		goto clean_pd;
358 	}
359 
360 	ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
361 				ctx->channel, 0);
362 	if (!ctx->cq) {
363 		fprintf(stderr, "Couldn't create CQ\n");
364 		goto clean_mr;
365 	}
366 
367 	{
368 		struct ibv_qp_attr attr;
369 		struct ibv_qp_init_attr init_attr = {
370 			.send_cq = ctx->cq,
371 			.recv_cq = ctx->cq,
372 			.cap     = {
373 				.max_send_wr  = 1,
374 				.max_recv_wr  = rx_depth,
375 				.max_send_sge = 1,
376 				.max_recv_sge = 1
377 			},
378 			.qp_type = IBV_QPT_UD,
379 		};
380 
381 		ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
382 		if (!ctx->qp)  {
383 			fprintf(stderr, "Couldn't create QP\n");
384 			goto clean_cq;
385 		}
386 
387 		ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
388 		if (init_attr.cap.max_inline_data >= size) {
389 			ctx->send_flags |= IBV_SEND_INLINE;
390 		}
391 	}
392 
393 	{
394 		struct ibv_qp_attr attr = {
395 			.qp_state        = IBV_QPS_INIT,
396 			.pkey_index      = 0,
397 			.port_num        = port,
398 			.qkey            = 0x11111111
399 		};
400 
401 		if (ibv_modify_qp(ctx->qp, &attr,
402 				  IBV_QP_STATE              |
403 				  IBV_QP_PKEY_INDEX         |
404 				  IBV_QP_PORT               |
405 				  IBV_QP_QKEY)) {
406 			fprintf(stderr, "Failed to modify QP to INIT\n");
407 			goto clean_qp;
408 		}
409 	}
410 
411 	return ctx;
412 
413 clean_qp:
414 	ibv_destroy_qp(ctx->qp);
415 
416 clean_cq:
417 	ibv_destroy_cq(ctx->cq);
418 
419 clean_mr:
420 	ibv_dereg_mr(ctx->mr);
421 
422 clean_pd:
423 	ibv_dealloc_pd(ctx->pd);
424 
425 clean_comp_channel:
426 	if (ctx->channel)
427 		ibv_destroy_comp_channel(ctx->channel);
428 
429 clean_device:
430 	ibv_close_device(ctx->context);
431 
432 clean_buffer:
433 	free(ctx->buf);
434 
435 clean_ctx:
436 	free(ctx);
437 
438 	return NULL;
439 }
440 
441 static int pp_close_ctx(struct pingpong_context *ctx)
442 {
443 	if (ibv_destroy_qp(ctx->qp)) {
444 		fprintf(stderr, "Couldn't destroy QP\n");
445 		return 1;
446 	}
447 
448 	if (ibv_destroy_cq(ctx->cq)) {
449 		fprintf(stderr, "Couldn't destroy CQ\n");
450 		return 1;
451 	}
452 
453 	if (ibv_dereg_mr(ctx->mr)) {
454 		fprintf(stderr, "Couldn't deregister MR\n");
455 		return 1;
456 	}
457 
458 	if (ibv_destroy_ah(ctx->ah)) {
459 		fprintf(stderr, "Couldn't destroy AH\n");
460 		return 1;
461 	}
462 
463 	if (ibv_dealloc_pd(ctx->pd)) {
464 		fprintf(stderr, "Couldn't deallocate PD\n");
465 		return 1;
466 	}
467 
468 	if (ctx->channel) {
469 		if (ibv_destroy_comp_channel(ctx->channel)) {
470 			fprintf(stderr, "Couldn't destroy completion channel\n");
471 			return 1;
472 		}
473 	}
474 
475 	if (ibv_close_device(ctx->context)) {
476 		fprintf(stderr, "Couldn't release context\n");
477 		return 1;
478 	}
479 
480 	free(ctx->buf);
481 	free(ctx);
482 
483 	return 0;
484 }
485 
486 static int pp_post_recv(struct pingpong_context *ctx, int n)
487 {
488 	struct ibv_sge list = {
489 		.addr	= (uintptr_t) ctx->buf,
490 		.length = ctx->size + 40,
491 		.lkey	= ctx->mr->lkey
492 	};
493 	struct ibv_recv_wr wr = {
494 		.wr_id	    = PINGPONG_RECV_WRID,
495 		.sg_list    = &list,
496 		.num_sge    = 1,
497 	};
498 	struct ibv_recv_wr *bad_wr;
499 	int i;
500 
501 	for (i = 0; i < n; ++i)
502 		if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
503 			break;
504 
505 	return i;
506 }
507 
508 static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
509 {
510 	struct ibv_sge list = {
511 		.addr	= (uintptr_t) ctx->buf + 40,
512 		.length = ctx->size,
513 		.lkey	= ctx->mr->lkey
514 	};
515 	struct ibv_send_wr wr = {
516 		.wr_id	    = PINGPONG_SEND_WRID,
517 		.sg_list    = &list,
518 		.num_sge    = 1,
519 		.opcode     = IBV_WR_SEND,
520 		.send_flags = ctx->send_flags,
521 		.wr         = {
522 			.ud = {
523 				 .ah          = ctx->ah,
524 				 .remote_qpn  = qpn,
525 				 .remote_qkey = 0x11111111
526 			 }
527 		}
528 	};
529 	struct ibv_send_wr *bad_wr;
530 
531 	return ibv_post_send(ctx->qp, &wr, &bad_wr);
532 }
533 
534 static void usage(const char *argv0)
535 {
536 	printf("Usage:\n");
537 	printf("  %s            start a server and wait for connection\n", argv0);
538 	printf("  %s <host>     connect to server at <host>\n", argv0);
539 	printf("\n");
540 	printf("Options:\n");
541 	printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
542 	printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
543 	printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
544 	printf("  -s, --size=<size>      size of message to exchange (default 2048)\n");
545 	printf("  -r, --rx-depth=<dep>   number of receives to post at a time (default 500)\n");
546 	printf("  -n, --iters=<iters>    number of exchanges (default 1000)\n");
547         printf("  -l, --sl=<SL>          send messages with service level <SL> (default 0)\n");
548 	printf("  -e, --events           sleep on CQ events (default poll)\n");
549 	printf("  -g, --gid-idx=<gid index> local port gid index\n");
550 }
551 
552 int main(int argc, char *argv[])
553 {
554 	struct ibv_device      **dev_list;
555 	struct ibv_device	*ib_dev;
556 	struct pingpong_context *ctx;
557 	struct pingpong_dest     my_dest;
558 	struct pingpong_dest    *rem_dest;
559 	struct timeval           start, end;
560 	char                    *ib_devname = NULL;
561 	char                    *servername = NULL;
562 	unsigned int             port = 18515;
563 	int                      ib_port = 1;
564 	unsigned int             size = 2048;
565 	unsigned int             rx_depth = 500;
566 	unsigned int             iters = 1000;
567 	int                      use_event = 0;
568 	int                      routs;
569 	int                      rcnt, scnt;
570 	int                      num_cq_events = 0;
571 	int                      sl = 0;
572 	int			 gidx = -1;
573 	char			 gid[33];
574 
575 	srand48(getpid() * time(NULL));
576 
577 	while (1) {
578 		int c;
579 
580 		static struct option long_options[] = {
581 			{ .name = "port",     .has_arg = 1, .val = 'p' },
582 			{ .name = "ib-dev",   .has_arg = 1, .val = 'd' },
583 			{ .name = "ib-port",  .has_arg = 1, .val = 'i' },
584 			{ .name = "size",     .has_arg = 1, .val = 's' },
585 			{ .name = "rx-depth", .has_arg = 1, .val = 'r' },
586 			{ .name = "iters",    .has_arg = 1, .val = 'n' },
587 			{ .name = "sl",       .has_arg = 1, .val = 'l' },
588 			{ .name = "events",   .has_arg = 0, .val = 'e' },
589 			{ .name = "gid-idx",  .has_arg = 1, .val = 'g' },
590 			{}
591 		};
592 
593 		c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:",
594 							long_options, NULL);
595 		if (c == -1)
596 			break;
597 
598 		switch (c) {
599 		case 'p':
600 			port = strtol(optarg, NULL, 0);
601 			if (port > 65535) {
602 				usage(argv[0]);
603 				return 1;
604 			}
605 			break;
606 
607 		case 'd':
608 			ib_devname = strdupa(optarg);
609 			break;
610 
611 		case 'i':
612 			ib_port = strtol(optarg, NULL, 0);
613 			if (ib_port < 1) {
614 				usage(argv[0]);
615 				return 1;
616 			}
617 			break;
618 
619 		case 's':
620 			size = strtoul(optarg, NULL, 0);
621 			break;
622 
623 		case 'r':
624 			rx_depth = strtoul(optarg, NULL, 0);
625 			break;
626 
627 		case 'n':
628 			iters = strtoul(optarg, NULL, 0);
629 			break;
630 
631 		case 'l':
632 			sl = strtol(optarg, NULL, 0);
633 			break;
634 
635 		case 'e':
636 			++use_event;
637 			break;
638 
639 		case 'g':
640 			gidx = strtol(optarg, NULL, 0);
641 			break;
642 
643 		default:
644 			usage(argv[0]);
645 			return 1;
646 		}
647 	}
648 
649 	if (optind == argc - 1)
650 		servername = strdupa(argv[optind]);
651 	else if (optind < argc) {
652 		usage(argv[0]);
653 		return 1;
654 	}
655 
656 	page_size = sysconf(_SC_PAGESIZE);
657 
658 	dev_list = ibv_get_device_list(NULL);
659 	if (!dev_list) {
660 		perror("Failed to get IB devices list");
661 		return 1;
662 	}
663 
664 	if (!ib_devname) {
665 		ib_dev = *dev_list;
666 		if (!ib_dev) {
667 			fprintf(stderr, "No IB devices found\n");
668 			return 1;
669 		}
670 	} else {
671 		int i;
672 		for (i = 0; dev_list[i]; ++i)
673 			if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
674 				break;
675 		ib_dev = dev_list[i];
676 		if (!ib_dev) {
677 			fprintf(stderr, "IB device %s not found\n", ib_devname);
678 			return 1;
679 		}
680 	}
681 
682 	ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
683 	if (!ctx)
684 		return 1;
685 
686 	routs = pp_post_recv(ctx, ctx->rx_depth);
687 	if (routs < ctx->rx_depth) {
688 		fprintf(stderr, "Couldn't post receive (%d)\n", routs);
689 		return 1;
690 	}
691 
692 	if (use_event)
693 		if (ibv_req_notify_cq(ctx->cq, 0)) {
694 			fprintf(stderr, "Couldn't request CQ notification\n");
695 			return 1;
696 		}
697 
698 	if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
699 		fprintf(stderr, "Couldn't get port info\n");
700 		return 1;
701 	}
702 	my_dest.lid = ctx->portinfo.lid;
703 
704 	my_dest.qpn = ctx->qp->qp_num;
705 	my_dest.psn = lrand48() & 0xffffff;
706 
707 	if (gidx >= 0) {
708 		if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
709 			fprintf(stderr, "Could not get local gid for gid index "
710 								"%d\n", gidx);
711 			return 1;
712 		}
713 	} else
714 		memset(&my_dest.gid, 0, sizeof my_dest.gid);
715 
716 	inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
717 	printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
718 	       my_dest.lid, my_dest.qpn, my_dest.psn, gid);
719 
720 	if (servername)
721 		rem_dest = pp_client_exch_dest(servername, port, &my_dest);
722 	else
723 		rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl,
724 							&my_dest, gidx);
725 
726 	if (!rem_dest)
727 		return 1;
728 
729 	inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
730 	printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
731 	       rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
732 
733 	if (servername)
734 		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest,
735 									gidx))
736 			return 1;
737 
738 	ctx->pending = PINGPONG_RECV_WRID;
739 
740 	if (servername) {
741 		if (pp_post_send(ctx, rem_dest->qpn)) {
742 			fprintf(stderr, "Couldn't post send\n");
743 			return 1;
744 		}
745 		ctx->pending |= PINGPONG_SEND_WRID;
746 	}
747 
748 	if (gettimeofday(&start, NULL)) {
749 		perror("gettimeofday");
750 		return 1;
751 	}
752 
753 	rcnt = scnt = 0;
754 	while (rcnt < iters || scnt < iters) {
755 		if (use_event) {
756 			struct ibv_cq *ev_cq;
757 			void          *ev_ctx;
758 
759 			if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
760 				fprintf(stderr, "Failed to get cq_event\n");
761 				return 1;
762 			}
763 
764 			++num_cq_events;
765 
766 			if (ev_cq != ctx->cq) {
767 				fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
768 				return 1;
769 			}
770 
771 			if (ibv_req_notify_cq(ctx->cq, 0)) {
772 				fprintf(stderr, "Couldn't request CQ notification\n");
773 				return 1;
774 			}
775 		}
776 
777 		{
778 			struct ibv_wc wc[2];
779 			int ne, i;
780 
781 			do {
782 				ne = ibv_poll_cq(ctx->cq, 2, wc);
783 				if (ne < 0) {
784 					fprintf(stderr, "poll CQ failed %d\n", ne);
785 					return 1;
786 				}
787 			} while (!use_event && ne < 1);
788 
789 			for (i = 0; i < ne; ++i) {
790 				if (wc[i].status != IBV_WC_SUCCESS) {
791 					fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
792 						ibv_wc_status_str(wc[i].status),
793 						wc[i].status, (int) wc[i].wr_id);
794 					return 1;
795 				}
796 
797 				switch ((int) wc[i].wr_id) {
798 				case PINGPONG_SEND_WRID:
799 					++scnt;
800 					break;
801 
802 				case PINGPONG_RECV_WRID:
803 					if (--routs <= 1) {
804 						routs += pp_post_recv(ctx, ctx->rx_depth - routs);
805 						if (routs < ctx->rx_depth) {
806 							fprintf(stderr,
807 								"Couldn't post receive (%d)\n",
808 								routs);
809 							return 1;
810 						}
811 					}
812 
813 					++rcnt;
814 					break;
815 
816 				default:
817 					fprintf(stderr, "Completion for unknown wr_id %d\n",
818 						(int) wc[i].wr_id);
819 					return 1;
820 				}
821 
822 				ctx->pending &= ~(int) wc[i].wr_id;
823 				if (scnt < iters && !ctx->pending) {
824 					if (pp_post_send(ctx, rem_dest->qpn)) {
825 						fprintf(stderr, "Couldn't post send\n");
826 						return 1;
827 					}
828 					ctx->pending = PINGPONG_RECV_WRID |
829 						       PINGPONG_SEND_WRID;
830 				}
831 			}
832 		}
833 	}
834 
835 	if (gettimeofday(&end, NULL)) {
836 		perror("gettimeofday");
837 		return 1;
838 	}
839 
840 	{
841 		float usec = (end.tv_sec - start.tv_sec) * 1000000 +
842 			(end.tv_usec - start.tv_usec);
843 		long long bytes = (long long) size * iters * 2;
844 
845 		printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
846 		       bytes, usec / 1000000., bytes * 8. / usec);
847 		printf("%d iters in %.2f seconds = %.2f usec/iter\n",
848 		       iters, usec / 1000000., usec / iters);
849 	}
850 
851 	ibv_ack_cq_events(ctx->cq, num_cq_events);
852 
853 	if (pp_close_ctx(ctx))
854 		return 1;
855 
856 	ibv_free_device_list(dev_list);
857 	free(rem_dest);
858 
859 	return 0;
860 }
861