xref: /freebsd/contrib/ofed/librdmacm/examples/udpong.c (revision d4eeb02986980bf33dd56c41ceb9fc5f180c0d47)
1 /*
2  * Copyright (c) 2012 Intel Corporation.  All rights reserved.
3  *
4  * This software is available to you under the OpenIB.org BSD license
5  * below:
6  *
7  *     Redistribution and use in source and binary forms, with or
8  *     without modification, are permitted provided that the following
9  *     conditions are met:
10  *
11  *      - Redistributions of source code must retain the above
12  *        copyright notice, this list of conditions and the following
13  *        disclaimer.
14  *
15  *      - Redistributions in binary form must reproduce the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer in the documentation and/or other materials
18  *        provided with the distribution.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27  * SOFTWARE.
28  */
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <strings.h>
34 #include <errno.h>
35 #include <getopt.h>
36 #include <sys/types.h>
37 #include <sys/socket.h>
38 #include <sys/time.h>
39 #include <sys/wait.h>
40 #include <netdb.h>
41 #include <fcntl.h>
42 #include <unistd.h>
43 #include <netinet/tcp.h>
44 #include <arpa/inet.h>
45 
46 #include <rdma/rdma_cma.h>
47 #include <rdma/rsocket.h>
48 #include <util/compiler.h>
49 #include "common.h"
50 
51 static int test_size[] = {
52 	(1 <<  6),
53 	(1 <<  7), ((1 <<  7) + (1 << 6)),
54 	(1 <<  8), ((1 <<  8) + (1 << 7)),
55 	(1 <<  9), ((1 <<  9) + (1 << 8)),
56 	(1 << 10), ((1 << 10) + (1 << 9)),
57 };
58 #define TEST_CNT (sizeof test_size / sizeof test_size[0])
59 
60 enum {
61 	msg_op_login,
62 	msg_op_start,
63 	msg_op_data,
64 	msg_op_echo,
65 	msg_op_end
66 };
67 
68 struct message {
69 	uint8_t op;
70 	uint8_t id;
71 	uint8_t seqno;
72 	uint8_t reserved;
73 	__be32 data;
74 	uint8_t  buf[2048];
75 };
76 
77 #define CTRL_MSG_SIZE 16
78 
79 struct client {
80 	uint64_t recvcnt;
81 };
82 
83 static struct client clients[256];
84 static uint8_t id;
85 
86 static int rs;
87 static int use_async;
88 static int flags = MSG_DONTWAIT;
89 static int poll_timeout;
90 static int custom;
91 static int echo;
92 static int transfer_size = 1000;
93 static int transfer_count = 1000;
94 static int buffer_size;
95 static char test_name[10] = "custom";
96 static const char *port = "7174";
97 static char *dst_addr;
98 static char *src_addr;
99 static union socket_addr g_addr;
100 static socklen_t g_addrlen;
101 static struct timeval start, end;
102 static struct message g_msg;
103 
104 static void show_perf(void)
105 {
106 	char str[32];
107 	float usec;
108 	long long bytes;
109 	int transfers;
110 
111 	usec = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_usec - start.tv_usec);
112 	transfers = echo ? transfer_count * 2 : be32toh(g_msg.data);
113 	bytes = (long long) transfers * transfer_size;
114 
115 	/* name size transfers bytes seconds Gb/sec usec/xfer */
116 	printf("%-10s", test_name);
117 	size_str(str, sizeof str, transfer_size);
118 	printf("%-8s", str);
119 	cnt_str(str, sizeof str, transfers);
120 	printf("%-8s", str);
121 	size_str(str, sizeof str, bytes);
122 	printf("%-8s", str);
123 	printf("%8.2fs%10.2f%11.2f\n",
124 		usec / 1000000., (bytes * 8) / (1000. * usec),
125 		(usec / transfers));
126 }
127 
128 static void init_latency_test(int size)
129 {
130 	char sstr[5];
131 
132 	size_str(sstr, sizeof sstr, size);
133 	snprintf(test_name, sizeof test_name, "%s_lat", sstr);
134 	transfer_size = size;
135 	transfer_count = size_to_count(transfer_size) / 10;
136 	echo = 1;
137 }
138 
139 static void init_bandwidth_test(int size)
140 {
141 	char sstr[5];
142 
143 	size_str(sstr, sizeof sstr, size);
144 	snprintf(test_name, sizeof test_name, "%s_bw", sstr);
145 	transfer_size = size;
146 	transfer_count = size_to_count(transfer_size);
147 	echo = 0;
148 }
149 
150 static void set_options(int fd)
151 {
152 	int val;
153 
154 	if (buffer_size) {
155 		rs_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (void *) &buffer_size,
156 			      sizeof buffer_size);
157 		rs_setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (void *) &buffer_size,
158 			      sizeof buffer_size);
159 	} else {
160 		val = 1 << 19;
161 		rs_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (void *) &val, sizeof val);
162 		rs_setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (void *) &val, sizeof val);
163 	}
164 
165 	if (flags & MSG_DONTWAIT)
166 		rs_fcntl(fd, F_SETFL, O_NONBLOCK);
167 }
168 
169 static ssize_t svr_send(struct message *msg, size_t size,
170 			union socket_addr *addr, socklen_t addrlen)
171 {
172 	struct pollfd fds;
173 	ssize_t ret;
174 
175 	if (use_async) {
176 		fds.fd = rs;
177 		fds.events = POLLOUT;
178 	}
179 
180 	do {
181 		if (use_async) {
182 			ret = do_poll(&fds, poll_timeout);
183 			if (ret)
184 				return ret;
185 		}
186 
187 		ret = rs_sendto(rs, msg, size, flags, &addr->sa, addrlen);
188 	} while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
189 
190 	if (ret < 0)
191 		perror("rsend");
192 
193 	return ret;
194 }
195 
196 static ssize_t svr_recv(struct message *msg, size_t size,
197 			union socket_addr *addr, socklen_t *addrlen)
198 {
199 	struct pollfd fds;
200 	ssize_t ret;
201 
202 	if (use_async) {
203 		fds.fd = rs;
204 		fds.events = POLLIN;
205 	}
206 
207 	do {
208 		if (use_async) {
209 			ret = do_poll(&fds, poll_timeout);
210 			if (ret)
211 				return ret;
212 		}
213 
214 		ret = rs_recvfrom(rs, msg, size, flags, &addr->sa, addrlen);
215 	} while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
216 
217 	if (ret < 0)
218 		perror("rrecv");
219 
220 	return ret;
221 }
222 
223 static int svr_process(struct message *msg, size_t size,
224 		       union socket_addr *addr, socklen_t addrlen)
225 {
226 	char str[64];
227 	ssize_t ret;
228 
229 	switch (msg->op) {
230 	case msg_op_login:
231 		if (addr->sa.sa_family == AF_INET) {
232 			printf("client login from %s\n",
233 			       inet_ntop(AF_INET, &addr->sin.sin_addr.s_addr,
234 					 str, sizeof str));
235 		} else {
236 			printf("client login from %s\n",
237 			       inet_ntop(AF_INET6, &addr->sin6.sin6_addr.s6_addr,
238 					 str, sizeof str));
239 		}
240 		msg->id = id++;
241 		/* fall through */
242 	case msg_op_start:
243 		memset(&clients[msg->id], 0, sizeof clients[msg->id]);
244 		break;
245 	case msg_op_echo:
246 		clients[msg->id].recvcnt++;
247 		break;
248 	case msg_op_end:
249 		msg->data = htobe32(clients[msg->id].recvcnt);
250 		break;
251 	default:
252 		clients[msg->id].recvcnt++;
253 		return 0;
254 	}
255 
256 	ret = svr_send(msg, size, addr, addrlen);
257 	return (ret == size) ? 0 : (int) ret;
258 }
259 
260 static int svr_bind(void)
261 {
262 	struct addrinfo hints, *res;
263 	int ret;
264 
265 	memset(&hints, 0, sizeof hints);
266 	hints.ai_socktype = SOCK_DGRAM;
267  	ret = getaddrinfo(src_addr, port, &hints, &res);
268 	if (ret) {
269 		printf("getaddrinfo: %s\n", gai_strerror(ret));
270 		return ret;
271 	}
272 
273 	rs = rs_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
274 	if (rs < 0) {
275 		perror("rsocket");
276 		ret = rs;
277 		goto out;
278 	}
279 
280 	set_options(rs);
281 	ret = rs_bind(rs, res->ai_addr, res->ai_addrlen);
282 	if (ret) {
283 		perror("rbind");
284 		rs_close(rs);
285 	}
286 
287 out:
288 	free(res);
289 	return ret;
290 }
291 
292 static int svr_run(void)
293 {
294 	ssize_t len;
295 	int ret;
296 
297 	ret = svr_bind();
298 	while (!ret) {
299 		g_addrlen = sizeof g_addr;
300 		len = svr_recv(&g_msg, sizeof g_msg, &g_addr, &g_addrlen);
301 		if (len < 0)
302 			return len;
303 
304 		ret = svr_process(&g_msg, len, &g_addr, g_addrlen);
305 	}
306 	return ret;
307 }
308 
309 static ssize_t client_send(struct message *msg, size_t size)
310 {
311 	struct pollfd fds;
312 	int ret;
313 
314 	if (use_async) {
315 		fds.fd = rs;
316 		fds.events = POLLOUT;
317 	}
318 
319 	do {
320 		if (use_async) {
321 			ret = do_poll(&fds, poll_timeout);
322 			if (ret)
323 				return ret;
324 		}
325 
326 		ret = rs_send(rs, msg, size, flags);
327 	} while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
328 
329 	if (ret < 0)
330 		perror("rsend");
331 
332 	return ret;
333 }
334 
335 static ssize_t client_recv(struct message *msg, size_t size, int timeout)
336 {
337 	struct pollfd fds;
338 	int ret;
339 
340 	if (timeout) {
341 		fds.fd = rs;
342 		fds.events = POLLIN;
343 
344 		ret = rs_poll(&fds, 1, timeout);
345 		if (ret <= 0)
346 			return ret;
347 	}
348 
349 	ret = rs_recv(rs, msg, size, flags | MSG_DONTWAIT);
350 	if (ret < 0 && errno != EWOULDBLOCK && errno != EAGAIN)
351 		perror("rrecv");
352 
353 	return ret;
354 }
355 
356 static int client_send_recv(struct message *msg, size_t size, int timeout)
357 {
358 	static uint8_t seqno;
359 	int ret;
360 
361 	msg->seqno = seqno;
362 	do {
363 		ret = client_send(msg, size);
364 		if (ret != size)
365 			return ret;
366 
367 		ret = client_recv(msg, size, timeout);
368 	} while (ret <= 0 || msg->seqno != seqno);
369 
370 	seqno++;
371 	return ret;
372 }
373 
374 static int run_test(void)
375 {
376 	int ret, i;
377 
378 	g_msg.op = msg_op_start;
379 	ret = client_send_recv(&g_msg, CTRL_MSG_SIZE, 1000);
380 	if (ret != CTRL_MSG_SIZE)
381 		goto out;
382 
383 	g_msg.op = echo ? msg_op_echo : msg_op_data;
384 	gettimeofday(&start, NULL);
385 	for (i = 0; i < transfer_count; i++) {
386 		ret = echo ? client_send_recv(&g_msg, transfer_size, 1) :
387 			     client_send(&g_msg, transfer_size);
388 		if (ret != transfer_size)
389 			goto out;
390 	}
391 
392 	g_msg.op = msg_op_end;
393 	ret = client_send_recv(&g_msg, CTRL_MSG_SIZE, 1);
394 	if (ret != CTRL_MSG_SIZE)
395 		goto out;
396 
397 	gettimeofday(&end, NULL);
398 	show_perf();
399 	ret = 0;
400 
401 out:
402 	return ret;
403 }
404 
405 static int client_connect(void)
406 {
407 	struct addrinfo hints, *res;
408 	int ret;
409 
410 	memset(&hints, 0, sizeof hints);
411 	hints.ai_socktype = SOCK_DGRAM;
412  	ret = getaddrinfo(dst_addr, port, &hints, &res);
413 	if (ret) {
414 		printf("getaddrinfo: %s\n", gai_strerror(ret));
415 		return ret;
416 	}
417 
418 	rs = rs_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
419 	if (rs < 0) {
420 		perror("rsocket");
421 		ret = rs;
422 		goto out;
423 	}
424 
425 	set_options(rs);
426 	ret = rs_connect(rs, res->ai_addr, res->ai_addrlen);
427 	if (ret) {
428 		perror("rconnect");
429 		rs_close(rs);
430 		goto out;
431 	}
432 
433 	g_msg.op = msg_op_login;
434 	ret = client_send_recv(&g_msg, CTRL_MSG_SIZE, 1000);
435 	if (ret == CTRL_MSG_SIZE)
436 		ret = 0;
437 
438 out:
439 	freeaddrinfo(res);
440 	return ret;
441 }
442 
443 static int client_run(void)
444 {
445 	int i, ret;
446 
447 	printf("%-10s%-8s%-8s%-8s%8s %10s%13s\n",
448 	       "name", "bytes", "xfers", "total", "time", "Gb/sec", "usec/xfer");
449 
450 	ret = client_connect();
451 	if (ret)
452 		return ret;
453 
454 	if (!custom) {
455 		for (i = 0; i < TEST_CNT; i++) {
456 			init_latency_test(test_size[i]);
457 			run_test();
458 		}
459 		for (i = 0; i < TEST_CNT; i++) {
460 			init_bandwidth_test(test_size[i]);
461 			run_test();
462 		}
463 	} else {
464 		run_test();
465 	}
466 	rs_close(rs);
467 
468 	return ret;
469 }
470 
471 static int set_test_opt(const char *arg)
472 {
473 	if (strlen(arg) == 1) {
474 		switch (arg[0]) {
475 		case 's':
476 			use_rs = 0;
477 			break;
478 		case 'a':
479 			use_async = 1;
480 			break;
481 		case 'b':
482 			flags = 0;
483 			break;
484 		case 'n':
485 			flags = MSG_DONTWAIT;
486 			break;
487 		case 'e':
488 			echo = 1;
489 			break;
490 		default:
491 			return -1;
492 		}
493 	} else {
494 		if (!strncasecmp("socket", arg, 6)) {
495 			use_rs = 0;
496 		} else if (!strncasecmp("async", arg, 5)) {
497 			use_async = 1;
498 		} else if (!strncasecmp("block", arg, 5)) {
499 			flags = 0;
500 		} else if (!strncasecmp("nonblock", arg, 8)) {
501 			flags = MSG_DONTWAIT;
502 		} else if (!strncasecmp("echo", arg, 4)) {
503 			echo = 1;
504 		} else {
505 			return -1;
506 		}
507 	}
508 	return 0;
509 }
510 
511 int main(int argc, char **argv)
512 {
513 	int op, ret;
514 
515 	while ((op = getopt(argc, argv, "s:b:B:C:S:p:T:")) != -1) {
516 		switch (op) {
517 		case 's':
518 			dst_addr = optarg;
519 			break;
520 		case 'b':
521 			src_addr = optarg;
522 			break;
523 		case 'B':
524 			buffer_size = atoi(optarg);
525 			break;
526 		case 'C':
527 			custom = 1;
528 			transfer_count = atoi(optarg);
529 			break;
530 		case 'S':
531 			custom = 1;
532 			transfer_size = atoi(optarg);
533 			if (transfer_size < CTRL_MSG_SIZE) {
534 				printf("size must be at least %d bytes\n",
535 				       CTRL_MSG_SIZE);
536 				exit(1);
537 			}
538 			break;
539 		case 'p':
540 			port = optarg;
541 			break;
542 		case 'T':
543 			if (!set_test_opt(optarg))
544 				break;
545 			/* invalid option - fall through */
546 			SWITCH_FALLTHROUGH;
547 		default:
548 			printf("usage: %s\n", argv[0]);
549 			printf("\t[-s server_address]\n");
550 			printf("\t[-b bind_address]\n");
551 			printf("\t[-B buffer_size]\n");
552 			printf("\t[-C transfer_count]\n");
553 			printf("\t[-S transfer_size]\n");
554 			printf("\t[-p port_number]\n");
555 			printf("\t[-T test_option]\n");
556 			printf("\t    s|sockets - use standard tcp/ip sockets\n");
557 			printf("\t    a|async - asynchronous operation (use poll)\n");
558 			printf("\t    b|blocking - use blocking calls\n");
559 			printf("\t    n|nonblocking - use nonblocking calls\n");
560 			printf("\t    e|echo - server echoes all messages\n");
561 			exit(1);
562 		}
563 	}
564 
565 	if (flags)
566 		poll_timeout = -1;
567 
568 	ret = dst_addr ? client_run() : svr_run();
569 	return ret;
570 }
571