xref: /linux/tools/testing/selftests/net/udpgso_bench_tx.c (revision bfd5bb6f90af092aa345b15cd78143956a13c2a8)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 #define _GNU_SOURCE
4 
5 #include <arpa/inet.h>
6 #include <errno.h>
7 #include <error.h>
8 #include <netinet/if_ether.h>
9 #include <netinet/in.h>
10 #include <netinet/ip.h>
11 #include <netinet/ip6.h>
12 #include <netinet/udp.h>
13 #include <poll.h>
14 #include <sched.h>
15 #include <signal.h>
16 #include <stdbool.h>
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <string.h>
20 #include <sys/socket.h>
21 #include <sys/time.h>
22 #include <sys/types.h>
23 #include <unistd.h>
24 
25 #ifndef ETH_MAX_MTU
26 #define ETH_MAX_MTU 0xFFFFU
27 #endif
28 
29 #ifndef UDP_SEGMENT
30 #define UDP_SEGMENT		103
31 #endif
32 
33 #ifndef SO_ZEROCOPY
34 #define SO_ZEROCOPY	60
35 #endif
36 
37 #ifndef MSG_ZEROCOPY
38 #define MSG_ZEROCOPY	0x4000000
39 #endif
40 
41 #define NUM_PKT		100
42 
43 static bool	cfg_cache_trash;
44 static int	cfg_cpu		= -1;
45 static int	cfg_connected	= true;
46 static int	cfg_family	= PF_UNSPEC;
47 static uint16_t	cfg_mss;
48 static int	cfg_payload_len	= (1472 * 42);
49 static int	cfg_port	= 8000;
50 static int	cfg_runtime_ms	= -1;
51 static bool	cfg_segment;
52 static bool	cfg_sendmmsg;
53 static bool	cfg_tcp;
54 static bool	cfg_zerocopy;
55 
56 static socklen_t cfg_alen;
57 static struct sockaddr_storage cfg_dst_addr;
58 
59 static bool interrupted;
60 static char buf[NUM_PKT][ETH_MAX_MTU];
61 
62 static void sigint_handler(int signum)
63 {
64 	if (signum == SIGINT)
65 		interrupted = true;
66 }
67 
68 static unsigned long gettimeofday_ms(void)
69 {
70 	struct timeval tv;
71 
72 	gettimeofday(&tv, NULL);
73 	return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
74 }
75 
76 static int set_cpu(int cpu)
77 {
78 	cpu_set_t mask;
79 
80 	CPU_ZERO(&mask);
81 	CPU_SET(cpu, &mask);
82 	if (sched_setaffinity(0, sizeof(mask), &mask))
83 		error(1, 0, "setaffinity %d", cpu);
84 
85 	return 0;
86 }
87 
88 static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr)
89 {
90 	struct sockaddr_in6 *addr6 = (void *) sockaddr;
91 	struct sockaddr_in *addr4 = (void *) sockaddr;
92 
93 	switch (domain) {
94 	case PF_INET:
95 		addr4->sin_family = AF_INET;
96 		addr4->sin_port = htons(cfg_port);
97 		if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
98 			error(1, 0, "ipv4 parse error: %s", str_addr);
99 		break;
100 	case PF_INET6:
101 		addr6->sin6_family = AF_INET6;
102 		addr6->sin6_port = htons(cfg_port);
103 		if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
104 			error(1, 0, "ipv6 parse error: %s", str_addr);
105 		break;
106 	default:
107 		error(1, 0, "illegal domain");
108 	}
109 }
110 
111 static void flush_zerocopy(int fd)
112 {
113 	struct msghdr msg = {0};	/* flush */
114 	int ret;
115 
116 	while (1) {
117 		ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
118 		if (ret == -1 && errno == EAGAIN)
119 			break;
120 		if (ret == -1)
121 			error(1, errno, "errqueue");
122 		if (msg.msg_flags != (MSG_ERRQUEUE | MSG_CTRUNC))
123 			error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags);
124 		msg.msg_flags = 0;
125 	}
126 }
127 
128 static int send_tcp(int fd, char *data)
129 {
130 	int ret, done = 0, count = 0;
131 
132 	while (done < cfg_payload_len) {
133 		ret = send(fd, data + done, cfg_payload_len - done,
134 			   cfg_zerocopy ? MSG_ZEROCOPY : 0);
135 		if (ret == -1)
136 			error(1, errno, "write");
137 
138 		done += ret;
139 		count++;
140 	}
141 
142 	return count;
143 }
144 
145 static int send_udp(int fd, char *data)
146 {
147 	int ret, total_len, len, count = 0;
148 
149 	total_len = cfg_payload_len;
150 
151 	while (total_len) {
152 		len = total_len < cfg_mss ? total_len : cfg_mss;
153 
154 		ret = sendto(fd, data, len, cfg_zerocopy ? MSG_ZEROCOPY : 0,
155 			     cfg_connected ? NULL : (void *)&cfg_dst_addr,
156 			     cfg_connected ? 0 : cfg_alen);
157 		if (ret == -1)
158 			error(1, errno, "write");
159 		if (ret != len)
160 			error(1, errno, "write: %uB != %uB\n", ret, len);
161 
162 		total_len -= len;
163 		count++;
164 	}
165 
166 	return count;
167 }
168 
169 static int send_udp_sendmmsg(int fd, char *data)
170 {
171 	const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN;
172 	struct mmsghdr mmsgs[max_nr_msg];
173 	struct iovec iov[max_nr_msg];
174 	unsigned int off = 0, left;
175 	int i = 0, ret;
176 
177 	memset(mmsgs, 0, sizeof(mmsgs));
178 
179 	left = cfg_payload_len;
180 	while (left) {
181 		if (i == max_nr_msg)
182 			error(1, 0, "sendmmsg: exceeds max_nr_msg");
183 
184 		iov[i].iov_base = data + off;
185 		iov[i].iov_len = cfg_mss < left ? cfg_mss : left;
186 
187 		mmsgs[i].msg_hdr.msg_iov = iov + i;
188 		mmsgs[i].msg_hdr.msg_iovlen = 1;
189 
190 		off += iov[i].iov_len;
191 		left -= iov[i].iov_len;
192 		i++;
193 	}
194 
195 	ret = sendmmsg(fd, mmsgs, i, cfg_zerocopy ? MSG_ZEROCOPY : 0);
196 	if (ret == -1)
197 		error(1, errno, "sendmmsg");
198 
199 	return ret;
200 }
201 
202 static void send_udp_segment_cmsg(struct cmsghdr *cm)
203 {
204 	uint16_t *valp;
205 
206 	cm->cmsg_level = SOL_UDP;
207 	cm->cmsg_type = UDP_SEGMENT;
208 	cm->cmsg_len = CMSG_LEN(sizeof(cfg_mss));
209 	valp = (void *)CMSG_DATA(cm);
210 	*valp = cfg_mss;
211 }
212 
213 static int send_udp_segment(int fd, char *data)
214 {
215 	char control[CMSG_SPACE(sizeof(cfg_mss))] = {0};
216 	struct msghdr msg = {0};
217 	struct iovec iov = {0};
218 	int ret;
219 
220 	iov.iov_base = data;
221 	iov.iov_len = cfg_payload_len;
222 
223 	msg.msg_iov = &iov;
224 	msg.msg_iovlen = 1;
225 
226 	msg.msg_control = control;
227 	msg.msg_controllen = sizeof(control);
228 	send_udp_segment_cmsg(CMSG_FIRSTHDR(&msg));
229 
230 	msg.msg_name = (void *)&cfg_dst_addr;
231 	msg.msg_namelen = cfg_alen;
232 
233 	ret = sendmsg(fd, &msg, cfg_zerocopy ? MSG_ZEROCOPY : 0);
234 	if (ret == -1)
235 		error(1, errno, "sendmsg");
236 	if (ret != iov.iov_len)
237 		error(1, 0, "sendmsg: %u != %lu\n", ret, iov.iov_len);
238 
239 	return 1;
240 }
241 
242 static void usage(const char *filepath)
243 {
244 	error(1, 0, "Usage: %s [-46cmStuz] [-C cpu] [-D dst ip] [-l secs] [-p port] [-s sendsize]",
245 		    filepath);
246 }
247 
248 static void parse_opts(int argc, char **argv)
249 {
250 	int max_len, hdrlen;
251 	int c;
252 
253 	while ((c = getopt(argc, argv, "46cC:D:l:mp:s:Stuz")) != -1) {
254 		switch (c) {
255 		case '4':
256 			if (cfg_family != PF_UNSPEC)
257 				error(1, 0, "Pass one of -4 or -6");
258 			cfg_family = PF_INET;
259 			cfg_alen = sizeof(struct sockaddr_in);
260 			break;
261 		case '6':
262 			if (cfg_family != PF_UNSPEC)
263 				error(1, 0, "Pass one of -4 or -6");
264 			cfg_family = PF_INET6;
265 			cfg_alen = sizeof(struct sockaddr_in6);
266 			break;
267 		case 'c':
268 			cfg_cache_trash = true;
269 			break;
270 		case 'C':
271 			cfg_cpu = strtol(optarg, NULL, 0);
272 			break;
273 		case 'D':
274 			setup_sockaddr(cfg_family, optarg, &cfg_dst_addr);
275 			break;
276 		case 'l':
277 			cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000;
278 			break;
279 		case 'm':
280 			cfg_sendmmsg = true;
281 			break;
282 		case 'p':
283 			cfg_port = strtoul(optarg, NULL, 0);
284 			break;
285 		case 's':
286 			cfg_payload_len = strtoul(optarg, NULL, 0);
287 			break;
288 		case 'S':
289 			cfg_segment = true;
290 			break;
291 		case 't':
292 			cfg_tcp = true;
293 			break;
294 		case 'u':
295 			cfg_connected = false;
296 			break;
297 		case 'z':
298 			cfg_zerocopy = true;
299 			break;
300 		}
301 	}
302 
303 	if (optind != argc)
304 		usage(argv[0]);
305 
306 	if (cfg_family == PF_UNSPEC)
307 		error(1, 0, "must pass one of -4 or -6");
308 	if (cfg_tcp && !cfg_connected)
309 		error(1, 0, "connectionless tcp makes no sense");
310 	if (cfg_segment && cfg_sendmmsg)
311 		error(1, 0, "cannot combine segment offload and sendmmsg");
312 
313 	if (cfg_family == PF_INET)
314 		hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr);
315 	else
316 		hdrlen = sizeof(struct ip6_hdr) + sizeof(struct udphdr);
317 
318 	cfg_mss = ETH_DATA_LEN - hdrlen;
319 	max_len = ETH_MAX_MTU - hdrlen;
320 
321 	if (cfg_payload_len > max_len)
322 		error(1, 0, "payload length %u exceeds max %u",
323 		      cfg_payload_len, max_len);
324 }
325 
326 static void set_pmtu_discover(int fd, bool is_ipv4)
327 {
328 	int level, name, val;
329 
330 	if (is_ipv4) {
331 		level	= SOL_IP;
332 		name	= IP_MTU_DISCOVER;
333 		val	= IP_PMTUDISC_DO;
334 	} else {
335 		level	= SOL_IPV6;
336 		name	= IPV6_MTU_DISCOVER;
337 		val	= IPV6_PMTUDISC_DO;
338 	}
339 
340 	if (setsockopt(fd, level, name, &val, sizeof(val)))
341 		error(1, errno, "setsockopt path mtu");
342 }
343 
344 int main(int argc, char **argv)
345 {
346 	unsigned long num_msgs, num_sends;
347 	unsigned long tnow, treport, tstop;
348 	int fd, i, val;
349 
350 	parse_opts(argc, argv);
351 
352 	if (cfg_cpu > 0)
353 		set_cpu(cfg_cpu);
354 
355 	for (i = 0; i < sizeof(buf[0]); i++)
356 		buf[0][i] = 'a' + (i % 26);
357 	for (i = 1; i < NUM_PKT; i++)
358 		memcpy(buf[i], buf[0], sizeof(buf[0]));
359 
360 	signal(SIGINT, sigint_handler);
361 
362 	fd = socket(cfg_family, cfg_tcp ? SOCK_STREAM : SOCK_DGRAM, 0);
363 	if (fd == -1)
364 		error(1, errno, "socket");
365 
366 	if (cfg_zerocopy) {
367 		val = 1;
368 		if (setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val)))
369 			error(1, errno, "setsockopt zerocopy");
370 	}
371 
372 	if (cfg_connected &&
373 	    connect(fd, (void *)&cfg_dst_addr, cfg_alen))
374 		error(1, errno, "connect");
375 
376 	if (cfg_segment)
377 		set_pmtu_discover(fd, cfg_family == PF_INET);
378 
379 	num_msgs = num_sends = 0;
380 	tnow = gettimeofday_ms();
381 	tstop = tnow + cfg_runtime_ms;
382 	treport = tnow + 1000;
383 
384 	i = 0;
385 	do {
386 		if (cfg_tcp)
387 			num_sends += send_tcp(fd, buf[i]);
388 		else if (cfg_segment)
389 			num_sends += send_udp_segment(fd, buf[i]);
390 		else if (cfg_sendmmsg)
391 			num_sends += send_udp_sendmmsg(fd, buf[i]);
392 		else
393 			num_sends += send_udp(fd, buf[i]);
394 		num_msgs++;
395 
396 		if (cfg_zerocopy && ((num_msgs & 0xF) == 0))
397 			flush_zerocopy(fd);
398 
399 		tnow = gettimeofday_ms();
400 		if (tnow > treport) {
401 			fprintf(stderr,
402 				"%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n",
403 				cfg_tcp ? "tcp" : "udp",
404 				(num_msgs * cfg_payload_len) >> 20,
405 				num_sends, num_msgs);
406 			num_msgs = num_sends = 0;
407 			treport = tnow + 1000;
408 		}
409 
410 		/* cold cache when writing buffer */
411 		if (cfg_cache_trash)
412 			i = ++i < NUM_PKT ? i : 0;
413 
414 	} while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop));
415 
416 	if (close(fd))
417 		error(1, errno, "close");
418 
419 	return 0;
420 }
421