1 /* SPDX-License-Identifier: MIT */ 2 /* based on linux-kernel/tools/testing/selftests/net/msg_zerocopy.c */ 3 #include <assert.h> 4 #include <errno.h> 5 #include <error.h> 6 #include <fcntl.h> 7 #include <limits.h> 8 #include <stdbool.h> 9 #include <stdint.h> 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <string.h> 13 #include <unistd.h> 14 15 #include <arpa/inet.h> 16 #include <linux/errqueue.h> 17 #include <linux/if_packet.h> 18 #include <linux/io_uring.h> 19 #include <linux/ipv6.h> 20 #include <linux/socket.h> 21 #include <linux/sockios.h> 22 #include <net/ethernet.h> 23 #include <net/if.h> 24 #include <netinet/in.h> 25 #include <netinet/ip.h> 26 #include <netinet/ip6.h> 27 #include <netinet/tcp.h> 28 #include <netinet/udp.h> 29 #include <sys/ioctl.h> 30 #include <sys/mman.h> 31 #include <sys/resource.h> 32 #include <sys/socket.h> 33 #include <sys/stat.h> 34 #include <sys/time.h> 35 #include <sys/types.h> 36 #include <sys/un.h> 37 #include <sys/wait.h> 38 39 #include <io_uring/mini_liburing.h> 40 41 #define NOTIF_TAG 0xfffffffULL 42 #define NONZC_TAG 0 43 #define ZC_TAG 1 44 45 enum { 46 MODE_NONZC = 0, 47 MODE_ZC = 1, 48 MODE_ZC_FIXED = 2, 49 MODE_MIXED = 3, 50 }; 51 52 static bool cfg_cork = false; 53 static int cfg_mode = MODE_ZC_FIXED; 54 static int cfg_nr_reqs = 8; 55 static int cfg_family = PF_UNSPEC; 56 static int cfg_payload_len; 57 static int cfg_port = 8000; 58 static int cfg_runtime_ms = 4200; 59 60 static socklen_t cfg_alen; 61 static struct sockaddr_storage cfg_dst_addr; 62 63 static char payload[IP_MAXPACKET] __attribute__((aligned(4096))); 64 65 static unsigned long gettimeofday_ms(void) 66 { 67 struct timeval tv; 68 69 gettimeofday(&tv, NULL); 70 return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); 71 } 72 73 static void do_setsockopt(int fd, int level, int optname, int val) 74 { 75 if (setsockopt(fd, level, optname, &val, sizeof(val))) 76 error(1, errno, "setsockopt %d.%d: %d", level, optname, val); 77 } 78 79 static int do_setup_tx(int domain, int type, int protocol) 80 { 81 int fd; 82 83 fd = socket(domain, type, protocol); 84 if (fd == -1) 85 error(1, errno, "socket t"); 86 87 do_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, 1 << 21); 88 89 if (connect(fd, (void *) &cfg_dst_addr, cfg_alen)) 90 error(1, errno, "connect"); 91 return fd; 92 } 93 94 static void do_tx(int domain, int type, int protocol) 95 { 96 struct io_uring_sqe *sqe; 97 struct io_uring_cqe *cqe; 98 unsigned long packets = 0, bytes = 0; 99 struct io_uring ring; 100 struct iovec iov; 101 uint64_t tstop; 102 int i, fd, ret; 103 int compl_cqes = 0; 104 105 fd = do_setup_tx(domain, type, protocol); 106 107 ret = io_uring_queue_init(512, &ring, 0); 108 if (ret) 109 error(1, ret, "io_uring: queue init"); 110 111 iov.iov_base = payload; 112 iov.iov_len = cfg_payload_len; 113 114 ret = io_uring_register_buffers(&ring, &iov, 1); 115 if (ret) 116 error(1, ret, "io_uring: buffer registration"); 117 118 tstop = gettimeofday_ms() + cfg_runtime_ms; 119 do { 120 if (cfg_cork) 121 do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 1); 122 123 for (i = 0; i < cfg_nr_reqs; i++) { 124 unsigned zc_flags = 0; 125 unsigned buf_idx = 0; 126 unsigned mode = cfg_mode; 127 unsigned msg_flags = MSG_WAITALL; 128 129 if (cfg_mode == MODE_MIXED) 130 mode = rand() % 3; 131 132 sqe = io_uring_get_sqe(&ring); 133 134 if (mode == MODE_NONZC) { 135 io_uring_prep_send(sqe, fd, payload, 136 cfg_payload_len, msg_flags); 137 sqe->user_data = NONZC_TAG; 138 } else { 139 io_uring_prep_sendzc(sqe, fd, payload, 140 cfg_payload_len, 141 msg_flags, zc_flags); 142 if (mode == MODE_ZC_FIXED) { 143 sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; 144 sqe->buf_index = buf_idx; 145 } 146 sqe->user_data = ZC_TAG; 147 } 148 } 149 150 ret = io_uring_submit(&ring); 151 if (ret != cfg_nr_reqs) 152 error(1, ret, "submit"); 153 154 if (cfg_cork) 155 do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0); 156 for (i = 0; i < cfg_nr_reqs; i++) { 157 ret = io_uring_wait_cqe(&ring, &cqe); 158 if (ret) 159 error(1, ret, "wait cqe"); 160 161 if (cqe->user_data != NONZC_TAG && 162 cqe->user_data != ZC_TAG) 163 error(1, -EINVAL, "invalid cqe->user_data"); 164 165 if (cqe->flags & IORING_CQE_F_NOTIF) { 166 if (cqe->flags & IORING_CQE_F_MORE) 167 error(1, -EINVAL, "invalid notif flags"); 168 if (compl_cqes <= 0) 169 error(1, -EINVAL, "notification mismatch"); 170 compl_cqes--; 171 i--; 172 io_uring_cqe_seen(&ring); 173 continue; 174 } 175 if (cqe->flags & IORING_CQE_F_MORE) { 176 if (cqe->user_data != ZC_TAG) 177 error(1, cqe->res, "unexpected F_MORE"); 178 compl_cqes++; 179 } 180 if (cqe->res >= 0) { 181 packets++; 182 bytes += cqe->res; 183 } else if (cqe->res != -EAGAIN) { 184 error(1, cqe->res, "send failed"); 185 } 186 io_uring_cqe_seen(&ring); 187 } 188 } while (gettimeofday_ms() < tstop); 189 190 while (compl_cqes) { 191 ret = io_uring_wait_cqe(&ring, &cqe); 192 if (ret) 193 error(1, ret, "wait cqe"); 194 if (cqe->flags & IORING_CQE_F_MORE) 195 error(1, -EINVAL, "invalid notif flags"); 196 if (!(cqe->flags & IORING_CQE_F_NOTIF)) 197 error(1, -EINVAL, "missing notif flag"); 198 199 io_uring_cqe_seen(&ring); 200 compl_cqes--; 201 } 202 203 fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n", 204 packets, bytes >> 20, 205 packets / (cfg_runtime_ms / 1000), 206 (bytes >> 20) / (cfg_runtime_ms / 1000)); 207 208 if (close(fd)) 209 error(1, errno, "close"); 210 } 211 212 static void do_test(int domain, int type, int protocol) 213 { 214 int i; 215 216 for (i = 0; i < IP_MAXPACKET; i++) 217 payload[i] = 'a' + (i % 26); 218 do_tx(domain, type, protocol); 219 } 220 221 static void usage(const char *filepath) 222 { 223 error(1, 0, "Usage: %s (-4|-6) (udp|tcp) -D<dst_ip> [-s<payload size>] " 224 "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>]", filepath); 225 } 226 227 static void parse_opts(int argc, char **argv) 228 { 229 const int max_payload_len = sizeof(payload) - 230 sizeof(struct ipv6hdr) - 231 sizeof(struct tcphdr) - 232 40 /* max tcp options */; 233 struct sockaddr_in6 *addr6 = (void *) &cfg_dst_addr; 234 struct sockaddr_in *addr4 = (void *) &cfg_dst_addr; 235 char *daddr = NULL; 236 int c; 237 238 if (argc <= 1) 239 usage(argv[0]); 240 cfg_payload_len = max_payload_len; 241 242 while ((c = getopt(argc, argv, "46D:p:s:t:n:c:m:")) != -1) { 243 switch (c) { 244 case '4': 245 if (cfg_family != PF_UNSPEC) 246 error(1, 0, "Pass one of -4 or -6"); 247 cfg_family = PF_INET; 248 cfg_alen = sizeof(struct sockaddr_in); 249 break; 250 case '6': 251 if (cfg_family != PF_UNSPEC) 252 error(1, 0, "Pass one of -4 or -6"); 253 cfg_family = PF_INET6; 254 cfg_alen = sizeof(struct sockaddr_in6); 255 break; 256 case 'D': 257 daddr = optarg; 258 break; 259 case 'p': 260 cfg_port = strtoul(optarg, NULL, 0); 261 break; 262 case 's': 263 cfg_payload_len = strtoul(optarg, NULL, 0); 264 break; 265 case 't': 266 cfg_runtime_ms = 200 + strtoul(optarg, NULL, 10) * 1000; 267 break; 268 case 'n': 269 cfg_nr_reqs = strtoul(optarg, NULL, 0); 270 break; 271 case 'c': 272 cfg_cork = strtol(optarg, NULL, 0); 273 break; 274 case 'm': 275 cfg_mode = strtol(optarg, NULL, 0); 276 break; 277 } 278 } 279 280 switch (cfg_family) { 281 case PF_INET: 282 memset(addr4, 0, sizeof(*addr4)); 283 addr4->sin_family = AF_INET; 284 addr4->sin_port = htons(cfg_port); 285 if (daddr && 286 inet_pton(AF_INET, daddr, &(addr4->sin_addr)) != 1) 287 error(1, 0, "ipv4 parse error: %s", daddr); 288 break; 289 case PF_INET6: 290 memset(addr6, 0, sizeof(*addr6)); 291 addr6->sin6_family = AF_INET6; 292 addr6->sin6_port = htons(cfg_port); 293 if (daddr && 294 inet_pton(AF_INET6, daddr, &(addr6->sin6_addr)) != 1) 295 error(1, 0, "ipv6 parse error: %s", daddr); 296 break; 297 default: 298 error(1, 0, "illegal domain"); 299 } 300 301 if (cfg_payload_len > max_payload_len) 302 error(1, 0, "-s: payload exceeds max (%d)", max_payload_len); 303 if (optind != argc - 1) 304 usage(argv[0]); 305 } 306 307 int main(int argc, char **argv) 308 { 309 const char *cfg_test = argv[argc - 1]; 310 311 parse_opts(argc, argv); 312 313 if (!strcmp(cfg_test, "tcp")) 314 do_test(cfg_family, SOCK_STREAM, 0); 315 else if (!strcmp(cfg_test, "udp")) 316 do_test(cfg_family, SOCK_DGRAM, 0); 317 else 318 error(1, 0, "unknown cfg_test %s", cfg_test); 319 return 0; 320 } 321