1 // SPDX-License-Identifier: GPL-2.0-only 2 /* MSG_ZEROCOPY feature tests for vsock 3 * 4 * Copyright (C) 2023 SberDevices. 5 * 6 * Author: Arseniy Krasnov <avkrasnov@salutedevices.com> 7 */ 8 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <sys/mman.h> 13 #include <unistd.h> 14 #include <poll.h> 15 #include <linux/errqueue.h> 16 #include <linux/kernel.h> 17 #include <errno.h> 18 19 #include "control.h" 20 #include "vsock_test_zerocopy.h" 21 #include "msg_zerocopy_common.h" 22 23 #ifndef PAGE_SIZE 24 #define PAGE_SIZE 4096 25 #endif 26 27 #define VSOCK_TEST_DATA_MAX_IOV 3 28 29 struct vsock_test_data { 30 /* This test case if for SOCK_STREAM only. */ 31 bool stream_only; 32 /* Data must be zerocopied. This field is checked against 33 * field 'ee_code' of the 'struct sock_extended_err', which 34 * contains bit to detect that zerocopy transmission was 35 * fallbacked to copy mode. 36 */ 37 bool zerocopied; 38 /* Enable SO_ZEROCOPY option on the socket. Without enabled 39 * SO_ZEROCOPY, every MSG_ZEROCOPY transmission will behave 40 * like without MSG_ZEROCOPY flag. 41 */ 42 bool so_zerocopy; 43 /* 'errno' after 'sendmsg()' call. */ 44 int sendmsg_errno; 45 /* Number of valid elements in 'vecs'. */ 46 int vecs_cnt; 47 struct iovec vecs[VSOCK_TEST_DATA_MAX_IOV]; 48 }; 49 50 static struct vsock_test_data test_data_array[] = { 51 /* Last element has non-page aligned size. */ 52 { 53 .zerocopied = true, 54 .so_zerocopy = true, 55 .sendmsg_errno = 0, 56 .vecs_cnt = 3, 57 { 58 { NULL, PAGE_SIZE }, 59 { NULL, PAGE_SIZE }, 60 { NULL, 200 } 61 } 62 }, 63 /* All elements have page aligned base and size. */ 64 { 65 .zerocopied = true, 66 .so_zerocopy = true, 67 .sendmsg_errno = 0, 68 .vecs_cnt = 3, 69 { 70 { NULL, PAGE_SIZE }, 71 { NULL, PAGE_SIZE * 2 }, 72 { NULL, PAGE_SIZE * 3 } 73 } 74 }, 75 /* All elements have page aligned base and size. But 76 * data length is bigger than 64Kb. 77 */ 78 { 79 .zerocopied = true, 80 .so_zerocopy = true, 81 .sendmsg_errno = 0, 82 .vecs_cnt = 3, 83 { 84 { NULL, PAGE_SIZE * 16 }, 85 { NULL, PAGE_SIZE * 16 }, 86 { NULL, PAGE_SIZE * 16 } 87 } 88 }, 89 /* Middle element has both non-page aligned base and size. */ 90 { 91 .zerocopied = true, 92 .so_zerocopy = true, 93 .sendmsg_errno = 0, 94 .vecs_cnt = 3, 95 { 96 { NULL, PAGE_SIZE }, 97 { (void *)1, 100 }, 98 { NULL, PAGE_SIZE } 99 } 100 }, 101 /* Middle element is unmapped. */ 102 { 103 .zerocopied = false, 104 .so_zerocopy = true, 105 .sendmsg_errno = ENOMEM, 106 .vecs_cnt = 3, 107 { 108 { NULL, PAGE_SIZE }, 109 { MAP_FAILED, PAGE_SIZE }, 110 { NULL, PAGE_SIZE } 111 } 112 }, 113 /* Valid data, but SO_ZEROCOPY is off. This 114 * will trigger fallback to copy. 115 */ 116 { 117 .zerocopied = false, 118 .so_zerocopy = false, 119 .sendmsg_errno = 0, 120 .vecs_cnt = 1, 121 { 122 { NULL, PAGE_SIZE } 123 } 124 }, 125 /* Valid data, but message is bigger than peer's 126 * buffer, so this will trigger fallback to copy. 127 * This test is for SOCK_STREAM only, because 128 * for SOCK_SEQPACKET, 'sendmsg()' returns EMSGSIZE. 129 */ 130 { 131 .stream_only = true, 132 .zerocopied = false, 133 .so_zerocopy = true, 134 .sendmsg_errno = 0, 135 .vecs_cnt = 1, 136 { 137 { NULL, 100 * PAGE_SIZE } 138 } 139 }, 140 }; 141 142 #define POLL_TIMEOUT_MS 100 143 144 static void test_client(const struct test_opts *opts, 145 const struct vsock_test_data *test_data, 146 bool sock_seqpacket) 147 { 148 struct pollfd fds = { 0 }; 149 struct msghdr msg = { 0 }; 150 ssize_t sendmsg_res; 151 struct iovec *iovec; 152 int fd; 153 154 if (sock_seqpacket) 155 fd = vsock_seqpacket_connect(opts->peer_cid, opts->peer_port); 156 else 157 fd = vsock_stream_connect(opts->peer_cid, opts->peer_port); 158 159 if (fd < 0) { 160 perror("connect"); 161 exit(EXIT_FAILURE); 162 } 163 164 if (test_data->so_zerocopy) 165 enable_so_zerocopy(fd); 166 167 iovec = alloc_test_iovec(test_data->vecs, test_data->vecs_cnt); 168 169 msg.msg_iov = iovec; 170 msg.msg_iovlen = test_data->vecs_cnt; 171 172 errno = 0; 173 174 sendmsg_res = sendmsg(fd, &msg, MSG_ZEROCOPY); 175 if (errno != test_data->sendmsg_errno) { 176 fprintf(stderr, "expected 'errno' == %i, got %i\n", 177 test_data->sendmsg_errno, errno); 178 exit(EXIT_FAILURE); 179 } 180 181 if (!errno) { 182 if (sendmsg_res != iovec_bytes(iovec, test_data->vecs_cnt)) { 183 fprintf(stderr, "expected 'sendmsg()' == %li, got %li\n", 184 iovec_bytes(iovec, test_data->vecs_cnt), 185 sendmsg_res); 186 exit(EXIT_FAILURE); 187 } 188 } 189 190 fds.fd = fd; 191 fds.events = 0; 192 193 if (poll(&fds, 1, POLL_TIMEOUT_MS) < 0) { 194 perror("poll"); 195 exit(EXIT_FAILURE); 196 } 197 198 if (fds.revents & POLLERR) { 199 vsock_recv_completion(fd, &test_data->zerocopied); 200 } else if (test_data->so_zerocopy && !test_data->sendmsg_errno) { 201 /* If we don't have data in the error queue, but 202 * SO_ZEROCOPY was enabled and 'sendmsg()' was 203 * successful - this is an error. 204 */ 205 fprintf(stderr, "POLLERR expected\n"); 206 exit(EXIT_FAILURE); 207 } 208 209 if (!test_data->sendmsg_errno) 210 control_writeulong(iovec_hash_djb2(iovec, test_data->vecs_cnt)); 211 else 212 control_writeulong(0); 213 214 control_writeln("DONE"); 215 free_test_iovec(test_data->vecs, iovec, test_data->vecs_cnt); 216 close(fd); 217 } 218 219 void test_stream_msgzcopy_client(const struct test_opts *opts) 220 { 221 int i; 222 223 for (i = 0; i < ARRAY_SIZE(test_data_array); i++) 224 test_client(opts, &test_data_array[i], false); 225 } 226 227 void test_seqpacket_msgzcopy_client(const struct test_opts *opts) 228 { 229 int i; 230 231 for (i = 0; i < ARRAY_SIZE(test_data_array); i++) { 232 if (test_data_array[i].stream_only) 233 continue; 234 235 test_client(opts, &test_data_array[i], true); 236 } 237 } 238 239 static void test_server(const struct test_opts *opts, 240 const struct vsock_test_data *test_data, 241 bool sock_seqpacket) 242 { 243 unsigned long remote_hash; 244 unsigned long local_hash; 245 ssize_t total_bytes_rec; 246 unsigned char *data; 247 size_t data_len; 248 int fd; 249 250 if (sock_seqpacket) 251 fd = vsock_seqpacket_accept(VMADDR_CID_ANY, opts->peer_port, NULL); 252 else 253 fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL); 254 255 if (fd < 0) { 256 perror("accept"); 257 exit(EXIT_FAILURE); 258 } 259 260 data_len = iovec_bytes(test_data->vecs, test_data->vecs_cnt); 261 262 data = malloc(data_len); 263 if (!data) { 264 perror("malloc"); 265 exit(EXIT_FAILURE); 266 } 267 268 total_bytes_rec = 0; 269 270 while (total_bytes_rec != data_len) { 271 ssize_t bytes_rec; 272 273 bytes_rec = read(fd, data + total_bytes_rec, 274 data_len - total_bytes_rec); 275 if (bytes_rec <= 0) 276 break; 277 278 total_bytes_rec += bytes_rec; 279 } 280 281 if (test_data->sendmsg_errno == 0) 282 local_hash = hash_djb2(data, data_len); 283 else 284 local_hash = 0; 285 286 free(data); 287 288 /* Waiting for some result. */ 289 remote_hash = control_readulong(); 290 if (remote_hash != local_hash) { 291 fprintf(stderr, "hash mismatch\n"); 292 exit(EXIT_FAILURE); 293 } 294 295 control_expectln("DONE"); 296 close(fd); 297 } 298 299 void test_stream_msgzcopy_server(const struct test_opts *opts) 300 { 301 int i; 302 303 for (i = 0; i < ARRAY_SIZE(test_data_array); i++) 304 test_server(opts, &test_data_array[i], false); 305 } 306 307 void test_seqpacket_msgzcopy_server(const struct test_opts *opts) 308 { 309 int i; 310 311 for (i = 0; i < ARRAY_SIZE(test_data_array); i++) { 312 if (test_data_array[i].stream_only) 313 continue; 314 315 test_server(opts, &test_data_array[i], true); 316 } 317 } 318 319 void test_stream_msgzcopy_empty_errq_client(const struct test_opts *opts) 320 { 321 struct msghdr msg = { 0 }; 322 char cmsg_data[128]; 323 ssize_t res; 324 int fd; 325 326 fd = vsock_stream_connect(opts->peer_cid, opts->peer_port); 327 if (fd < 0) { 328 perror("connect"); 329 exit(EXIT_FAILURE); 330 } 331 332 msg.msg_control = cmsg_data; 333 msg.msg_controllen = sizeof(cmsg_data); 334 335 res = recvmsg(fd, &msg, MSG_ERRQUEUE); 336 if (res != -1) { 337 fprintf(stderr, "expected 'recvmsg(2)' failure, got %zi\n", 338 res); 339 exit(EXIT_FAILURE); 340 } 341 342 control_writeln("DONE"); 343 close(fd); 344 } 345 346 void test_stream_msgzcopy_empty_errq_server(const struct test_opts *opts) 347 { 348 int fd; 349 350 fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL); 351 if (fd < 0) { 352 perror("accept"); 353 exit(EXIT_FAILURE); 354 } 355 356 control_expectln("DONE"); 357 close(fd); 358 } 359