1 // SPDX-License-Identifier: GPL-2.0 2 #include <test_progs.h> 3 #include <network_helpers.h> 4 5 static void test_xdp_adjust_tail_shrink(void) 6 { 7 const char *file = "./test_xdp_adjust_tail_shrink.bpf.o"; 8 __u32 expect_sz; 9 struct bpf_object *obj; 10 int err, prog_fd; 11 char buf[128]; 12 LIBBPF_OPTS(bpf_test_run_opts, topts, 13 .data_in = &pkt_v4, 14 .data_size_in = sizeof(pkt_v4), 15 .data_out = buf, 16 .data_size_out = sizeof(buf), 17 .repeat = 1, 18 ); 19 20 err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 21 if (!ASSERT_OK(err, "test_xdp_adjust_tail_shrink")) 22 return; 23 24 err = bpf_prog_test_run_opts(prog_fd, &topts); 25 ASSERT_OK(err, "ipv4"); 26 ASSERT_EQ(topts.retval, XDP_DROP, "ipv4 retval"); 27 28 expect_sz = sizeof(pkt_v6) - 20; /* Test shrink with 20 bytes */ 29 topts.data_in = &pkt_v6; 30 topts.data_size_in = sizeof(pkt_v6); 31 topts.data_size_out = sizeof(buf); 32 err = bpf_prog_test_run_opts(prog_fd, &topts); 33 ASSERT_OK(err, "ipv6"); 34 ASSERT_EQ(topts.retval, XDP_TX, "ipv6 retval"); 35 ASSERT_EQ(topts.data_size_out, expect_sz, "ipv6 size"); 36 37 bpf_object__close(obj); 38 } 39 40 static void test_xdp_adjust_tail_grow(bool is_64k_pagesize) 41 { 42 const char *file = "./test_xdp_adjust_tail_grow.bpf.o"; 43 struct bpf_object *obj; 44 char buf[8192]; /* avoid segfault: large buf to hold grow results */ 45 __u32 expect_sz; 46 int err, prog_fd; 47 LIBBPF_OPTS(bpf_test_run_opts, topts, 48 .data_in = &pkt_v4, 49 .data_out = buf, 50 .data_size_out = sizeof(buf), 51 .repeat = 1, 52 ); 53 54 /* topts.data_size_in as a special signal to bpf prog */ 55 if (is_64k_pagesize) 56 topts.data_size_in = sizeof(pkt_v4) - 1; 57 else 58 topts.data_size_in = sizeof(pkt_v4); 59 60 err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 61 if (!ASSERT_OK(err, "test_xdp_adjust_tail_grow")) 62 return; 63 64 err = bpf_prog_test_run_opts(prog_fd, &topts); 65 ASSERT_OK(err, "ipv4"); 66 ASSERT_EQ(topts.retval, XDP_DROP, "ipv4 retval"); 67 68 expect_sz = sizeof(pkt_v6) + 40; /* Test grow with 40 bytes */ 69 topts.data_in = &pkt_v6; 70 topts.data_size_in = sizeof(pkt_v6); 71 topts.data_size_out = sizeof(buf); 72 err = bpf_prog_test_run_opts(prog_fd, &topts); 73 ASSERT_OK(err, "ipv6"); 74 ASSERT_EQ(topts.retval, XDP_TX, "ipv6 retval"); 75 ASSERT_EQ(topts.data_size_out, expect_sz, "ipv6 size"); 76 77 bpf_object__close(obj); 78 } 79 80 static void test_xdp_adjust_tail_grow2(void) 81 { 82 const char *file = "./test_xdp_adjust_tail_grow.bpf.o"; 83 char buf[4096]; /* avoid segfault: large buf to hold grow results */ 84 struct bpf_object *obj; 85 int err, cnt, i; 86 int max_grow, prog_fd; 87 /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) */ 88 #if defined(__s390x__) 89 int tailroom = 512; 90 #elif defined(__powerpc__) 91 int tailroom = 384; 92 #else 93 int tailroom = 320; 94 #endif 95 96 LIBBPF_OPTS(bpf_test_run_opts, tattr, 97 .repeat = 1, 98 .data_in = &buf, 99 .data_out = &buf, 100 .data_size_in = 0, /* Per test */ 101 .data_size_out = 0, /* Per test */ 102 ); 103 104 err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 105 if (!ASSERT_OK(err, "test_xdp_adjust_tail_grow")) 106 return; 107 108 /* Test case-64 */ 109 memset(buf, 1, sizeof(buf)); 110 tattr.data_size_in = 64; /* Determine test case via pkt size */ 111 tattr.data_size_out = 128; /* Limit copy_size */ 112 /* Kernel side alloc packet memory area that is zero init */ 113 err = bpf_prog_test_run_opts(prog_fd, &tattr); 114 115 ASSERT_EQ(errno, ENOSPC, "case-64 errno"); /* Due limit copy_size in bpf_test_finish */ 116 ASSERT_EQ(tattr.retval, XDP_TX, "case-64 retval"); 117 ASSERT_EQ(tattr.data_size_out, 192, "case-64 data_size_out"); /* Expected grow size */ 118 119 /* Extra checks for data contents */ 120 ASSERT_EQ(buf[0], 1, "case-64-data buf[0]"); /* 0-63 memset to 1 */ 121 ASSERT_EQ(buf[63], 1, "case-64-data buf[63]"); 122 ASSERT_EQ(buf[64], 0, "case-64-data buf[64]"); /* 64-127 memset to 0 */ 123 ASSERT_EQ(buf[127], 0, "case-64-data buf[127]"); 124 ASSERT_EQ(buf[128], 1, "case-64-data buf[128]"); /* 128-191 memset to 1 */ 125 ASSERT_EQ(buf[191], 1, "case-64-data buf[191]"); 126 127 /* Test case-128 */ 128 memset(buf, 2, sizeof(buf)); 129 tattr.data_size_in = 128; /* Determine test case via pkt size */ 130 tattr.data_size_out = sizeof(buf); /* Copy everything */ 131 err = bpf_prog_test_run_opts(prog_fd, &tattr); 132 133 max_grow = 4096 - XDP_PACKET_HEADROOM - tailroom; /* 3520 */ 134 ASSERT_OK(err, "case-128"); 135 ASSERT_EQ(tattr.retval, XDP_TX, "case-128 retval"); 136 ASSERT_EQ(tattr.data_size_out, max_grow, "case-128 data_size_out"); /* Expect max grow */ 137 138 /* Extra checks for data content: Count grow size, will contain zeros */ 139 for (i = 0, cnt = 0; i < sizeof(buf); i++) { 140 if (buf[i] == 0) 141 cnt++; 142 } 143 ASSERT_EQ(cnt, max_grow - tattr.data_size_in, "case-128-data cnt"); /* Grow increase */ 144 ASSERT_EQ(tattr.data_size_out, max_grow, "case-128-data data_size_out"); /* Total grow */ 145 146 bpf_object__close(obj); 147 } 148 149 static void test_xdp_adjust_frags_tail_shrink(void) 150 { 151 const char *file = "./test_xdp_adjust_tail_shrink.bpf.o"; 152 __u32 exp_size; 153 struct bpf_program *prog; 154 struct bpf_object *obj; 155 int err, prog_fd; 156 __u8 *buf; 157 LIBBPF_OPTS(bpf_test_run_opts, topts); 158 159 /* For the individual test cases, the first byte in the packet 160 * indicates which test will be run. 161 */ 162 obj = bpf_object__open(file); 163 if (libbpf_get_error(obj)) 164 return; 165 166 prog = bpf_object__next_program(obj, NULL); 167 if (bpf_object__load(obj)) 168 return; 169 170 prog_fd = bpf_program__fd(prog); 171 172 buf = malloc(9000); 173 if (!ASSERT_OK_PTR(buf, "alloc buf 9Kb")) 174 goto out; 175 176 memset(buf, 0, 9000); 177 178 /* Test case removing 10 bytes from last frag, NOT freeing it */ 179 exp_size = 8990; /* 9000 - 10 */ 180 topts.data_in = buf; 181 topts.data_out = buf; 182 topts.data_size_in = 9000; 183 topts.data_size_out = 9000; 184 err = bpf_prog_test_run_opts(prog_fd, &topts); 185 186 ASSERT_OK(err, "9Kb-10b"); 187 ASSERT_EQ(topts.retval, XDP_TX, "9Kb-10b retval"); 188 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-10b size"); 189 190 /* Test case removing one of two pages, assuming 4K pages */ 191 buf[0] = 1; 192 exp_size = 4900; /* 9000 - 4100 */ 193 194 topts.data_size_out = 9000; /* reset from previous invocation */ 195 err = bpf_prog_test_run_opts(prog_fd, &topts); 196 197 ASSERT_OK(err, "9Kb-4Kb"); 198 ASSERT_EQ(topts.retval, XDP_TX, "9Kb-4Kb retval"); 199 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-4Kb size"); 200 201 /* Test case removing two pages resulting in a linear xdp_buff */ 202 buf[0] = 2; 203 exp_size = 800; /* 9000 - 8200 */ 204 topts.data_size_out = 9000; /* reset from previous invocation */ 205 err = bpf_prog_test_run_opts(prog_fd, &topts); 206 207 ASSERT_OK(err, "9Kb-9Kb"); 208 ASSERT_EQ(topts.retval, XDP_TX, "9Kb-9Kb retval"); 209 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-9Kb size"); 210 211 free(buf); 212 out: 213 bpf_object__close(obj); 214 } 215 216 static void test_xdp_adjust_frags_tail_grow_4k(void) 217 { 218 const char *file = "./test_xdp_adjust_tail_grow.bpf.o"; 219 __u32 exp_size; 220 struct bpf_program *prog; 221 struct bpf_object *obj; 222 int err, i, prog_fd; 223 __u8 *buf; 224 LIBBPF_OPTS(bpf_test_run_opts, topts); 225 226 obj = bpf_object__open(file); 227 if (libbpf_get_error(obj)) 228 return; 229 230 prog = bpf_object__next_program(obj, NULL); 231 if (bpf_object__load(obj)) 232 goto out; 233 234 prog_fd = bpf_program__fd(prog); 235 236 buf = malloc(16384); 237 if (!ASSERT_OK_PTR(buf, "alloc buf 16Kb")) 238 goto out; 239 240 /* Test case add 10 bytes to last frag */ 241 memset(buf, 1, 16384); 242 exp_size = 9000 + 10; 243 244 topts.data_in = buf; 245 topts.data_out = buf; 246 topts.data_size_in = 9000; 247 topts.data_size_out = 16384; 248 err = bpf_prog_test_run_opts(prog_fd, &topts); 249 250 ASSERT_OK(err, "9Kb+10b"); 251 ASSERT_EQ(topts.retval, XDP_TX, "9Kb+10b retval"); 252 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb+10b size"); 253 254 for (i = 0; i < 9000; i++) { 255 if (buf[i] != 1) 256 ASSERT_EQ(buf[i], 1, "9Kb+10b-old"); 257 } 258 259 for (i = 9000; i < 9010; i++) { 260 if (buf[i] != 0) 261 ASSERT_EQ(buf[i], 0, "9Kb+10b-new"); 262 } 263 264 for (i = 9010; i < 16384; i++) { 265 if (buf[i] != 1) 266 ASSERT_EQ(buf[i], 1, "9Kb+10b-untouched"); 267 } 268 269 /* Test a too large grow */ 270 memset(buf, 1, 16384); 271 exp_size = 9001; 272 273 topts.data_in = topts.data_out = buf; 274 topts.data_size_in = 9001; 275 topts.data_size_out = 16384; 276 err = bpf_prog_test_run_opts(prog_fd, &topts); 277 278 ASSERT_OK(err, "9Kb+10b"); 279 ASSERT_EQ(topts.retval, XDP_DROP, "9Kb+10b retval"); 280 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb+10b size"); 281 282 free(buf); 283 out: 284 bpf_object__close(obj); 285 } 286 287 static void test_xdp_adjust_frags_tail_grow_64k(void) 288 { 289 const char *file = "./test_xdp_adjust_tail_grow.bpf.o"; 290 __u32 exp_size; 291 struct bpf_program *prog; 292 struct bpf_object *obj; 293 int err, i, prog_fd; 294 __u8 *buf; 295 LIBBPF_OPTS(bpf_test_run_opts, topts); 296 297 obj = bpf_object__open(file); 298 if (libbpf_get_error(obj)) 299 return; 300 301 prog = bpf_object__next_program(obj, NULL); 302 if (bpf_object__load(obj)) 303 goto out; 304 305 prog_fd = bpf_program__fd(prog); 306 307 buf = malloc(262144); 308 if (!ASSERT_OK_PTR(buf, "alloc buf 256Kb")) 309 goto out; 310 311 /* Test case add 10 bytes to last frag */ 312 memset(buf, 1, 262144); 313 exp_size = 90000 + 10; 314 315 topts.data_in = buf; 316 topts.data_out = buf; 317 topts.data_size_in = 90000; 318 topts.data_size_out = 262144; 319 err = bpf_prog_test_run_opts(prog_fd, &topts); 320 321 ASSERT_OK(err, "90Kb+10b"); 322 ASSERT_EQ(topts.retval, XDP_TX, "90Kb+10b retval"); 323 ASSERT_EQ(topts.data_size_out, exp_size, "90Kb+10b size"); 324 325 for (i = 0; i < 90000; i++) { 326 if (buf[i] != 1) 327 ASSERT_EQ(buf[i], 1, "90Kb+10b-old"); 328 } 329 330 for (i = 90000; i < 90010; i++) { 331 if (buf[i] != 0) 332 ASSERT_EQ(buf[i], 0, "90Kb+10b-new"); 333 } 334 335 for (i = 90010; i < 262144; i++) { 336 if (buf[i] != 1) 337 ASSERT_EQ(buf[i], 1, "90Kb+10b-untouched"); 338 } 339 340 /* Test a too large grow */ 341 memset(buf, 1, 262144); 342 exp_size = 90001; 343 344 topts.data_in = topts.data_out = buf; 345 topts.data_size_in = 90001; 346 topts.data_size_out = 262144; 347 err = bpf_prog_test_run_opts(prog_fd, &topts); 348 349 ASSERT_OK(err, "90Kb+10b"); 350 ASSERT_EQ(topts.retval, XDP_DROP, "90Kb+10b retval"); 351 ASSERT_EQ(topts.data_size_out, exp_size, "90Kb+10b size"); 352 353 free(buf); 354 out: 355 bpf_object__close(obj); 356 } 357 358 void test_xdp_adjust_tail(void) 359 { 360 int page_size = getpagesize(); 361 362 if (test__start_subtest("xdp_adjust_tail_shrink")) 363 test_xdp_adjust_tail_shrink(); 364 if (test__start_subtest("xdp_adjust_tail_grow")) 365 test_xdp_adjust_tail_grow(page_size == 65536); 366 if (test__start_subtest("xdp_adjust_tail_grow2")) 367 test_xdp_adjust_tail_grow2(); 368 if (test__start_subtest("xdp_adjust_frags_tail_shrink")) 369 test_xdp_adjust_frags_tail_shrink(); 370 if (test__start_subtest("xdp_adjust_frags_tail_grow")) { 371 if (page_size == 65536) 372 test_xdp_adjust_frags_tail_grow_64k(); 373 else 374 test_xdp_adjust_frags_tail_grow_4k(); 375 } 376 } 377