1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include <network_helpers.h>
4
test_xdp_adjust_tail_shrink(void)5 static void test_xdp_adjust_tail_shrink(void)
6 {
7 const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
8 __u32 expect_sz;
9 struct bpf_object *obj;
10 int err, prog_fd;
11 char buf[128];
12 LIBBPF_OPTS(bpf_test_run_opts, topts,
13 .data_in = &pkt_v4,
14 .data_size_in = sizeof(pkt_v4),
15 .data_out = buf,
16 .data_size_out = sizeof(buf),
17 .repeat = 1,
18 );
19
20 err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
21 if (!ASSERT_OK(err, "test_xdp_adjust_tail_shrink"))
22 return;
23
24 err = bpf_prog_test_run_opts(prog_fd, &topts);
25 ASSERT_OK(err, "ipv4");
26 ASSERT_EQ(topts.retval, XDP_DROP, "ipv4 retval");
27
28 expect_sz = sizeof(pkt_v6) - 20; /* Test shrink with 20 bytes */
29 topts.data_in = &pkt_v6;
30 topts.data_size_in = sizeof(pkt_v6);
31 topts.data_size_out = sizeof(buf);
32 err = bpf_prog_test_run_opts(prog_fd, &topts);
33 ASSERT_OK(err, "ipv6");
34 ASSERT_EQ(topts.retval, XDP_TX, "ipv6 retval");
35 ASSERT_EQ(topts.data_size_out, expect_sz, "ipv6 size");
36
37 bpf_object__close(obj);
38 }
39
test_xdp_adjust_tail_grow(bool is_64k_pagesize)40 static void test_xdp_adjust_tail_grow(bool is_64k_pagesize)
41 {
42 const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
43 struct bpf_object *obj;
44 char buf[8192]; /* avoid segfault: large buf to hold grow results */
45 __u32 expect_sz;
46 int err, prog_fd;
47 LIBBPF_OPTS(bpf_test_run_opts, topts,
48 .data_in = &pkt_v4,
49 .data_out = buf,
50 .data_size_out = sizeof(buf),
51 .repeat = 1,
52 );
53
54 /* topts.data_size_in as a special signal to bpf prog */
55 if (is_64k_pagesize)
56 topts.data_size_in = sizeof(pkt_v4) - 1;
57 else
58 topts.data_size_in = sizeof(pkt_v4);
59
60 err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
61 if (!ASSERT_OK(err, "test_xdp_adjust_tail_grow"))
62 return;
63
64 err = bpf_prog_test_run_opts(prog_fd, &topts);
65 ASSERT_OK(err, "ipv4");
66 ASSERT_EQ(topts.retval, XDP_DROP, "ipv4 retval");
67
68 expect_sz = sizeof(pkt_v6) + 40; /* Test grow with 40 bytes */
69 topts.data_in = &pkt_v6;
70 topts.data_size_in = sizeof(pkt_v6);
71 topts.data_size_out = sizeof(buf);
72 err = bpf_prog_test_run_opts(prog_fd, &topts);
73 ASSERT_OK(err, "ipv6");
74 ASSERT_EQ(topts.retval, XDP_TX, "ipv6 retval");
75 ASSERT_EQ(topts.data_size_out, expect_sz, "ipv6 size");
76
77 bpf_object__close(obj);
78 }
79
test_xdp_adjust_tail_grow2(void)80 static void test_xdp_adjust_tail_grow2(void)
81 {
82 const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
83 char buf[4096]; /* avoid segfault: large buf to hold grow results */
84 struct bpf_object *obj;
85 int err, cnt, i;
86 int max_grow, prog_fd;
87 /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) */
88 #if defined(__s390x__)
89 int tailroom = 512;
90 #elif defined(__powerpc__)
91 int tailroom = 384;
92 #else
93 int tailroom = 320;
94 #endif
95
96 LIBBPF_OPTS(bpf_test_run_opts, tattr,
97 .repeat = 1,
98 .data_in = &buf,
99 .data_out = &buf,
100 .data_size_in = 0, /* Per test */
101 .data_size_out = 0, /* Per test */
102 );
103
104 err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
105 if (!ASSERT_OK(err, "test_xdp_adjust_tail_grow"))
106 return;
107
108 /* Test case-64 */
109 memset(buf, 1, sizeof(buf));
110 tattr.data_size_in = 64; /* Determine test case via pkt size */
111 tattr.data_size_out = 128; /* Limit copy_size */
112 /* Kernel side alloc packet memory area that is zero init */
113 err = bpf_prog_test_run_opts(prog_fd, &tattr);
114
115 ASSERT_EQ(errno, ENOSPC, "case-64 errno"); /* Due limit copy_size in bpf_test_finish */
116 ASSERT_EQ(tattr.retval, XDP_TX, "case-64 retval");
117 ASSERT_EQ(tattr.data_size_out, 192, "case-64 data_size_out"); /* Expected grow size */
118
119 /* Extra checks for data contents */
120 ASSERT_EQ(buf[0], 1, "case-64-data buf[0]"); /* 0-63 memset to 1 */
121 ASSERT_EQ(buf[63], 1, "case-64-data buf[63]");
122 ASSERT_EQ(buf[64], 0, "case-64-data buf[64]"); /* 64-127 memset to 0 */
123 ASSERT_EQ(buf[127], 0, "case-64-data buf[127]");
124 ASSERT_EQ(buf[128], 1, "case-64-data buf[128]"); /* 128-191 memset to 1 */
125 ASSERT_EQ(buf[191], 1, "case-64-data buf[191]");
126
127 /* Test case-128 */
128 memset(buf, 2, sizeof(buf));
129 tattr.data_size_in = 128; /* Determine test case via pkt size */
130 tattr.data_size_out = sizeof(buf); /* Copy everything */
131 err = bpf_prog_test_run_opts(prog_fd, &tattr);
132
133 max_grow = 4096 - XDP_PACKET_HEADROOM - tailroom; /* 3520 */
134 ASSERT_OK(err, "case-128");
135 ASSERT_EQ(tattr.retval, XDP_TX, "case-128 retval");
136 ASSERT_EQ(tattr.data_size_out, max_grow, "case-128 data_size_out"); /* Expect max grow */
137
138 /* Extra checks for data content: Count grow size, will contain zeros */
139 for (i = 0, cnt = 0; i < sizeof(buf); i++) {
140 if (buf[i] == 0)
141 cnt++;
142 }
143 ASSERT_EQ(cnt, max_grow - tattr.data_size_in, "case-128-data cnt"); /* Grow increase */
144 ASSERT_EQ(tattr.data_size_out, max_grow, "case-128-data data_size_out"); /* Total grow */
145
146 bpf_object__close(obj);
147 }
148
test_xdp_adjust_frags_tail_shrink(void)149 static void test_xdp_adjust_frags_tail_shrink(void)
150 {
151 const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
152 __u32 exp_size;
153 struct bpf_program *prog;
154 struct bpf_object *obj;
155 int err, prog_fd;
156 __u8 *buf;
157 LIBBPF_OPTS(bpf_test_run_opts, topts);
158
159 /* For the individual test cases, the first byte in the packet
160 * indicates which test will be run.
161 */
162 obj = bpf_object__open(file);
163 if (libbpf_get_error(obj))
164 return;
165
166 prog = bpf_object__next_program(obj, NULL);
167 if (bpf_object__load(obj))
168 return;
169
170 prog_fd = bpf_program__fd(prog);
171
172 buf = malloc(9000);
173 if (!ASSERT_OK_PTR(buf, "alloc buf 9Kb"))
174 goto out;
175
176 memset(buf, 0, 9000);
177
178 /* Test case removing 10 bytes from last frag, NOT freeing it */
179 exp_size = 8990; /* 9000 - 10 */
180 topts.data_in = buf;
181 topts.data_out = buf;
182 topts.data_size_in = 9000;
183 topts.data_size_out = 9000;
184 err = bpf_prog_test_run_opts(prog_fd, &topts);
185
186 ASSERT_OK(err, "9Kb-10b");
187 ASSERT_EQ(topts.retval, XDP_TX, "9Kb-10b retval");
188 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-10b size");
189
190 /* Test case removing one of two pages, assuming 4K pages */
191 buf[0] = 1;
192 exp_size = 4900; /* 9000 - 4100 */
193
194 topts.data_size_out = 9000; /* reset from previous invocation */
195 err = bpf_prog_test_run_opts(prog_fd, &topts);
196
197 ASSERT_OK(err, "9Kb-4Kb");
198 ASSERT_EQ(topts.retval, XDP_TX, "9Kb-4Kb retval");
199 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-4Kb size");
200
201 /* Test case removing two pages resulting in a linear xdp_buff */
202 buf[0] = 2;
203 exp_size = 800; /* 9000 - 8200 */
204 topts.data_size_out = 9000; /* reset from previous invocation */
205 err = bpf_prog_test_run_opts(prog_fd, &topts);
206
207 ASSERT_OK(err, "9Kb-9Kb");
208 ASSERT_EQ(topts.retval, XDP_TX, "9Kb-9Kb retval");
209 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-9Kb size");
210
211 free(buf);
212 out:
213 bpf_object__close(obj);
214 }
215
test_xdp_adjust_frags_tail_grow_4k(void)216 static void test_xdp_adjust_frags_tail_grow_4k(void)
217 {
218 const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
219 __u32 exp_size;
220 struct bpf_program *prog;
221 struct bpf_object *obj;
222 int err, i, prog_fd;
223 __u8 *buf;
224 LIBBPF_OPTS(bpf_test_run_opts, topts);
225
226 obj = bpf_object__open(file);
227 if (libbpf_get_error(obj))
228 return;
229
230 prog = bpf_object__next_program(obj, NULL);
231 if (bpf_object__load(obj))
232 goto out;
233
234 prog_fd = bpf_program__fd(prog);
235
236 buf = malloc(16384);
237 if (!ASSERT_OK_PTR(buf, "alloc buf 16Kb"))
238 goto out;
239
240 /* Test case add 10 bytes to last frag */
241 memset(buf, 1, 16384);
242 exp_size = 9000 + 10;
243
244 topts.data_in = buf;
245 topts.data_out = buf;
246 topts.data_size_in = 9000;
247 topts.data_size_out = 16384;
248 err = bpf_prog_test_run_opts(prog_fd, &topts);
249
250 ASSERT_OK(err, "9Kb+10b");
251 ASSERT_EQ(topts.retval, XDP_TX, "9Kb+10b retval");
252 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb+10b size");
253
254 for (i = 0; i < 9000; i++) {
255 if (buf[i] != 1)
256 ASSERT_EQ(buf[i], 1, "9Kb+10b-old");
257 }
258
259 for (i = 9000; i < 9010; i++) {
260 if (buf[i] != 0)
261 ASSERT_EQ(buf[i], 0, "9Kb+10b-new");
262 }
263
264 for (i = 9010; i < 16384; i++) {
265 if (buf[i] != 1)
266 ASSERT_EQ(buf[i], 1, "9Kb+10b-untouched");
267 }
268
269 /* Test a too large grow */
270 memset(buf, 1, 16384);
271 exp_size = 9001;
272
273 topts.data_in = topts.data_out = buf;
274 topts.data_size_in = 9001;
275 topts.data_size_out = 16384;
276 err = bpf_prog_test_run_opts(prog_fd, &topts);
277
278 ASSERT_OK(err, "9Kb+10b");
279 ASSERT_EQ(topts.retval, XDP_DROP, "9Kb+10b retval");
280 ASSERT_EQ(topts.data_size_out, exp_size, "9Kb+10b size");
281
282 free(buf);
283 out:
284 bpf_object__close(obj);
285 }
286
test_xdp_adjust_frags_tail_grow_64k(void)287 static void test_xdp_adjust_frags_tail_grow_64k(void)
288 {
289 const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
290 __u32 exp_size;
291 struct bpf_program *prog;
292 struct bpf_object *obj;
293 int err, i, prog_fd;
294 __u8 *buf;
295 LIBBPF_OPTS(bpf_test_run_opts, topts);
296
297 obj = bpf_object__open(file);
298 if (libbpf_get_error(obj))
299 return;
300
301 prog = bpf_object__next_program(obj, NULL);
302 if (bpf_object__load(obj))
303 goto out;
304
305 prog_fd = bpf_program__fd(prog);
306
307 buf = malloc(262144);
308 if (!ASSERT_OK_PTR(buf, "alloc buf 256Kb"))
309 goto out;
310
311 /* Test case add 10 bytes to last frag */
312 memset(buf, 1, 262144);
313 exp_size = 90000 + 10;
314
315 topts.data_in = buf;
316 topts.data_out = buf;
317 topts.data_size_in = 90000;
318 topts.data_size_out = 262144;
319 err = bpf_prog_test_run_opts(prog_fd, &topts);
320
321 ASSERT_OK(err, "90Kb+10b");
322 ASSERT_EQ(topts.retval, XDP_TX, "90Kb+10b retval");
323 ASSERT_EQ(topts.data_size_out, exp_size, "90Kb+10b size");
324
325 for (i = 0; i < 90000; i++) {
326 if (buf[i] != 1)
327 ASSERT_EQ(buf[i], 1, "90Kb+10b-old");
328 }
329
330 for (i = 90000; i < 90010; i++) {
331 if (buf[i] != 0)
332 ASSERT_EQ(buf[i], 0, "90Kb+10b-new");
333 }
334
335 for (i = 90010; i < 262144; i++) {
336 if (buf[i] != 1)
337 ASSERT_EQ(buf[i], 1, "90Kb+10b-untouched");
338 }
339
340 /* Test a too large grow */
341 memset(buf, 1, 262144);
342 exp_size = 90001;
343
344 topts.data_in = topts.data_out = buf;
345 topts.data_size_in = 90001;
346 topts.data_size_out = 262144;
347 err = bpf_prog_test_run_opts(prog_fd, &topts);
348
349 ASSERT_OK(err, "90Kb+10b");
350 ASSERT_EQ(topts.retval, XDP_DROP, "90Kb+10b retval");
351 ASSERT_EQ(topts.data_size_out, exp_size, "90Kb+10b size");
352
353 free(buf);
354 out:
355 bpf_object__close(obj);
356 }
357
test_xdp_adjust_tail(void)358 void test_xdp_adjust_tail(void)
359 {
360 int page_size = getpagesize();
361
362 if (test__start_subtest("xdp_adjust_tail_shrink"))
363 test_xdp_adjust_tail_shrink();
364 if (test__start_subtest("xdp_adjust_tail_grow"))
365 test_xdp_adjust_tail_grow(page_size == 65536);
366 if (test__start_subtest("xdp_adjust_tail_grow2"))
367 test_xdp_adjust_tail_grow2();
368 if (test__start_subtest("xdp_adjust_frags_tail_shrink"))
369 test_xdp_adjust_frags_tail_shrink();
370 if (test__start_subtest("xdp_adjust_frags_tail_grow")) {
371 if (page_size == 65536)
372 test_xdp_adjust_frags_tail_grow_64k();
373 else
374 test_xdp_adjust_frags_tail_grow_4k();
375 }
376 }
377