1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include <network_helpers.h>
4 #include <net/if.h>
5 #include <linux/if_ether.h>
6 #include <linux/if_packet.h>
7 #include <linux/if_link.h>
8 #include <linux/ipv6.h>
9 #include <linux/in6.h>
10 #include <netinet/udp.h>
11 #include <bpf/bpf_endian.h>
12 #include <uapi/linux/netdev.h>
13 #include "test_xdp_do_redirect.skel.h"
14 #include "xdp_dummy.skel.h"
15
16 struct udp_packet {
17 struct ethhdr eth;
18 struct ipv6hdr iph;
19 struct udphdr udp;
20 __u8 payload[64 - sizeof(struct udphdr)
21 - sizeof(struct ethhdr) - sizeof(struct ipv6hdr)];
22 } __packed;
23
24 static struct udp_packet pkt_udp = {
25 .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6),
26 .eth.h_dest = {0x00, 0x11, 0x22, 0x33, 0x44, 0x55},
27 .eth.h_source = {0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb},
28 .iph.version = 6,
29 .iph.nexthdr = IPPROTO_UDP,
30 .iph.payload_len = bpf_htons(sizeof(struct udp_packet)
31 - offsetof(struct udp_packet, udp)),
32 .iph.hop_limit = 2,
33 .iph.saddr.s6_addr16 = {bpf_htons(0xfc00), 0, 0, 0, 0, 0, 0, bpf_htons(1)},
34 .iph.daddr.s6_addr16 = {bpf_htons(0xfc00), 0, 0, 0, 0, 0, 0, bpf_htons(2)},
35 .udp.source = bpf_htons(1),
36 .udp.dest = bpf_htons(1),
37 .udp.len = bpf_htons(sizeof(struct udp_packet)
38 - offsetof(struct udp_packet, udp)),
39 .payload = {0x42}, /* receiver XDP program matches on this */
40 };
41
attach_tc_prog(struct bpf_tc_hook * hook,int fd)42 static int attach_tc_prog(struct bpf_tc_hook *hook, int fd)
43 {
44 DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts, .handle = 1, .priority = 1, .prog_fd = fd);
45 int ret;
46
47 ret = bpf_tc_hook_create(hook);
48 if (!ASSERT_OK(ret, "create tc hook"))
49 return ret;
50
51 ret = bpf_tc_attach(hook, &opts);
52 if (!ASSERT_OK(ret, "bpf_tc_attach")) {
53 bpf_tc_hook_destroy(hook);
54 return ret;
55 }
56
57 return 0;
58 }
59
60 /* The maximum permissible size is: PAGE_SIZE - sizeof(struct xdp_page_head) -
61 * SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) - XDP_PACKET_HEADROOM =
62 * 3408 bytes for 64-byte cacheline and 3216 for 256-byte one.
63 */
64 #if defined(__s390x__)
65 #define MAX_PKT_SIZE 3216
66 #else
67 #define MAX_PKT_SIZE 3408
68 #endif
69
70 #define PAGE_SIZE_4K 4096
71 #define PAGE_SIZE_64K 65536
72
test_max_pkt_size(int fd)73 static void test_max_pkt_size(int fd)
74 {
75 char data[PAGE_SIZE_64K + 1] = {};
76 int err;
77 DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts,
78 .data_in = &data,
79 .flags = BPF_F_TEST_XDP_LIVE_FRAMES,
80 .repeat = 1,
81 );
82
83 if (getpagesize() == PAGE_SIZE_64K)
84 opts.data_size_in = MAX_PKT_SIZE + PAGE_SIZE_64K - PAGE_SIZE_4K;
85 else
86 opts.data_size_in = MAX_PKT_SIZE;
87
88 err = bpf_prog_test_run_opts(fd, &opts);
89 ASSERT_OK(err, "prog_run_max_size");
90
91 opts.data_size_in += 1;
92 err = bpf_prog_test_run_opts(fd, &opts);
93 ASSERT_EQ(err, -EINVAL, "prog_run_too_big");
94 }
95
96 #define NUM_PKTS 10000
test_xdp_do_redirect(void)97 void test_xdp_do_redirect(void)
98 {
99 int err, xdp_prog_fd, tc_prog_fd, ifindex_src, ifindex_dst;
100 char data[sizeof(pkt_udp) + sizeof(__u64)];
101 struct test_xdp_do_redirect *skel = NULL;
102 struct nstoken *nstoken = NULL;
103 struct bpf_link *link;
104 LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);
105 struct xdp_md ctx_in = { .data = sizeof(__u64),
106 .data_end = sizeof(data) };
107 DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts,
108 .data_in = &data,
109 .data_size_in = sizeof(data),
110 .ctx_in = &ctx_in,
111 .ctx_size_in = sizeof(ctx_in),
112 .flags = BPF_F_TEST_XDP_LIVE_FRAMES,
113 .repeat = NUM_PKTS,
114 .batch_size = 64,
115 );
116 DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
117 .attach_point = BPF_TC_INGRESS);
118
119 memcpy(&data[sizeof(__u64)], &pkt_udp, sizeof(pkt_udp));
120 ((__u32 *)data)[0] = 0x42; /* metadata test value */
121 ((__u32 *)data)[1] = 0;
122
123 skel = test_xdp_do_redirect__open();
124 if (!ASSERT_OK_PTR(skel, "skel"))
125 return;
126
127 /* The XDP program we run with bpf_prog_run() will cycle through all
128 * three xmit (PASS/TX/REDIRECT) return codes starting from above, and
129 * ending up with PASS, so we should end up with two packets on the dst
130 * iface and NUM_PKTS-2 in the TC hook. We match the packets on the UDP
131 * payload.
132 */
133 SYS(out, "ip netns add testns");
134 nstoken = open_netns("testns");
135 if (!ASSERT_OK_PTR(nstoken, "setns"))
136 goto out;
137
138 SYS(out, "ip link add veth_src type veth peer name veth_dst");
139 SYS(out, "ip link set dev veth_src address 00:11:22:33:44:55");
140 SYS(out, "ip link set dev veth_dst address 66:77:88:99:aa:bb");
141 SYS(out, "ip link set dev veth_src up");
142 SYS(out, "ip link set dev veth_dst up");
143 SYS(out, "ip addr add dev veth_src fc00::1/64");
144 SYS(out, "ip addr add dev veth_dst fc00::2/64");
145 SYS(out, "ip neigh add fc00::2 dev veth_src lladdr 66:77:88:99:aa:bb");
146
147 /* We enable forwarding in the test namespace because that will cause
148 * the packets that go through the kernel stack (with XDP_PASS) to be
149 * forwarded back out the same interface (because of the packet dst
150 * combined with the interface addresses). When this happens, the
151 * regular forwarding path will end up going through the same
152 * veth_xdp_xmit() call as the XDP_REDIRECT code, which can cause a
153 * deadlock if it happens on the same CPU. There's a local_bh_disable()
154 * in the test_run code to prevent this, but an earlier version of the
155 * code didn't have this, so we keep the test behaviour to make sure the
156 * bug doesn't resurface.
157 */
158 SYS(out, "sysctl -qw net.ipv6.conf.all.forwarding=1");
159
160 ifindex_src = if_nametoindex("veth_src");
161 ifindex_dst = if_nametoindex("veth_dst");
162 if (!ASSERT_NEQ(ifindex_src, 0, "ifindex_src") ||
163 !ASSERT_NEQ(ifindex_dst, 0, "ifindex_dst"))
164 goto out;
165
166 /* Check xdp features supported by veth driver */
167 err = bpf_xdp_query(ifindex_src, XDP_FLAGS_DRV_MODE, &query_opts);
168 if (!ASSERT_OK(err, "veth_src bpf_xdp_query"))
169 goto out;
170
171 if (!ASSERT_EQ(query_opts.feature_flags,
172 NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
173 NETDEV_XDP_ACT_RX_SG,
174 "veth_src query_opts.feature_flags"))
175 goto out;
176
177 err = bpf_xdp_query(ifindex_dst, XDP_FLAGS_DRV_MODE, &query_opts);
178 if (!ASSERT_OK(err, "veth_dst bpf_xdp_query"))
179 goto out;
180
181 if (!ASSERT_EQ(query_opts.feature_flags,
182 NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
183 NETDEV_XDP_ACT_RX_SG,
184 "veth_dst query_opts.feature_flags"))
185 goto out;
186
187 /* Enable GRO */
188 SYS(out, "ethtool -K veth_src gro on");
189 SYS(out, "ethtool -K veth_dst gro on");
190
191 err = bpf_xdp_query(ifindex_src, XDP_FLAGS_DRV_MODE, &query_opts);
192 if (!ASSERT_OK(err, "veth_src bpf_xdp_query gro on"))
193 goto out;
194
195 if (!ASSERT_EQ(query_opts.feature_flags,
196 NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
197 NETDEV_XDP_ACT_NDO_XMIT | NETDEV_XDP_ACT_RX_SG |
198 NETDEV_XDP_ACT_NDO_XMIT_SG,
199 "veth_src query_opts.feature_flags gro on"))
200 goto out;
201
202 err = bpf_xdp_query(ifindex_dst, XDP_FLAGS_DRV_MODE, &query_opts);
203 if (!ASSERT_OK(err, "veth_dst bpf_xdp_query gro on"))
204 goto out;
205
206 if (!ASSERT_EQ(query_opts.feature_flags,
207 NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
208 NETDEV_XDP_ACT_NDO_XMIT | NETDEV_XDP_ACT_RX_SG |
209 NETDEV_XDP_ACT_NDO_XMIT_SG,
210 "veth_dst query_opts.feature_flags gro on"))
211 goto out;
212
213 memcpy(skel->rodata->expect_dst, &pkt_udp.eth.h_dest, ETH_ALEN);
214 skel->rodata->ifindex_out = ifindex_src; /* redirect back to the same iface */
215 skel->rodata->ifindex_in = ifindex_src;
216 ctx_in.ingress_ifindex = ifindex_src;
217 tc_hook.ifindex = ifindex_src;
218
219 if (!ASSERT_OK(test_xdp_do_redirect__load(skel), "load"))
220 goto out;
221
222 link = bpf_program__attach_xdp(skel->progs.xdp_count_pkts, ifindex_dst);
223 if (!ASSERT_OK_PTR(link, "prog_attach"))
224 goto out;
225 skel->links.xdp_count_pkts = link;
226
227 tc_prog_fd = bpf_program__fd(skel->progs.tc_count_pkts);
228 if (attach_tc_prog(&tc_hook, tc_prog_fd))
229 goto out;
230
231 xdp_prog_fd = bpf_program__fd(skel->progs.xdp_redirect);
232 err = bpf_prog_test_run_opts(xdp_prog_fd, &opts);
233 if (!ASSERT_OK(err, "prog_run"))
234 goto out_tc;
235
236 /* wait for the packets to be flushed */
237 kern_sync_rcu();
238
239 /* There will be one packet sent through XDP_REDIRECT and one through
240 * XDP_TX; these will show up on the XDP counting program, while the
241 * rest will be counted at the TC ingress hook (and the counting program
242 * resets the packet payload so they don't get counted twice even though
243 * they are re-xmited out the veth device
244 */
245 ASSERT_EQ(skel->bss->pkts_seen_xdp, 2, "pkt_count_xdp");
246 ASSERT_EQ(skel->bss->pkts_seen_zero, 2, "pkt_count_zero");
247 ASSERT_EQ(skel->bss->pkts_seen_tc, NUM_PKTS - 2, "pkt_count_tc");
248
249 test_max_pkt_size(bpf_program__fd(skel->progs.xdp_count_pkts));
250
251 out_tc:
252 bpf_tc_hook_destroy(&tc_hook);
253 out:
254 if (nstoken)
255 close_netns(nstoken);
256 SYS_NOFAIL("ip netns del testns");
257 test_xdp_do_redirect__destroy(skel);
258 }
259
260 #define NS_NB 3
261 #define NS0 "NS0"
262 #define NS1 "NS1"
263 #define NS2 "NS2"
264 #define IPV4_NETWORK "10.1.1"
265 #define VETH1_INDEX 111
266 #define VETH2_INDEX 222
267
268 struct test_data {
269 struct netns_obj *ns[NS_NB];
270 u32 xdp_flags;
271 };
272
cleanup(struct test_data * data)273 static void cleanup(struct test_data *data)
274 {
275 int i;
276
277 for (i = 0; i < NS_NB; i++)
278 netns_free(data->ns[i]);
279 }
280
281 /**
282 * ping_setup -
283 * Create two veth peers and forward packets in-between using XDP
284 *
285 * ------------ ------------
286 * | NS1 | | NS2 |
287 * | veth0 | | veth0 |
288 * | 10.1.1.1 | | 10.1.1.2 |
289 * -----|------ ------|-----
290 * | |
291 * | |
292 * -----|-----------------------|-------
293 * | veth1 veth2 |
294 * | (id:111) (id:222) |
295 * | | | |
296 * | ----- xdp forwarding ----- |
297 * | |
298 * | NS0 |
299 * -------------------------------------
300 */
ping_setup(struct test_data * data)301 static int ping_setup(struct test_data *data)
302 {
303 int i;
304
305 data->ns[0] = netns_new(NS0, false);
306 if (!ASSERT_OK_PTR(data->ns[0], "create ns"))
307 return -1;
308
309 for (i = 1; i < NS_NB; i++) {
310 char ns_name[4] = {};
311
312 snprintf(ns_name, 4, "NS%d", i);
313 data->ns[i] = netns_new(ns_name, false);
314 if (!ASSERT_OK_PTR(data->ns[i], "create ns"))
315 goto fail;
316
317 SYS(fail,
318 "ip -n %s link add veth%d index %d%d%d type veth peer name veth0 netns %s",
319 NS0, i, i, i, i, ns_name);
320 SYS(fail, "ip -n %s link set veth%d up", NS0, i);
321
322 SYS(fail, "ip -n %s addr add %s.%d/24 dev veth0", ns_name, IPV4_NETWORK, i);
323 SYS(fail, "ip -n %s link set veth0 up", ns_name);
324 }
325
326 return 0;
327
328 fail:
329 cleanup(data);
330 return -1;
331 }
332
ping_test(struct test_data * data)333 static void ping_test(struct test_data *data)
334 {
335 struct test_xdp_do_redirect *skel = NULL;
336 struct xdp_dummy *skel_dummy = NULL;
337 struct nstoken *nstoken = NULL;
338 int i, ret;
339
340 skel_dummy = xdp_dummy__open_and_load();
341 if (!ASSERT_OK_PTR(skel_dummy, "open and load xdp_dummy skeleton"))
342 goto close;
343
344 for (i = 1; i < NS_NB; i++) {
345 char ns_name[4] = {};
346
347 snprintf(ns_name, 4, "NS%d", i);
348 nstoken = open_netns(ns_name);
349 if (!ASSERT_OK_PTR(nstoken, "open ns"))
350 goto close;
351
352 ret = bpf_xdp_attach(if_nametoindex("veth0"),
353 bpf_program__fd(skel_dummy->progs.xdp_dummy_prog),
354 data->xdp_flags, NULL);
355 if (!ASSERT_GE(ret, 0, "bpf_xdp_attach dummy_prog"))
356 goto close;
357
358 close_netns(nstoken);
359 nstoken = NULL;
360 }
361
362 skel = test_xdp_do_redirect__open_and_load();
363 if (!ASSERT_OK_PTR(skel, "open and load skeleton"))
364 goto close;
365
366 nstoken = open_netns(NS0);
367 if (!ASSERT_OK_PTR(nstoken, "open NS0"))
368 goto close;
369
370 ret = bpf_xdp_attach(VETH2_INDEX,
371 bpf_program__fd(skel->progs.xdp_redirect_to_111),
372 data->xdp_flags, NULL);
373 if (!ASSERT_GE(ret, 0, "bpf_xdp_attach"))
374 goto close;
375
376 ret = bpf_xdp_attach(VETH1_INDEX,
377 bpf_program__fd(skel->progs.xdp_redirect_to_222),
378 data->xdp_flags, NULL);
379 if (!ASSERT_GE(ret, 0, "bpf_xdp_attach"))
380 goto close;
381
382 close_netns(nstoken);
383 nstoken = NULL;
384
385 nstoken = open_netns(NS1);
386 if (!ASSERT_OK_PTR(nstoken, "open NS1"))
387 goto close;
388
389 SYS(close, "ping -c 1 %s.2 > /dev/null", IPV4_NETWORK);
390
391 close:
392 close_netns(nstoken);
393 xdp_dummy__destroy(skel_dummy);
394 test_xdp_do_redirect__destroy(skel);
395 }
396
397
xdp_redirect_ping(u32 xdp_flags)398 static void xdp_redirect_ping(u32 xdp_flags)
399 {
400 struct test_data data = {};
401
402 if (ping_setup(&data) < 0)
403 return;
404
405 data.xdp_flags = xdp_flags;
406 ping_test(&data);
407 cleanup(&data);
408 }
409
test_xdp_index_redirect(void)410 void test_xdp_index_redirect(void)
411 {
412 if (test__start_subtest("noflag"))
413 xdp_redirect_ping(0);
414
415 if (test__start_subtest("drvflag"))
416 xdp_redirect_ping(XDP_FLAGS_DRV_MODE);
417
418 if (test__start_subtest("skbflag"))
419 xdp_redirect_ping(XDP_FLAGS_SKB_MODE);
420 }
421
422