1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include "bpf/libbpf_internal.h"
4 #include "test_global_percpu_data.skel.h"
5 #include "test_global_percpu_data.lskel.h"
6
test_global_data_init(void)7 void test_global_data_init(void)
8 {
9 const char *file = "./test_global_data.bpf.o";
10 int err = -ENOMEM, map_fd, zero = 0;
11 __u8 *buff = NULL, *newval = NULL;
12 struct bpf_object *obj;
13 struct bpf_map *map;
14 __u32 duration = 0;
15 size_t sz;
16
17 obj = bpf_object__open_file(file, NULL);
18 err = libbpf_get_error(obj);
19 if (CHECK_FAIL(err))
20 return;
21
22 map = bpf_object__find_map_by_name(obj, ".rodata");
23 if (CHECK_FAIL(!map || !bpf_map__is_internal(map)))
24 goto out;
25
26 sz = bpf_map__value_size(map);
27 newval = malloc(sz);
28 if (CHECK_FAIL(!newval))
29 goto out;
30
31 memset(newval, 0, sz);
32 /* wrong size, should fail */
33 err = bpf_map__set_initial_value(map, newval, sz - 1);
34 if (CHECK(!err, "reject set initial value wrong size", "err %d\n", err))
35 goto out;
36
37 err = bpf_map__set_initial_value(map, newval, sz);
38 if (CHECK(err, "set initial value", "err %d\n", err))
39 goto out;
40
41 err = bpf_object__load(obj);
42 if (CHECK_FAIL(err))
43 goto out;
44
45 map_fd = bpf_map__fd(map);
46 if (CHECK_FAIL(map_fd < 0))
47 goto out;
48
49 buff = malloc(sz);
50 if (buff)
51 err = bpf_map_lookup_elem(map_fd, &zero, buff);
52 if (CHECK(!buff || err || memcmp(buff, newval, sz),
53 "compare .rodata map data override",
54 "err %d errno %d\n", err, errno))
55 goto out;
56
57 memset(newval, 1, sz);
58 /* object loaded - should fail */
59 err = bpf_map__set_initial_value(map, newval, sz);
60 CHECK(!err, "reject set initial value after load", "err %d\n", err);
61 out:
62 free(buff);
63 free(newval);
64 bpf_object__close(obj);
65 }
66
test_percpu_data_on_cpus(struct bpf_map * map,int map_fd,int prog_fd,int * runp)67 static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_fd, int *runp)
68 {
69 struct test_global_percpu_data__percpu *data = NULL;
70 int i, err, key = 0, num_online, run = 0;
71 __u64 args[2] = {0x1234ULL, 0x5678ULL};
72 size_t data_sz;
73 bool *online;
74 LIBBPF_OPTS(bpf_test_run_opts, topts,
75 .ctx_in = args,
76 .ctx_size_in = sizeof(args),
77 .flags = BPF_F_TEST_RUN_ON_CPU,
78 );
79
80 err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online);
81 if (!ASSERT_OK(err, "parse_cpu_mask_file"))
82 return;
83
84 data_sz = map ? bpf_map__value_size(map) : sizeof(*data);
85 data = calloc(1, data_sz);
86 if (!ASSERT_OK_PTR(data, "calloc percpu data"))
87 goto out;
88
89 /* run on every online-CPU */
90 for (i = 0; i < num_online; i++) {
91 __u64 flags;
92
93 if (!online[i])
94 continue;
95
96 topts.cpu = i;
97 topts.retval = -1;
98 err = bpf_prog_test_run_opts(prog_fd, &topts);
99 ASSERT_OK(err, "bpf_prog_test_run_opts");
100 ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval");
101
102 memset(data, 0, data_sz);
103 flags = ((__u64) i << 32) | BPF_F_CPU;
104 if (map)
105 err = bpf_map__lookup_elem(map, &key, sizeof(key), data, data_sz, flags);
106 else
107 err = bpf_map_lookup_elem_flags(map_fd, &key, data, flags);
108 if (!ASSERT_OK(err, "lookup_elem on cpu"))
109 break;
110
111 ASSERT_EQ(*runp, ++run, "run");
112 ASSERT_EQ(data->cpu_id[0], i, "cpu_id");
113 ASSERT_EQ(data->data, 1, "data");
114 ASSERT_TRUE(data->set, "set");
115 ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]");
116 ASSERT_EQ(data->struct_data.i, 1, "struct_data.i");
117 ASSERT_TRUE(data->struct_data.set, "struct_data.set");
118 ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]");
119 }
120
121 out:
122 free(data);
123 free(online);
124 }
125
test_global_percpu_data_init(void)126 static void test_global_percpu_data_init(void)
127 {
128 struct test_global_percpu_data__percpu init_value = {};
129 struct test_global_percpu_data__percpu *init_data;
130 const __u32 desired_sz = sysconf(_SC_PAGE_SIZE);
131 struct test_global_percpu_data *skel = NULL;
132 size_t init_data_sz;
133 struct bpf_map *map;
134 int prog_fd, err;
135
136 skel = test_global_percpu_data__open();
137 if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open"))
138 goto out;
139 if (!ASSERT_OK_PTR(skel->percpu, "skel->percpu"))
140 goto out;
141 if (!ASSERT_OK_PTR(skel->data_percpu, "skel->data_percpu"))
142 goto out;
143 if (!ASSERT_OK_PTR(skel->percpu_data, "skel->percpu_data"))
144 goto out;
145 if (!ASSERT_OK_PTR(skel->percpu_looooooooong, "skel->percpu_looooooooong"))
146 goto out;
147
148 ASSERT_STREQ(bpf_map__name(skel->maps.percpu_data), ".percpu.data",
149 ".percpu.data map name");
150 ASSERT_STREQ(bpf_map__name(skel->maps.data_percpu), ".data.percpu",
151 ".data.percpu map name");
152 ASSERT_STREQ(bpf_map__name(skel->maps.percpu_looooooooong), ".percpu.looooooooong",
153 "long map name");
154 ASSERT_STREQ(bpf_map__name(skel->maps.percpu), ".percpu", "map name");
155 ASSERT_EQ(skel->percpu->data, -1, "skel->percpu->data");
156 ASSERT_FALSE(skel->percpu->set, "skel->percpu->set");
157 ASSERT_EQ(skel->percpu->nums[6], 0, "skel->percpu->nums[6]");
158 ASSERT_EQ(skel->percpu->struct_data.i, -1, "struct_data.i");
159 ASSERT_FALSE(skel->percpu->struct_data.set, "struct_data.set");
160 ASSERT_EQ(skel->percpu->struct_data.nums[6], 0, "struct_data.nums[6]");
161
162 map = skel->maps.percpu;
163 if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type"))
164 goto out;
165
166 init_value.data = 2;
167 init_value.nums[6] = -1;
168 init_value.struct_data.i = 2;
169 init_value.struct_data.nums[6] = -1;
170 err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value));
171 if (!ASSERT_OK(err, "bpf_map__set_initial_value"))
172 goto out;
173
174 init_data = bpf_map__initial_value(map, &init_data_sz);
175 if (!ASSERT_OK_PTR(init_data, "bpf_map__initial_value"))
176 goto out;
177
178 ASSERT_EQ(init_data->data, init_value.data, "init_value data");
179 ASSERT_EQ(init_data->set, init_value.set, "init_value set");
180 ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i");
181 ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6],
182 "init_value struct_data.nums[6]");
183 ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size");
184 ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data");
185 ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data");
186 ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set");
187 ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i,
188 "skel->percpu->struct_data.i");
189 ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6],
190 "skel->percpu->struct_data.nums[6]");
191
192 ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz");
193 err = bpf_map__set_value_size(map, desired_sz);
194 if (!ASSERT_OK(err, "bpf_map__set_value_size"))
195 goto out;
196 if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size"))
197 goto out;
198 if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type"))
199 goto out;
200
201 init_data = bpf_map__initial_value(map, &init_data_sz);
202 if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value"))
203 goto out;
204 if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size"))
205 goto out;
206 if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data"))
207 goto out;
208
209 err = test_global_percpu_data__load(skel);
210 if (!ASSERT_OK(err, "test_global_percpu_data__load"))
211 goto out;
212
213 ASSERT_OK_PTR(skel->percpu, "skel->percpu");
214
215 prog_fd = bpf_program__fd(skel->progs.update_percpu_data);
216 test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run);
217
218 out:
219 test_global_percpu_data__destroy(skel);
220 }
221
test_global_percpu_data_lskel(void)222 static void test_global_percpu_data_lskel(void)
223 {
224 struct test_global_percpu_data_lskel *lskel = NULL;
225 int prog_fd, map_fd;
226
227 lskel = test_global_percpu_data_lskel__open_and_load();
228 if (!ASSERT_OK_PTR(lskel, "test_global_percpu_data_lskel__open_and_load"))
229 goto out;
230
231 map_fd = lskel->maps.percpu.map_fd;
232 prog_fd = lskel->progs.update_percpu_data.prog_fd;
233 test_percpu_data_on_cpus(NULL, map_fd, prog_fd, &lskel->bss->run);
234
235 out:
236 test_global_percpu_data_lskel__destroy(lskel);
237 }
238
create_rdonly_percpu_array(void)239 static int create_rdonly_percpu_array(void)
240 {
241 LIBBPF_OPTS(bpf_map_create_opts, map_opts,
242 .map_flags = BPF_F_RDONLY_PROG,
243 );
244 int key = 0, map_fd, err;
245 __u64 value = 0;
246
247 map_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "percpu_ro_map", sizeof(int),
248 sizeof(__u64), 1, &map_opts);
249 if (!ASSERT_GE(map_fd, 0, "bpf_map_create"))
250 return -1;
251
252 err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS);
253 if (!ASSERT_OK(err, "bpf_map_update_elem"))
254 goto out;
255
256 err = bpf_map_freeze(map_fd);
257 if (!ASSERT_OK(err, "bpf_map_freeze"))
258 goto out;
259
260 return map_fd;
261
262 out:
263 close(map_fd);
264 return -1;
265 }
266
test_global_percpu_data_rdonly_direct_read(void)267 static void test_global_percpu_data_rdonly_direct_read(void)
268 {
269 /*
270 * Raw instructions with manually prepared rdonly percpu_array map
271 * for testing direct-read global percpu data, because libbpf
272 * doesn't have rdonly internal percpu_array map support for
273 * global percpu data.
274 */
275 struct bpf_insn insns[] = {
276 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0),
277 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
278 BPF_EXIT_INSN(),
279 };
280 int map_fd, prog_fd;
281
282 map_fd = create_rdonly_percpu_array();
283 if (map_fd < 0)
284 return;
285
286 insns[0].imm = map_fd;
287 prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns,
288 ARRAY_SIZE(insns), NULL);
289 if (ASSERT_GE(prog_fd, 0, "bpf_prog_load"))
290 close(prog_fd);
291 close(map_fd);
292 }
293
test_global_percpu_data_rdonly_direct_write(void)294 static void test_global_percpu_data_rdonly_direct_write(void)
295 {
296 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts);
297 /* See the comment in test_global_percpu_data_rdonly_direct_read() */
298 struct bpf_insn insns[] = {
299 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0),
300 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
301 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0),
302 BPF_EXIT_INSN(),
303 };
304 char log_buf[256] = {};
305 int map_fd, prog_fd;
306
307 prog_opts.log_buf = log_buf;
308 prog_opts.log_size = sizeof(log_buf);
309 prog_opts.log_level = 1;
310
311 map_fd = create_rdonly_percpu_array();
312 if (map_fd < 0)
313 return;
314
315 insns[0].imm = map_fd;
316 prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns,
317 ARRAY_SIZE(insns), &prog_opts);
318 if (!ASSERT_LT(prog_fd, 0, "bpf_prog_load"))
319 close(prog_fd);
320 else
321 ASSERT_HAS_SUBSTR(log_buf, "write into map forbidden", "verifier log");
322 close(map_fd);
323 }
324
test_global_percpu_data_verifier_log(void)325 static void test_global_percpu_data_verifier_log(void)
326 {
327 RUN_TESTS(test_global_percpu_data);
328 }
329
test_global_percpu_data_iter(void)330 static void test_global_percpu_data_iter(void)
331 {
332 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
333 struct test_global_percpu_data *skel;
334 union bpf_iter_link_info linfo = {};
335 struct bpf_link *link = NULL;
336 int fd, num_cpus, len, err;
337 char buf[16];
338
339 num_cpus = libbpf_num_possible_cpus();
340 if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus"))
341 return;
342
343 skel = test_global_percpu_data__open();
344 if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open"))
345 return;
346
347 skel->rodata->num_cpus = num_cpus;
348 skel->rodata->num_off = offsetof(struct test_global_percpu_data__percpu,
349 struct_data.nums[6]);
350 skel->rodata->elem_sz = roundup(sizeof(struct test_global_percpu_data__percpu), 8);
351 skel->percpu->struct_data.nums[6] = 0xc0de;
352
353 err = test_global_percpu_data__load(skel);
354 if (!ASSERT_OK(err, "test_global_percpu_data__load"))
355 goto out;
356
357 linfo.map.map_fd = bpf_map__fd(skel->maps.percpu);
358 opts.link_info = &linfo;
359 opts.link_info_len = sizeof(linfo);
360 link = bpf_program__attach_iter(skel->progs.dump_percpu_data, &opts);
361 if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter"))
362 goto out;
363
364 fd = bpf_iter_create(bpf_link__fd(link));
365 if (!ASSERT_GE(fd, 0, "bpf_iter_create"))
366 goto out;
367
368 while ((len = read(fd, buf, sizeof(buf))) > 0)
369 do { } while (0);
370 ASSERT_EQ(len, 0, "read iter");
371 ASSERT_TRUE(skel->bss->run_iter, "run_iter");
372 ASSERT_EQ(skel->bss->sum, 0xc0de * num_cpus, "sum");
373
374 close(fd);
375 out:
376 bpf_link__destroy(link);
377 test_global_percpu_data__destroy(skel);
378 }
379
test_global_percpu_data(void)380 void test_global_percpu_data(void)
381 {
382 if (!feat_supported(NULL, FEAT_PERCPU_DATA)) {
383 test__skip();
384 return;
385 }
386
387 if (test__start_subtest("init"))
388 test_global_percpu_data_init();
389 if (test__start_subtest("lskel"))
390 test_global_percpu_data_lskel();
391 if (test__start_subtest("rdonly_direct_read"))
392 test_global_percpu_data_rdonly_direct_read();
393 if (test__start_subtest("rdonly_direct_write"))
394 test_global_percpu_data_rdonly_direct_write();
395 test_global_percpu_data_verifier_log();
396 if (test__start_subtest("iter"))
397 test_global_percpu_data_iter();
398 }
399