1 // SPDX-License-Identifier: GPL-2.0 2 #include <test_progs.h> 3 #include "bpf/libbpf_internal.h" 4 #include "test_global_percpu_data.skel.h" 5 #include "test_global_percpu_data.lskel.h" 6 7 void test_global_data_init(void) 8 { 9 const char *file = "./test_global_data.bpf.o"; 10 int err = -ENOMEM, map_fd, zero = 0; 11 __u8 *buff = NULL, *newval = NULL; 12 struct bpf_object *obj; 13 struct bpf_map *map; 14 __u32 duration = 0; 15 size_t sz; 16 17 obj = bpf_object__open_file(file, NULL); 18 err = libbpf_get_error(obj); 19 if (CHECK_FAIL(err)) 20 return; 21 22 map = bpf_object__find_map_by_name(obj, ".rodata"); 23 if (CHECK_FAIL(!map || !bpf_map__is_internal(map))) 24 goto out; 25 26 sz = bpf_map__value_size(map); 27 newval = malloc(sz); 28 if (CHECK_FAIL(!newval)) 29 goto out; 30 31 memset(newval, 0, sz); 32 /* wrong size, should fail */ 33 err = bpf_map__set_initial_value(map, newval, sz - 1); 34 if (CHECK(!err, "reject set initial value wrong size", "err %d\n", err)) 35 goto out; 36 37 err = bpf_map__set_initial_value(map, newval, sz); 38 if (CHECK(err, "set initial value", "err %d\n", err)) 39 goto out; 40 41 err = bpf_object__load(obj); 42 if (CHECK_FAIL(err)) 43 goto out; 44 45 map_fd = bpf_map__fd(map); 46 if (CHECK_FAIL(map_fd < 0)) 47 goto out; 48 49 buff = malloc(sz); 50 if (buff) 51 err = bpf_map_lookup_elem(map_fd, &zero, buff); 52 if (CHECK(!buff || err || memcmp(buff, newval, sz), 53 "compare .rodata map data override", 54 "err %d errno %d\n", err, errno)) 55 goto out; 56 57 memset(newval, 1, sz); 58 /* object loaded - should fail */ 59 err = bpf_map__set_initial_value(map, newval, sz); 60 CHECK(!err, "reject set initial value after load", "err %d\n", err); 61 out: 62 free(buff); 63 free(newval); 64 bpf_object__close(obj); 65 } 66 67 static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_fd, int *runp) 68 { 69 struct test_global_percpu_data__percpu *data = NULL; 70 int i, err, key = 0, num_online, run = 0; 71 __u64 args[2] = {0x1234ULL, 0x5678ULL}; 72 size_t data_sz; 73 bool *online; 74 LIBBPF_OPTS(bpf_test_run_opts, topts, 75 .ctx_in = args, 76 .ctx_size_in = sizeof(args), 77 .flags = BPF_F_TEST_RUN_ON_CPU, 78 ); 79 80 err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online); 81 if (!ASSERT_OK(err, "parse_cpu_mask_file")) 82 return; 83 84 data_sz = map ? bpf_map__value_size(map) : sizeof(*data); 85 data = calloc(1, data_sz); 86 if (!ASSERT_OK_PTR(data, "calloc percpu data")) 87 goto out; 88 89 /* run on every online-CPU */ 90 for (i = 0; i < num_online; i++) { 91 __u64 flags; 92 93 if (!online[i]) 94 continue; 95 96 topts.cpu = i; 97 topts.retval = -1; 98 err = bpf_prog_test_run_opts(prog_fd, &topts); 99 ASSERT_OK(err, "bpf_prog_test_run_opts"); 100 ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval"); 101 102 memset(data, 0, data_sz); 103 flags = ((__u64) i << 32) | BPF_F_CPU; 104 if (map) 105 err = bpf_map__lookup_elem(map, &key, sizeof(key), data, data_sz, flags); 106 else 107 err = bpf_map_lookup_elem_flags(map_fd, &key, data, flags); 108 if (!ASSERT_OK(err, "lookup_elem on cpu")) 109 break; 110 111 ASSERT_EQ(*runp, ++run, "run"); 112 ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); 113 ASSERT_EQ(data->data, 1, "data"); 114 ASSERT_TRUE(data->set, "set"); 115 ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); 116 ASSERT_EQ(data->struct_data.i, 1, "struct_data.i"); 117 ASSERT_TRUE(data->struct_data.set, "struct_data.set"); 118 ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]"); 119 } 120 121 out: 122 free(data); 123 free(online); 124 } 125 126 static void test_global_percpu_data_init(void) 127 { 128 struct test_global_percpu_data__percpu init_value = {}; 129 struct test_global_percpu_data__percpu *init_data; 130 const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); 131 struct test_global_percpu_data *skel = NULL; 132 size_t init_data_sz; 133 struct bpf_map *map; 134 int prog_fd, err; 135 136 skel = test_global_percpu_data__open(); 137 if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) 138 goto out; 139 if (!ASSERT_OK_PTR(skel->percpu, "skel->percpu")) 140 goto out; 141 if (!ASSERT_OK_PTR(skel->data_percpu, "skel->data_percpu")) 142 goto out; 143 if (!ASSERT_OK_PTR(skel->percpu_data, "skel->percpu_data")) 144 goto out; 145 if (!ASSERT_OK_PTR(skel->percpu_looooooooong, "skel->percpu_looooooooong")) 146 goto out; 147 148 ASSERT_STREQ(bpf_map__name(skel->maps.percpu_data), ".percpu.data", 149 ".percpu.data map name"); 150 ASSERT_STREQ(bpf_map__name(skel->maps.data_percpu), ".data.percpu", 151 ".data.percpu map name"); 152 ASSERT_STREQ(bpf_map__name(skel->maps.percpu_looooooooong), ".percpu.looooooooong", 153 "long map name"); 154 ASSERT_STREQ(bpf_map__name(skel->maps.percpu), ".percpu", "map name"); 155 ASSERT_EQ(skel->percpu->data, -1, "skel->percpu->data"); 156 ASSERT_FALSE(skel->percpu->set, "skel->percpu->set"); 157 ASSERT_EQ(skel->percpu->nums[6], 0, "skel->percpu->nums[6]"); 158 ASSERT_EQ(skel->percpu->struct_data.i, -1, "struct_data.i"); 159 ASSERT_FALSE(skel->percpu->struct_data.set, "struct_data.set"); 160 ASSERT_EQ(skel->percpu->struct_data.nums[6], 0, "struct_data.nums[6]"); 161 162 map = skel->maps.percpu; 163 if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) 164 goto out; 165 166 init_value.data = 2; 167 init_value.nums[6] = -1; 168 init_value.struct_data.i = 2; 169 init_value.struct_data.nums[6] = -1; 170 err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); 171 if (!ASSERT_OK(err, "bpf_map__set_initial_value")) 172 goto out; 173 174 init_data = bpf_map__initial_value(map, &init_data_sz); 175 if (!ASSERT_OK_PTR(init_data, "bpf_map__initial_value")) 176 goto out; 177 178 ASSERT_EQ(init_data->data, init_value.data, "init_value data"); 179 ASSERT_EQ(init_data->set, init_value.set, "init_value set"); 180 ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); 181 ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], 182 "init_value struct_data.nums[6]"); 183 ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); 184 ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); 185 ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); 186 ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); 187 ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, 188 "skel->percpu->struct_data.i"); 189 ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], 190 "skel->percpu->struct_data.nums[6]"); 191 192 ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); 193 err = bpf_map__set_value_size(map, desired_sz); 194 if (!ASSERT_OK(err, "bpf_map__set_value_size")) 195 goto out; 196 if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) 197 goto out; 198 if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) 199 goto out; 200 201 init_data = bpf_map__initial_value(map, &init_data_sz); 202 if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) 203 goto out; 204 if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) 205 goto out; 206 if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) 207 goto out; 208 209 err = test_global_percpu_data__load(skel); 210 if (!ASSERT_OK(err, "test_global_percpu_data__load")) 211 goto out; 212 213 ASSERT_OK_PTR(skel->percpu, "skel->percpu"); 214 215 prog_fd = bpf_program__fd(skel->progs.update_percpu_data); 216 test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); 217 218 out: 219 test_global_percpu_data__destroy(skel); 220 } 221 222 static void test_global_percpu_data_lskel(void) 223 { 224 struct test_global_percpu_data_lskel *lskel = NULL; 225 int prog_fd, map_fd; 226 227 lskel = test_global_percpu_data_lskel__open_and_load(); 228 if (!ASSERT_OK_PTR(lskel, "test_global_percpu_data_lskel__open_and_load")) 229 goto out; 230 231 map_fd = lskel->maps.percpu.map_fd; 232 prog_fd = lskel->progs.update_percpu_data.prog_fd; 233 test_percpu_data_on_cpus(NULL, map_fd, prog_fd, &lskel->bss->run); 234 235 out: 236 test_global_percpu_data_lskel__destroy(lskel); 237 } 238 239 static int create_rdonly_percpu_array(void) 240 { 241 LIBBPF_OPTS(bpf_map_create_opts, map_opts, 242 .map_flags = BPF_F_RDONLY_PROG, 243 ); 244 int key = 0, map_fd, err; 245 __u64 value = 0; 246 247 map_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "percpu_ro_map", sizeof(int), 248 sizeof(__u64), 1, &map_opts); 249 if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) 250 return -1; 251 252 err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); 253 if (!ASSERT_OK(err, "bpf_map_update_elem")) 254 goto out; 255 256 err = bpf_map_freeze(map_fd); 257 if (!ASSERT_OK(err, "bpf_map_freeze")) 258 goto out; 259 260 return map_fd; 261 262 out: 263 close(map_fd); 264 return -1; 265 } 266 267 static void test_global_percpu_data_rdonly_direct_read(void) 268 { 269 /* 270 * Raw instructions with manually prepared rdonly percpu_array map 271 * for testing direct-read global percpu data, because libbpf 272 * doesn't have rdonly internal percpu_array map support for 273 * global percpu data. 274 */ 275 struct bpf_insn insns[] = { 276 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), 277 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), 278 BPF_EXIT_INSN(), 279 }; 280 int map_fd, prog_fd; 281 282 map_fd = create_rdonly_percpu_array(); 283 if (map_fd < 0) 284 return; 285 286 insns[0].imm = map_fd; 287 prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, 288 ARRAY_SIZE(insns), NULL); 289 if (ASSERT_GE(prog_fd, 0, "bpf_prog_load")) 290 close(prog_fd); 291 close(map_fd); 292 } 293 294 static void test_global_percpu_data_rdonly_direct_write(void) 295 { 296 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); 297 /* See the comment in test_global_percpu_data_rdonly_direct_read() */ 298 struct bpf_insn insns[] = { 299 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), 300 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), 301 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0), 302 BPF_EXIT_INSN(), 303 }; 304 char log_buf[256] = {}; 305 int map_fd, prog_fd; 306 307 prog_opts.log_buf = log_buf; 308 prog_opts.log_size = sizeof(log_buf); 309 prog_opts.log_level = 1; 310 311 map_fd = create_rdonly_percpu_array(); 312 if (map_fd < 0) 313 return; 314 315 insns[0].imm = map_fd; 316 prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, 317 ARRAY_SIZE(insns), &prog_opts); 318 if (!ASSERT_LT(prog_fd, 0, "bpf_prog_load")) 319 close(prog_fd); 320 else 321 ASSERT_HAS_SUBSTR(log_buf, "write into map forbidden", "verifier log"); 322 close(map_fd); 323 } 324 325 static void test_global_percpu_data_verifier_log(void) 326 { 327 RUN_TESTS(test_global_percpu_data); 328 } 329 330 static void test_global_percpu_data_iter(void) 331 { 332 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 333 struct test_global_percpu_data *skel; 334 union bpf_iter_link_info linfo = {}; 335 struct bpf_link *link = NULL; 336 int fd, num_cpus, len, err; 337 char buf[16]; 338 339 num_cpus = libbpf_num_possible_cpus(); 340 if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) 341 return; 342 343 skel = test_global_percpu_data__open(); 344 if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) 345 return; 346 347 skel->rodata->num_cpus = num_cpus; 348 skel->rodata->num_off = offsetof(struct test_global_percpu_data__percpu, 349 struct_data.nums[6]); 350 skel->rodata->elem_sz = roundup(sizeof(struct test_global_percpu_data__percpu), 8); 351 skel->percpu->struct_data.nums[6] = 0xc0de; 352 353 err = test_global_percpu_data__load(skel); 354 if (!ASSERT_OK(err, "test_global_percpu_data__load")) 355 goto out; 356 357 linfo.map.map_fd = bpf_map__fd(skel->maps.percpu); 358 opts.link_info = &linfo; 359 opts.link_info_len = sizeof(linfo); 360 link = bpf_program__attach_iter(skel->progs.dump_percpu_data, &opts); 361 if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter")) 362 goto out; 363 364 fd = bpf_iter_create(bpf_link__fd(link)); 365 if (!ASSERT_GE(fd, 0, "bpf_iter_create")) 366 goto out; 367 368 while ((len = read(fd, buf, sizeof(buf))) > 0) 369 do { } while (0); 370 ASSERT_EQ(len, 0, "read iter"); 371 ASSERT_TRUE(skel->bss->run_iter, "run_iter"); 372 ASSERT_EQ(skel->bss->sum, 0xc0de * num_cpus, "sum"); 373 374 close(fd); 375 out: 376 bpf_link__destroy(link); 377 test_global_percpu_data__destroy(skel); 378 } 379 380 void test_global_percpu_data(void) 381 { 382 if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { 383 test__skip(); 384 return; 385 } 386 387 if (test__start_subtest("init")) 388 test_global_percpu_data_init(); 389 if (test__start_subtest("lskel")) 390 test_global_percpu_data_lskel(); 391 if (test__start_subtest("rdonly_direct_read")) 392 test_global_percpu_data_rdonly_direct_read(); 393 if (test__start_subtest("rdonly_direct_write")) 394 test_global_percpu_data_rdonly_direct_write(); 395 test_global_percpu_data_verifier_log(); 396 if (test__start_subtest("iter")) 397 test_global_percpu_data_iter(); 398 } 399