1 // SPDX-License-Identifier: GPL-2.0 2 #define _GNU_SOURCE 3 #include <sched.h> 4 #include <test_progs.h> 5 #include "cgroup_helpers.h" 6 #include "percpu_alloc_array.skel.h" 7 #include "percpu_alloc_cgrp_local_storage.skel.h" 8 #include "percpu_alloc_fail.skel.h" 9 10 static void test_array(void) 11 { 12 struct percpu_alloc_array *skel; 13 int err, prog_fd; 14 LIBBPF_OPTS(bpf_test_run_opts, topts); 15 16 skel = percpu_alloc_array__open(); 17 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open")) 18 return; 19 20 bpf_program__set_autoload(skel->progs.test_array_map_1, true); 21 bpf_program__set_autoload(skel->progs.test_array_map_2, true); 22 bpf_program__set_autoload(skel->progs.test_array_map_3, true); 23 bpf_program__set_autoload(skel->progs.test_array_map_4, true); 24 25 skel->bss->my_pid = getpid(); 26 skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 27 28 err = percpu_alloc_array__load(skel); 29 if (!ASSERT_OK(err, "percpu_alloc_array__load")) 30 goto out; 31 32 err = percpu_alloc_array__attach(skel); 33 if (!ASSERT_OK(err, "percpu_alloc_array__attach")) 34 goto out; 35 36 prog_fd = bpf_program__fd(skel->progs.test_array_map_1); 37 err = bpf_prog_test_run_opts(prog_fd, &topts); 38 ASSERT_OK(err, "test_run array_map 1-4"); 39 ASSERT_EQ(topts.retval, 0, "test_run array_map 1-4"); 40 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 41 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 42 out: 43 percpu_alloc_array__destroy(skel); 44 } 45 46 static void test_array_sleepable(void) 47 { 48 struct percpu_alloc_array *skel; 49 int err, prog_fd; 50 LIBBPF_OPTS(bpf_test_run_opts, topts); 51 52 skel = percpu_alloc_array__open(); 53 if (!ASSERT_OK_PTR(skel, "percpu_alloc__open")) 54 return; 55 56 bpf_program__set_autoload(skel->progs.test_array_map_10, true); 57 58 skel->bss->my_pid = getpid(); 59 skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 60 61 err = percpu_alloc_array__load(skel); 62 if (!ASSERT_OK(err, "percpu_alloc_array__load")) 63 goto out; 64 65 err = percpu_alloc_array__attach(skel); 66 if (!ASSERT_OK(err, "percpu_alloc_array__attach")) 67 goto out; 68 69 prog_fd = bpf_program__fd(skel->progs.test_array_map_10); 70 err = bpf_prog_test_run_opts(prog_fd, &topts); 71 ASSERT_OK(err, "test_run array_map_10"); 72 ASSERT_EQ(topts.retval, 0, "test_run array_map_10"); 73 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 74 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 75 out: 76 percpu_alloc_array__destroy(skel); 77 } 78 79 static void test_cgrp_local_storage(void) 80 { 81 struct percpu_alloc_cgrp_local_storage *skel; 82 int err, cgroup_fd, prog_fd; 83 LIBBPF_OPTS(bpf_test_run_opts, topts); 84 85 cgroup_fd = test__join_cgroup("/percpu_alloc"); 86 if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /percpu_alloc")) 87 return; 88 89 skel = percpu_alloc_cgrp_local_storage__open(); 90 if (!ASSERT_OK_PTR(skel, "percpu_alloc_cgrp_local_storage__open")) 91 goto close_fd; 92 93 skel->bss->my_pid = getpid(); 94 skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 95 96 err = percpu_alloc_cgrp_local_storage__load(skel); 97 if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__load")) 98 goto destroy_skel; 99 100 err = percpu_alloc_cgrp_local_storage__attach(skel); 101 if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__attach")) 102 goto destroy_skel; 103 104 prog_fd = bpf_program__fd(skel->progs.test_cgrp_local_storage_1); 105 err = bpf_prog_test_run_opts(prog_fd, &topts); 106 ASSERT_OK(err, "test_run cgrp_local_storage 1-3"); 107 ASSERT_EQ(topts.retval, 0, "test_run cgrp_local_storage 1-3"); 108 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 109 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 110 111 destroy_skel: 112 percpu_alloc_cgrp_local_storage__destroy(skel); 113 close_fd: 114 close(cgroup_fd); 115 } 116 117 static void test_failure(void) { 118 RUN_TESTS(percpu_alloc_fail); 119 } 120 121 static void test_percpu_map_op_cpu_flag(struct bpf_map *map, void *keys, size_t key_sz, u32 entries, 122 int nr_cpus, bool test_batch) 123 { 124 size_t value_sz = sizeof(u32), value_sz_cpus, value_sz_total; 125 u32 *values = NULL, *values_percpu = NULL; 126 const u32 value = 0xDEADC0DE; 127 int i, j, cpu, map_fd, err; 128 u64 batch = 0, flags; 129 void *values_row; 130 u32 count, v; 131 LIBBPF_OPTS(bpf_map_batch_opts, batch_opts); 132 133 value_sz_cpus = value_sz * nr_cpus; 134 values = calloc(entries, value_sz_cpus); 135 if (!ASSERT_OK_PTR(values, "calloc values")) 136 return; 137 138 values_percpu = calloc(entries, roundup(value_sz, 8) * nr_cpus); 139 if (!ASSERT_OK_PTR(values_percpu, "calloc values_percpu")) { 140 free(values); 141 return; 142 } 143 144 value_sz_total = value_sz_cpus * entries; 145 memset(values, 0, value_sz_total); 146 147 map_fd = bpf_map__fd(map); 148 flags = BPF_F_CPU | BPF_F_ALL_CPUS; 149 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 150 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags cpu|all_cpus")) 151 goto out; 152 153 err = bpf_map_update_elem(map_fd, keys, values, flags); 154 if (!ASSERT_ERR(err, "bpf_map_update_elem cpu|all_cpus")) 155 goto out; 156 157 flags = BPF_F_ALL_CPUS; 158 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 159 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags all_cpus")) 160 goto out; 161 162 flags = BPF_F_LOCK | BPF_F_CPU; 163 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 164 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags BPF_F_LOCK")) 165 goto out; 166 167 flags = BPF_F_LOCK | BPF_F_ALL_CPUS; 168 err = bpf_map_update_elem(map_fd, keys, values, flags); 169 if (!ASSERT_ERR(err, "bpf_map_update_elem BPF_F_LOCK")) 170 goto out; 171 172 flags = (u64)nr_cpus << 32 | BPF_F_CPU; 173 err = bpf_map_update_elem(map_fd, keys, values, flags); 174 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_update_elem -ERANGE")) 175 goto out; 176 177 err = bpf_map__update_elem(map, keys, key_sz, values, value_sz, flags); 178 if (!ASSERT_EQ(err, -ERANGE, "bpf_map__update_elem -ERANGE")) 179 goto out; 180 181 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 182 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_lookup_elem_flags -ERANGE")) 183 goto out; 184 185 err = bpf_map__lookup_elem(map, keys, key_sz, values, value_sz, flags); 186 if (!ASSERT_EQ(err, -ERANGE, "bpf_map__lookup_elem -ERANGE")) 187 goto out; 188 189 for (cpu = 0; cpu < nr_cpus; cpu++) { 190 /* clear value on all cpus */ 191 values[0] = 0; 192 flags = BPF_F_ALL_CPUS; 193 for (i = 0; i < entries; i++) { 194 err = bpf_map__update_elem(map, keys + i * key_sz, key_sz, values, 195 value_sz, flags); 196 if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus")) 197 goto out; 198 } 199 200 /* update value on specified cpu */ 201 for (i = 0; i < entries; i++) { 202 values[0] = value; 203 flags = (u64)cpu << 32 | BPF_F_CPU; 204 err = bpf_map__update_elem(map, keys + i * key_sz, key_sz, values, 205 value_sz, flags); 206 if (!ASSERT_OK(err, "bpf_map__update_elem specified cpu")) 207 goto out; 208 209 /* lookup then check value on CPUs */ 210 for (j = 0; j < nr_cpus; j++) { 211 flags = (u64)j << 32 | BPF_F_CPU; 212 err = bpf_map__lookup_elem(map, keys + i * key_sz, key_sz, values, 213 value_sz, flags); 214 if (!ASSERT_OK(err, "bpf_map__lookup_elem specified cpu")) 215 goto out; 216 if (!ASSERT_EQ(values[0], j != cpu ? 0 : value, 217 "bpf_map__lookup_elem value on specified cpu")) 218 goto out; 219 } 220 } 221 } 222 223 if (!test_batch) 224 goto out; 225 226 count = entries; 227 batch_opts.elem_flags = (u64)nr_cpus << 32 | BPF_F_CPU; 228 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts); 229 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_update_batch -ERANGE")) 230 goto out; 231 232 for (cpu = 0; cpu < nr_cpus; cpu++) { 233 memset(values, 0, value_sz_total); 234 235 /* clear values across all CPUs */ 236 count = entries; 237 batch_opts.elem_flags = BPF_F_ALL_CPUS; 238 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts); 239 if (!ASSERT_OK(err, "bpf_map_update_batch all_cpus")) 240 goto out; 241 if (!ASSERT_EQ(count, entries, "bpf_map_update_batch count")) 242 goto out; 243 244 /* update values on specified CPU */ 245 for (i = 0; i < entries; i++) 246 values[i] = value; 247 248 count = entries; 249 batch_opts.elem_flags = (u64)cpu << 32 | BPF_F_CPU; 250 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts); 251 if (!ASSERT_OK(err, "bpf_map_update_batch specified cpu")) 252 goto out; 253 if (!ASSERT_EQ(count, entries, "bpf_map_update_batch count")) 254 goto out; 255 256 /* lookup values on specified CPU */ 257 batch = 0; 258 count = entries; 259 memset(values, 0, entries * value_sz); 260 err = bpf_map_lookup_batch(map_fd, NULL, &batch, keys, values, &count, &batch_opts); 261 if (!ASSERT_TRUE(!err || err == -ENOENT, "bpf_map_lookup_batch specified cpu")) 262 goto out; 263 if (!ASSERT_EQ(count, entries, "bpf_map_lookup_batch count")) 264 goto out; 265 266 for (i = 0; i < entries; i++) 267 if (!ASSERT_EQ(values[i], value, 268 "bpf_map_lookup_batch value on specified cpu")) 269 goto out; 270 271 /* lookup values from all CPUs */ 272 batch = 0; 273 count = entries; 274 batch_opts.elem_flags = 0; 275 memset(values_percpu, 0, roundup(value_sz, 8) * nr_cpus * entries); 276 err = bpf_map_lookup_batch(map_fd, NULL, &batch, keys, values_percpu, &count, 277 &batch_opts); 278 if (!ASSERT_TRUE(!err || err == -ENOENT, "bpf_map_lookup_batch all_cpus")) 279 goto out; 280 if (!ASSERT_EQ(count, entries, "bpf_map_lookup_batch count")) 281 goto out; 282 283 for (i = 0; i < entries; i++) { 284 values_row = (void *) values_percpu + 285 roundup(value_sz, 8) * i * nr_cpus; 286 for (j = 0; j < nr_cpus; j++) { 287 v = *(u32 *) (values_row + roundup(value_sz, 8) * j); 288 if (!ASSERT_EQ(v, j != cpu ? 0 : value, 289 "bpf_map_lookup_batch value all_cpus")) 290 goto out; 291 } 292 } 293 } 294 295 out: 296 free(values_percpu); 297 free(values); 298 } 299 300 static void test_percpu_map_cpu_flag(enum bpf_map_type map_type) 301 { 302 struct percpu_alloc_array *skel; 303 size_t key_sz = sizeof(int); 304 int *keys, nr_cpus, i, err; 305 struct bpf_map *map; 306 u32 max_entries; 307 308 nr_cpus = libbpf_num_possible_cpus(); 309 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) 310 return; 311 312 max_entries = nr_cpus * 2; 313 keys = calloc(max_entries, key_sz); 314 if (!ASSERT_OK_PTR(keys, "calloc keys")) 315 return; 316 317 for (i = 0; i < max_entries; i++) 318 keys[i] = i; 319 320 skel = percpu_alloc_array__open(); 321 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open")) { 322 free(keys); 323 return; 324 } 325 326 map = skel->maps.percpu; 327 bpf_map__set_type(map, map_type); 328 bpf_map__set_max_entries(map, max_entries); 329 330 err = percpu_alloc_array__load(skel); 331 if (!ASSERT_OK(err, "test_percpu_alloc__load")) 332 goto out; 333 334 test_percpu_map_op_cpu_flag(map, keys, key_sz, nr_cpus, nr_cpus, true); 335 out: 336 percpu_alloc_array__destroy(skel); 337 free(keys); 338 } 339 340 static void test_percpu_array_cpu_flag(void) 341 { 342 test_percpu_map_cpu_flag(BPF_MAP_TYPE_PERCPU_ARRAY); 343 } 344 345 static void test_percpu_hash_cpu_flag(void) 346 { 347 test_percpu_map_cpu_flag(BPF_MAP_TYPE_PERCPU_HASH); 348 } 349 350 static void test_lru_percpu_hash_cpu_flag(void) 351 { 352 test_percpu_map_cpu_flag(BPF_MAP_TYPE_LRU_PERCPU_HASH); 353 } 354 355 /* 356 * A BPF_F_CPU update that creates an element must zero the value on the other 357 * cpus, rather than leave them holding whatever the recycled element last 358 * contained. max_entries is 1 so the second key can only reuse the element 359 * the first one released. 360 */ 361 static void test_percpu_map_cpu_flag_create(enum bpf_map_type map_type, __u32 map_flags) 362 { 363 LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags); 364 const u32 stale = 0xDEADC0DE, fresh = 0xC0FFEE; 365 int nr_cpus, cpu, map_fd, err, key; 366 int pinned_cpu, value_cpu; 367 cpu_set_t old_mask, new_mask; 368 bool restore_mask = false; 369 u32 value; 370 u64 flags; 371 372 nr_cpus = libbpf_num_possible_cpus(); 373 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) 374 return; 375 376 if (nr_cpus < 2) { 377 test__skip(); 378 return; 379 } 380 381 map_fd = bpf_map_create(map_type, "cpu_flag_create", sizeof(key), sizeof(value), 1, &opts); 382 if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) 383 return; 384 385 /* NO_PREALLOC recycles per cpu, so keep the delete and the create on one cpu. */ 386 err = sched_getaffinity(0, sizeof(old_mask), &old_mask); 387 if (!ASSERT_OK(err, "sched_getaffinity")) 388 goto out; 389 390 pinned_cpu = sched_getcpu(); 391 if (!ASSERT_GE(pinned_cpu, 0, "sched_getcpu")) 392 goto out; 393 394 CPU_ZERO(&new_mask); 395 CPU_SET(pinned_cpu, &new_mask); 396 err = sched_setaffinity(0, sizeof(new_mask), &new_mask); 397 if (!ASSERT_OK(err, "sched_setaffinity")) 398 goto out; 399 restore_mask = true; 400 401 value_cpu = pinned_cpu ? 0 : 1; 402 403 key = 1; 404 value = stale; 405 err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); 406 if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus")) 407 goto out; 408 409 err = bpf_map_delete_elem(map_fd, &key); 410 if (!ASSERT_OK(err, "bpf_map_delete_elem")) 411 goto out; 412 413 key = 2; 414 value = fresh; 415 flags = (u64)value_cpu << 32 | BPF_F_CPU; 416 err = bpf_map_update_elem(map_fd, &key, &value, flags); 417 if (!ASSERT_OK(err, "bpf_map_update_elem specified cpu")) 418 goto out; 419 420 for (cpu = 0; cpu < nr_cpus; cpu++) { 421 value = 0; 422 flags = (u64)cpu << 32 | BPF_F_CPU; 423 err = bpf_map_lookup_elem_flags(map_fd, &key, &value, flags); 424 if (!ASSERT_OK(err, "bpf_map_lookup_elem_flags specified cpu")) 425 goto out; 426 if (!ASSERT_EQ(value, cpu == value_cpu ? fresh : 0, "value on specified cpu")) 427 goto out; 428 } 429 430 out: 431 if (restore_mask) 432 sched_setaffinity(0, sizeof(old_mask), &old_mask); 433 close(map_fd); 434 } 435 436 static void test_percpu_hash_cpu_flag_create(void) 437 { 438 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, 0); 439 } 440 441 static void test_percpu_hash_cpu_flag_create_malloc(void) 442 { 443 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, BPF_F_NO_PREALLOC); 444 } 445 446 static void test_lru_percpu_hash_cpu_flag_create(void) 447 { 448 /* lru without prealloc is -ENOTSUPP, so there is no malloc variant */ 449 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); 450 } 451 452 static void test_percpu_cgroup_storage_cpu_flag(void) 453 { 454 struct percpu_alloc_array *skel = NULL; 455 struct bpf_cgroup_storage_key key; 456 int cgroup, prog_fd, nr_cpus, err; 457 struct bpf_map *map; 458 459 nr_cpus = libbpf_num_possible_cpus(); 460 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) 461 return; 462 463 err = setup_cgroup_environment(); 464 if (!ASSERT_OK(err, "setup_cgroup_environment")) 465 return; 466 467 cgroup = create_and_get_cgroup("/cg_percpu"); 468 if (!ASSERT_GE(cgroup, 0, "create_and_get_cgroup")) { 469 cleanup_cgroup_environment(); 470 return; 471 } 472 473 err = join_cgroup("/cg_percpu"); 474 if (!ASSERT_OK(err, "join_cgroup")) 475 goto out; 476 477 skel = percpu_alloc_array__open_and_load(); 478 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open_and_load")) 479 goto out; 480 481 prog_fd = bpf_program__fd(skel->progs.cgroup_egress); 482 err = bpf_prog_attach(prog_fd, cgroup, BPF_CGROUP_INET_EGRESS, 0); 483 if (!ASSERT_OK(err, "bpf_prog_attach")) 484 goto out; 485 486 map = skel->maps.percpu_cgroup_storage; 487 err = bpf_map_get_next_key(bpf_map__fd(map), NULL, &key); 488 if (!ASSERT_OK(err, "bpf_map_get_next_key")) 489 goto out; 490 491 test_percpu_map_op_cpu_flag(map, &key, sizeof(key), 1, nr_cpus, false); 492 out: 493 bpf_prog_detach2(-1, cgroup, BPF_CGROUP_INET_EGRESS); 494 close(cgroup); 495 cleanup_cgroup_environment(); 496 percpu_alloc_array__destroy(skel); 497 } 498 499 static void test_map_op_cpu_flag(enum bpf_map_type map_type) 500 { 501 u32 max_entries = 1, count = max_entries; 502 u64 flags, batch = 0, val = 0; 503 int err, map_fd, key = 0; 504 LIBBPF_OPTS(bpf_map_batch_opts, batch_opts); 505 506 map_fd = bpf_map_create(map_type, "test_cpu_flag", sizeof(int), sizeof(u64), max_entries, 507 NULL); 508 if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) 509 return; 510 511 flags = BPF_F_ALL_CPUS; 512 err = bpf_map_update_elem(map_fd, &key, &val, flags); 513 ASSERT_ERR(err, "bpf_map_update_elem all_cpus"); 514 515 batch_opts.elem_flags = BPF_F_ALL_CPUS; 516 err = bpf_map_update_batch(map_fd, &key, &val, &count, &batch_opts); 517 ASSERT_ERR(err, "bpf_map_update_batch all_cpus"); 518 519 flags = BPF_F_CPU; 520 err = bpf_map_lookup_elem_flags(map_fd, &key, &val, flags); 521 ASSERT_ERR(err, "bpf_map_lookup_elem_flags cpu"); 522 523 batch_opts.elem_flags = BPF_F_CPU; 524 err = bpf_map_lookup_batch(map_fd, NULL, &batch, &key, &val, &count, &batch_opts); 525 ASSERT_ERR(err, "bpf_map_lookup_batch cpu"); 526 527 close(map_fd); 528 } 529 530 static void test_array_cpu_flag(void) 531 { 532 test_map_op_cpu_flag(BPF_MAP_TYPE_ARRAY); 533 } 534 535 static void test_hash_cpu_flag(void) 536 { 537 test_map_op_cpu_flag(BPF_MAP_TYPE_HASH); 538 } 539 540 void test_percpu_alloc(void) 541 { 542 if (test__start_subtest("array")) 543 test_array(); 544 if (test__start_subtest("array_sleepable")) 545 test_array_sleepable(); 546 if (test__start_subtest("cgrp_local_storage")) 547 test_cgrp_local_storage(); 548 if (test__start_subtest("failure_tests")) 549 test_failure(); 550 if (test__start_subtest("cpu_flag_percpu_array")) 551 test_percpu_array_cpu_flag(); 552 if (test__start_subtest("cpu_flag_percpu_hash")) 553 test_percpu_hash_cpu_flag(); 554 if (test__start_subtest("cpu_flag_lru_percpu_hash")) 555 test_lru_percpu_hash_cpu_flag(); 556 if (test__start_subtest("cpu_flag_create_percpu_hash")) 557 test_percpu_hash_cpu_flag_create(); 558 if (test__start_subtest("cpu_flag_create_percpu_hash_malloc")) 559 test_percpu_hash_cpu_flag_create_malloc(); 560 if (test__start_subtest("cpu_flag_create_lru_percpu_hash")) 561 test_lru_percpu_hash_cpu_flag_create(); 562 if (test__start_subtest("cpu_flag_percpu_cgroup_storage")) 563 test_percpu_cgroup_storage_cpu_flag(); 564 if (test__start_subtest("cpu_flag_array")) 565 test_array_cpu_flag(); 566 if (test__start_subtest("cpu_flag_hash")) 567 test_hash_cpu_flag(); 568 } 569