1 // SPDX-License-Identifier: GPL-2.0 2 #define _GNU_SOURCE 3 #include <sched.h> 4 #include <test_progs.h> 5 #include "cgroup_helpers.h" 6 #include "percpu_alloc_array.skel.h" 7 #include "percpu_alloc_cgrp_local_storage.skel.h" 8 #include "percpu_alloc_fail.skel.h" 9 10 static void test_array(void) 11 { 12 struct percpu_alloc_array *skel; 13 int err, prog_fd; 14 LIBBPF_OPTS(bpf_test_run_opts, topts); 15 16 skel = percpu_alloc_array__open(); 17 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open")) 18 return; 19 20 bpf_program__set_autoload(skel->progs.test_array_map_1, true); 21 bpf_program__set_autoload(skel->progs.test_array_map_2, true); 22 bpf_program__set_autoload(skel->progs.test_array_map_3, true); 23 bpf_program__set_autoload(skel->progs.test_array_map_4, true); 24 25 skel->bss->my_pid = getpid(); 26 skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 27 28 err = percpu_alloc_array__load(skel); 29 if (!ASSERT_OK(err, "percpu_alloc_array__load")) 30 goto out; 31 32 err = percpu_alloc_array__attach(skel); 33 if (!ASSERT_OK(err, "percpu_alloc_array__attach")) 34 goto out; 35 36 prog_fd = bpf_program__fd(skel->progs.test_array_map_1); 37 err = bpf_prog_test_run_opts(prog_fd, &topts); 38 ASSERT_OK(err, "test_run array_map 1-4"); 39 ASSERT_EQ(topts.retval, 0, "test_run array_map 1-4"); 40 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 41 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 42 out: 43 percpu_alloc_array__destroy(skel); 44 } 45 46 static void test_array_sleepable(void) 47 { 48 struct percpu_alloc_array *skel; 49 int err, prog_fd; 50 LIBBPF_OPTS(bpf_test_run_opts, topts); 51 52 skel = percpu_alloc_array__open(); 53 if (!ASSERT_OK_PTR(skel, "percpu_alloc__open")) 54 return; 55 56 bpf_program__set_autoload(skel->progs.test_array_map_10, true); 57 58 skel->bss->my_pid = getpid(); 59 skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 60 61 err = percpu_alloc_array__load(skel); 62 if (!ASSERT_OK(err, "percpu_alloc_array__load")) 63 goto out; 64 65 err = percpu_alloc_array__attach(skel); 66 if (!ASSERT_OK(err, "percpu_alloc_array__attach")) 67 goto out; 68 69 prog_fd = bpf_program__fd(skel->progs.test_array_map_10); 70 err = bpf_prog_test_run_opts(prog_fd, &topts); 71 ASSERT_OK(err, "test_run array_map_10"); 72 ASSERT_EQ(topts.retval, 0, "test_run array_map_10"); 73 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 74 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 75 out: 76 percpu_alloc_array__destroy(skel); 77 } 78 79 static void test_cgrp_local_storage(void) 80 { 81 struct percpu_alloc_cgrp_local_storage *skel; 82 int err, cgroup_fd, prog_fd; 83 LIBBPF_OPTS(bpf_test_run_opts, topts); 84 85 cgroup_fd = test__join_cgroup("/percpu_alloc"); 86 if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /percpu_alloc")) 87 return; 88 89 skel = percpu_alloc_cgrp_local_storage__open(); 90 if (!ASSERT_OK_PTR(skel, "percpu_alloc_cgrp_local_storage__open")) 91 goto close_fd; 92 93 skel->bss->my_pid = getpid(); 94 skel->rodata->nr_cpus = libbpf_num_possible_cpus(); 95 96 err = percpu_alloc_cgrp_local_storage__load(skel); 97 if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__load")) 98 goto destroy_skel; 99 100 err = percpu_alloc_cgrp_local_storage__attach(skel); 101 if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__attach")) 102 goto destroy_skel; 103 104 prog_fd = bpf_program__fd(skel->progs.test_cgrp_local_storage_1); 105 err = bpf_prog_test_run_opts(prog_fd, &topts); 106 ASSERT_OK(err, "test_run cgrp_local_storage 1-3"); 107 ASSERT_EQ(topts.retval, 0, "test_run cgrp_local_storage 1-3"); 108 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d"); 109 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c"); 110 111 destroy_skel: 112 percpu_alloc_cgrp_local_storage__destroy(skel); 113 close_fd: 114 close(cgroup_fd); 115 } 116 117 static void test_failure(void) { 118 RUN_TESTS(percpu_alloc_fail); 119 } 120 121 static void test_percpu_map_op_cpu_flag(struct bpf_map *map, void *keys, size_t key_sz, u32 entries, 122 int nr_cpus, bool test_batch) 123 { 124 size_t value_sz = sizeof(u32), value_sz_cpus, value_sz_total; 125 u32 *values = NULL, *values_percpu = NULL; 126 const u32 value = 0xDEADC0DE; 127 int i, j, cpu, map_fd, err; 128 u64 batch = 0, flags; 129 void *values_row; 130 u32 count, v; 131 LIBBPF_OPTS(bpf_map_batch_opts, batch_opts); 132 133 value_sz_cpus = value_sz * nr_cpus; 134 values = calloc(entries, value_sz_cpus); 135 if (!ASSERT_OK_PTR(values, "calloc values")) 136 return; 137 138 values_percpu = calloc(entries, roundup(value_sz, 8) * nr_cpus); 139 if (!ASSERT_OK_PTR(values_percpu, "calloc values_percpu")) { 140 free(values); 141 return; 142 } 143 144 value_sz_total = value_sz_cpus * entries; 145 memset(values, 0, value_sz_total); 146 147 map_fd = bpf_map__fd(map); 148 flags = BPF_F_CPU | BPF_F_ALL_CPUS; 149 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 150 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags cpu|all_cpus")) 151 goto out; 152 153 err = bpf_map_update_elem(map_fd, keys, values, flags); 154 if (!ASSERT_ERR(err, "bpf_map_update_elem cpu|all_cpus")) 155 goto out; 156 157 flags = BPF_F_ALL_CPUS; 158 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 159 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags all_cpus")) 160 goto out; 161 162 flags = BPF_F_LOCK | BPF_F_CPU; 163 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 164 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags BPF_F_LOCK")) 165 goto out; 166 167 flags = BPF_F_LOCK | BPF_F_ALL_CPUS; 168 err = bpf_map_update_elem(map_fd, keys, values, flags); 169 if (!ASSERT_ERR(err, "bpf_map_update_elem BPF_F_LOCK")) 170 goto out; 171 172 flags = (u64)nr_cpus << 32 | BPF_F_CPU; 173 err = bpf_map_update_elem(map_fd, keys, values, flags); 174 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_update_elem -ERANGE")) 175 goto out; 176 177 err = bpf_map__update_elem(map, keys, key_sz, values, value_sz, flags); 178 if (!ASSERT_EQ(err, -ERANGE, "bpf_map__update_elem -ERANGE")) 179 goto out; 180 181 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags); 182 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_lookup_elem_flags -ERANGE")) 183 goto out; 184 185 err = bpf_map__lookup_elem(map, keys, key_sz, values, value_sz, flags); 186 if (!ASSERT_EQ(err, -ERANGE, "bpf_map__lookup_elem -ERANGE")) 187 goto out; 188 189 for (cpu = 0; cpu < nr_cpus; cpu++) { 190 /* clear value on all cpus */ 191 values[0] = 0; 192 flags = BPF_F_ALL_CPUS; 193 for (i = 0; i < entries; i++) { 194 err = bpf_map__update_elem(map, keys + i * key_sz, key_sz, values, 195 value_sz, flags); 196 if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus")) 197 goto out; 198 } 199 200 /* update value on specified cpu */ 201 for (i = 0; i < entries; i++) { 202 values[0] = value; 203 flags = (u64)cpu << 32 | BPF_F_CPU; 204 err = bpf_map__update_elem(map, keys + i * key_sz, key_sz, values, 205 value_sz, flags); 206 if (!ASSERT_OK(err, "bpf_map__update_elem specified cpu")) 207 goto out; 208 209 /* lookup then check value on CPUs */ 210 for (j = 0; j < nr_cpus; j++) { 211 flags = (u64)j << 32 | BPF_F_CPU; 212 err = bpf_map__lookup_elem(map, keys + i * key_sz, key_sz, values, 213 value_sz, flags); 214 if (!ASSERT_OK(err, "bpf_map__lookup_elem specified cpu")) 215 goto out; 216 if (!ASSERT_EQ(values[0], j != cpu ? 0 : value, 217 "bpf_map__lookup_elem value on specified cpu")) 218 goto out; 219 } 220 } 221 } 222 223 if (!test_batch) 224 goto out; 225 226 count = entries; 227 batch_opts.elem_flags = (u64)nr_cpus << 32 | BPF_F_CPU; 228 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts); 229 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_update_batch -ERANGE")) 230 goto out; 231 232 for (cpu = 0; cpu < nr_cpus; cpu++) { 233 memset(values, 0, value_sz_total); 234 235 /* clear values across all CPUs */ 236 count = entries; 237 batch_opts.elem_flags = BPF_F_ALL_CPUS; 238 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts); 239 if (!ASSERT_OK(err, "bpf_map_update_batch all_cpus")) 240 goto out; 241 if (!ASSERT_EQ(count, entries, "bpf_map_update_batch count")) 242 goto out; 243 244 /* update values on specified CPU */ 245 for (i = 0; i < entries; i++) 246 values[i] = value; 247 248 count = entries; 249 batch_opts.elem_flags = (u64)cpu << 32 | BPF_F_CPU; 250 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts); 251 if (!ASSERT_OK(err, "bpf_map_update_batch specified cpu")) 252 goto out; 253 if (!ASSERT_EQ(count, entries, "bpf_map_update_batch count")) 254 goto out; 255 256 /* lookup values on specified CPU */ 257 batch = 0; 258 count = entries; 259 memset(values, 0, entries * value_sz); 260 err = bpf_map_lookup_batch(map_fd, NULL, &batch, keys, values, &count, &batch_opts); 261 if (!ASSERT_TRUE(!err || err == -ENOENT, "bpf_map_lookup_batch specified cpu")) 262 goto out; 263 if (!ASSERT_EQ(count, entries, "bpf_map_lookup_batch count")) 264 goto out; 265 266 for (i = 0; i < entries; i++) 267 if (!ASSERT_EQ(values[i], value, 268 "bpf_map_lookup_batch value on specified cpu")) 269 goto out; 270 271 /* lookup values from all CPUs */ 272 batch = 0; 273 count = entries; 274 batch_opts.elem_flags = 0; 275 memset(values_percpu, 0, roundup(value_sz, 8) * nr_cpus * entries); 276 err = bpf_map_lookup_batch(map_fd, NULL, &batch, keys, values_percpu, &count, 277 &batch_opts); 278 if (!ASSERT_TRUE(!err || err == -ENOENT, "bpf_map_lookup_batch all_cpus")) 279 goto out; 280 if (!ASSERT_EQ(count, entries, "bpf_map_lookup_batch count")) 281 goto out; 282 283 for (i = 0; i < entries; i++) { 284 values_row = (void *) values_percpu + 285 roundup(value_sz, 8) * i * nr_cpus; 286 for (j = 0; j < nr_cpus; j++) { 287 v = *(u32 *) (values_row + roundup(value_sz, 8) * j); 288 if (!ASSERT_EQ(v, j != cpu ? 0 : value, 289 "bpf_map_lookup_batch value all_cpus")) 290 goto out; 291 } 292 } 293 } 294 295 out: 296 free(values_percpu); 297 free(values); 298 } 299 300 static void test_percpu_map_cpu_flag(enum bpf_map_type map_type) 301 { 302 struct percpu_alloc_array *skel; 303 size_t key_sz = sizeof(int); 304 int *keys, nr_cpus, i, err; 305 struct bpf_map *map; 306 u32 max_entries; 307 308 nr_cpus = libbpf_num_possible_cpus(); 309 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) 310 return; 311 312 max_entries = nr_cpus * 2; 313 keys = calloc(max_entries, key_sz); 314 if (!ASSERT_OK_PTR(keys, "calloc keys")) 315 return; 316 317 for (i = 0; i < max_entries; i++) 318 keys[i] = i; 319 320 skel = percpu_alloc_array__open(); 321 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open")) { 322 free(keys); 323 return; 324 } 325 326 map = skel->maps.percpu; 327 bpf_map__set_type(map, map_type); 328 bpf_map__set_max_entries(map, max_entries); 329 330 err = percpu_alloc_array__load(skel); 331 if (!ASSERT_OK(err, "test_percpu_alloc__load")) 332 goto out; 333 334 test_percpu_map_op_cpu_flag(map, keys, key_sz, nr_cpus, nr_cpus, true); 335 out: 336 percpu_alloc_array__destroy(skel); 337 free(keys); 338 } 339 340 static void test_percpu_array_cpu_flag(void) 341 { 342 test_percpu_map_cpu_flag(BPF_MAP_TYPE_PERCPU_ARRAY); 343 } 344 345 static void test_percpu_hash_cpu_flag(void) 346 { 347 test_percpu_map_cpu_flag(BPF_MAP_TYPE_PERCPU_HASH); 348 } 349 350 static void test_lru_percpu_hash_cpu_flag(void) 351 { 352 test_percpu_map_cpu_flag(BPF_MAP_TYPE_LRU_PERCPU_HASH); 353 } 354 355 /* 356 * A BPF_F_CPU update that creates an element must zero the value on the other 357 * cpus, rather than leave them holding whatever the recycled element last 358 * contained. max_entries is 1 so the second key can only reuse the element 359 * the first one released. 360 */ 361 static void test_percpu_map_cpu_flag_create(enum bpf_map_type map_type, __u32 map_flags) 362 { 363 LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags); 364 const u32 stale = 0xDEADC0DE, fresh = 0xC0FFEE; 365 int nr_cpus, cpu, map_fd, err, key; 366 int pinned_cpu, value_cpu; 367 cpu_set_t old_mask, new_mask; 368 bool restore_mask = false; 369 u32 value; 370 u64 flags; 371 372 nr_cpus = libbpf_num_possible_cpus(); 373 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) 374 return; 375 376 if (nr_cpus < 2) { 377 test__skip(); 378 return; 379 } 380 381 map_fd = bpf_map_create(map_type, "cpu_flag_create", sizeof(key), sizeof(value), 1, &opts); 382 if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) 383 return; 384 385 /* NO_PREALLOC recycles per cpu, so keep the delete and the create on one cpu. */ 386 err = sched_getaffinity(0, sizeof(old_mask), &old_mask); 387 if (!ASSERT_OK(err, "sched_getaffinity")) 388 goto out; 389 390 pinned_cpu = sched_getcpu(); 391 if (!ASSERT_GE(pinned_cpu, 0, "sched_getcpu")) 392 goto out; 393 394 CPU_ZERO(&new_mask); 395 CPU_SET(pinned_cpu, &new_mask); 396 err = sched_setaffinity(0, sizeof(new_mask), &new_mask); 397 if (!ASSERT_OK(err, "sched_setaffinity")) 398 goto out; 399 restore_mask = true; 400 401 value_cpu = pinned_cpu ? 0 : 1; 402 403 key = 1; 404 value = stale; 405 err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); 406 if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus")) 407 goto out; 408 409 err = bpf_map_delete_elem(map_fd, &key); 410 if (!ASSERT_OK(err, "bpf_map_delete_elem")) 411 goto out; 412 413 key = 2; 414 value = fresh; 415 flags = (u64)value_cpu << 32 | BPF_F_CPU; 416 err = bpf_map_update_elem(map_fd, &key, &value, flags); 417 if (!ASSERT_OK(err, "bpf_map_update_elem specified cpu")) 418 goto out; 419 420 for (cpu = 0; cpu < nr_cpus; cpu++) { 421 value = 0; 422 flags = (u64)cpu << 32 | BPF_F_CPU; 423 err = bpf_map_lookup_elem_flags(map_fd, &key, &value, flags); 424 if (!ASSERT_OK(err, "bpf_map_lookup_elem_flags specified cpu")) 425 goto out; 426 if (!ASSERT_EQ(value, cpu == value_cpu ? fresh : 0, "value on specified cpu")) 427 goto out; 428 } 429 430 out: 431 if (restore_mask) 432 sched_setaffinity(0, sizeof(old_mask), &old_mask); 433 close(map_fd); 434 } 435 436 static void test_percpu_hash_cpu_flag_create(void) 437 { 438 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, 0); 439 } 440 441 static void test_percpu_hash_cpu_flag_create_malloc(void) 442 { 443 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, BPF_F_NO_PREALLOC); 444 } 445 446 static void test_lru_percpu_hash_cpu_flag_create(void) 447 { 448 /* lru without prealloc is -ENOTSUPP, so there is no malloc variant */ 449 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); 450 } 451 452 static bool verify_map_value_all_cpus(int map_fd, void *key, void *values, size_t elem_sz, 453 u32 expected_value, int nr_cpus) 454 { 455 int i, err; 456 457 err = bpf_map_lookup_elem(map_fd, key, values); 458 if (!ASSERT_OK(err, "bpf_map_lookup_elem all_cpus")) 459 return false; 460 461 for (i = 0 ; i < nr_cpus ; i++) { 462 if (!ASSERT_EQ(*(u32 *)(values + elem_sz * i), expected_value, 463 "bpf_map_lookup_elem value all_cpus")) 464 return false; 465 } 466 467 return true; 468 } 469 470 static bool verify_map_value_first_cpu(int map_fd, void *key, void *values, size_t elem_sz, 471 u32 expected_value, int nr_cpus) 472 { 473 int i, err; 474 475 err = bpf_map_lookup_elem(map_fd, key, values); 476 if (!ASSERT_OK(err, "bpf_map_lookup_elem cpu")) 477 return false; 478 479 if (!ASSERT_EQ(*(u32 *)values, expected_value, "bpf_map_lookup_elem value cpu")) 480 return false; 481 482 for (i = 1 ; i < nr_cpus ; i++) { 483 if (!ASSERT_NEQ(*(u32 *)(values + elem_sz * i), expected_value, 484 "bpf_map_lookup_elem value cpu")) 485 return false; 486 } 487 488 return true; 489 } 490 491 static void test_percpu_cgroup_storage_flags_combination(struct bpf_map *map, int nr_cpus, 492 struct bpf_cgroup_storage_key *key) 493 { 494 size_t value_sz = sizeof(u32), key_sz, elem_sz; 495 u32 *values, test_value = 0xAABBCCDD; 496 int err, map_fd; 497 u64 flags; 498 499 key_sz = sizeof(*key); 500 elem_sz = roundup(value_sz, 8); 501 map_fd = bpf_map__fd(map); 502 values = calloc(nr_cpus, elem_sz); 503 if (!ASSERT_OK_PTR(values, "calloc values")) 504 return; 505 506 flags = BPF_NOEXIST | BPF_EXIST; 507 err = bpf_map__update_elem(map, key, key_sz, values, elem_sz * nr_cpus, flags); 508 if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem noexist|exist")) 509 goto out; 510 511 flags = BPF_F_ALL_CPUS | BPF_NOEXIST; 512 err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); 513 if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem all_cpus|noexist")) 514 goto out; 515 516 flags = BPF_F_CPU | BPF_NOEXIST; 517 err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); 518 if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem cpu|noexist")) 519 goto out; 520 521 flags = BPF_F_ALL_CPUS | BPF_EXIST; 522 values[0] = test_value; 523 err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); 524 if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus|exist")) 525 goto out; 526 527 if (!verify_map_value_all_cpus(map_fd, key, values, elem_sz, test_value, nr_cpus)) 528 goto out; 529 530 flags = BPF_F_CPU | BPF_EXIST; 531 values[0] = --test_value; 532 err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); 533 if (!ASSERT_OK(err, "bpf_map__update_elem cpu|exist")) 534 goto out; 535 536 verify_map_value_first_cpu(map_fd, key, values, elem_sz, test_value, nr_cpus); 537 out: 538 free(values); 539 } 540 541 static void test_percpu_cgroup_storage_cpu_flag(void) 542 { 543 struct percpu_alloc_array *skel = NULL; 544 struct bpf_cgroup_storage_key key; 545 int cgroup, prog_fd, nr_cpus, err; 546 struct bpf_map *map; 547 548 nr_cpus = libbpf_num_possible_cpus(); 549 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) 550 return; 551 552 err = setup_cgroup_environment(); 553 if (!ASSERT_OK(err, "setup_cgroup_environment")) 554 return; 555 556 cgroup = create_and_get_cgroup("/cg_percpu"); 557 if (!ASSERT_GE(cgroup, 0, "create_and_get_cgroup")) { 558 cleanup_cgroup_environment(); 559 return; 560 } 561 562 err = join_cgroup("/cg_percpu"); 563 if (!ASSERT_OK(err, "join_cgroup")) 564 goto out; 565 566 skel = percpu_alloc_array__open_and_load(); 567 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open_and_load")) 568 goto out; 569 570 prog_fd = bpf_program__fd(skel->progs.cgroup_egress); 571 err = bpf_prog_attach(prog_fd, cgroup, BPF_CGROUP_INET_EGRESS, 0); 572 if (!ASSERT_OK(err, "bpf_prog_attach")) 573 goto out; 574 575 map = skel->maps.percpu_cgroup_storage; 576 err = bpf_map_get_next_key(bpf_map__fd(map), NULL, &key); 577 if (!ASSERT_OK(err, "bpf_map_get_next_key")) 578 goto out; 579 580 test_percpu_map_op_cpu_flag(map, &key, sizeof(key), 1, nr_cpus, false); 581 test_percpu_cgroup_storage_flags_combination(map, nr_cpus, &key); 582 out: 583 bpf_prog_detach2(-1, cgroup, BPF_CGROUP_INET_EGRESS); 584 close(cgroup); 585 cleanup_cgroup_environment(); 586 percpu_alloc_array__destroy(skel); 587 } 588 589 static void test_map_op_cpu_flag(enum bpf_map_type map_type) 590 { 591 u32 max_entries = 1, count = max_entries; 592 u64 flags, batch = 0, val = 0; 593 int err, map_fd, key = 0; 594 LIBBPF_OPTS(bpf_map_batch_opts, batch_opts); 595 596 map_fd = bpf_map_create(map_type, "test_cpu_flag", sizeof(int), sizeof(u64), max_entries, 597 NULL); 598 if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) 599 return; 600 601 flags = BPF_F_ALL_CPUS; 602 err = bpf_map_update_elem(map_fd, &key, &val, flags); 603 ASSERT_ERR(err, "bpf_map_update_elem all_cpus"); 604 605 batch_opts.elem_flags = BPF_F_ALL_CPUS; 606 err = bpf_map_update_batch(map_fd, &key, &val, &count, &batch_opts); 607 ASSERT_ERR(err, "bpf_map_update_batch all_cpus"); 608 609 flags = BPF_F_CPU; 610 err = bpf_map_lookup_elem_flags(map_fd, &key, &val, flags); 611 ASSERT_ERR(err, "bpf_map_lookup_elem_flags cpu"); 612 613 batch_opts.elem_flags = BPF_F_CPU; 614 err = bpf_map_lookup_batch(map_fd, NULL, &batch, &key, &val, &count, &batch_opts); 615 ASSERT_ERR(err, "bpf_map_lookup_batch cpu"); 616 617 close(map_fd); 618 } 619 620 static void test_array_cpu_flag(void) 621 { 622 test_map_op_cpu_flag(BPF_MAP_TYPE_ARRAY); 623 } 624 625 static void test_hash_cpu_flag(void) 626 { 627 test_map_op_cpu_flag(BPF_MAP_TYPE_HASH); 628 } 629 630 static void test_percpu_map_flags_combination(enum bpf_map_type map_type) 631 { 632 u32 max_entries = 3, key = 0, first_value = 0xAABBCCDD, test_value = first_value; 633 size_t value_sz = sizeof(u32), elem_sz; 634 int err, map_fd, nr_cpus; 635 bool is_hash_map; 636 u32 *values; 637 u64 flags; 638 639 is_hash_map = (map_type == BPF_MAP_TYPE_PERCPU_HASH || 640 map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); 641 nr_cpus = libbpf_num_possible_cpus(); 642 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) 643 return; 644 645 elem_sz = roundup(value_sz, 8); 646 values = calloc(nr_cpus, elem_sz); 647 if (!ASSERT_OK_PTR(values, "calloc values")) 648 return; 649 650 map_fd = bpf_map_create(map_type, "test_flags_combination_map", 651 sizeof(u32), value_sz, max_entries, NULL); 652 if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) { 653 free(values); 654 return; 655 } 656 657 flags = BPF_NOEXIST | BPF_EXIST; 658 err = bpf_map_update_elem(map_fd, &key, values, flags); 659 if (!ASSERT_EQ(err, -EINVAL, "bpf_map_update_elem noexist|exist")) 660 goto out; 661 662 flags = BPF_F_ALL_CPUS | BPF_NOEXIST; 663 values[0] = test_value; 664 key = 0; 665 err = bpf_map_update_elem(map_fd, &key, values, flags); 666 if (is_hash_map) { 667 if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|noexist")) 668 goto out; 669 670 if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) 671 goto out; 672 673 err = bpf_map_update_elem(map_fd, &key, values, flags); 674 } 675 if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem all_cpus|noexist")) 676 goto out; 677 678 flags = BPF_F_CPU | BPF_NOEXIST; 679 values[0] = --test_value; 680 key = 1; 681 err = bpf_map_update_elem(map_fd, &key, values, flags); 682 if (is_hash_map) { 683 if (!ASSERT_OK(err, "bpf_map_update_elem cpu|noexist")) 684 goto out; 685 686 if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) 687 goto out; 688 689 err = bpf_map_update_elem(map_fd, &key, values, flags); 690 } 691 if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem cpu|noexist")) 692 goto out; 693 694 flags = BPF_F_ALL_CPUS | BPF_EXIST; 695 values[0] = --test_value; 696 key = 2; 697 err = bpf_map_update_elem(map_fd, &key, values, flags); 698 if (is_hash_map) { 699 if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem all_cpus|exist")) 700 goto out; 701 } else { 702 if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|exist")) 703 goto out; 704 705 if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) 706 goto out; 707 } 708 709 flags = BPF_F_CPU | BPF_EXIST; 710 values[0] = --test_value; 711 err = bpf_map_update_elem(map_fd, &key, values, flags); 712 if (is_hash_map) { 713 if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem cpu|exist")) 714 goto out; 715 } else { 716 if (!ASSERT_OK(err, "bpf_map_update_elem cpu|exist")) 717 goto out; 718 719 if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) 720 goto out; 721 } 722 723 if (map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) 724 goto out; 725 726 /* Percpu LRU hash map should not delete old entries unnecessarily */ 727 flags = BPF_F_ALL_CPUS | BPF_NOEXIST; 728 key = 2; 729 err = bpf_map_update_elem(map_fd, &key, values, flags); 730 if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|noexist")) 731 goto out; 732 733 flags = BPF_F_ALL_CPUS | BPF_EXIST; 734 err = bpf_map_update_elem(map_fd, &key, values, flags); 735 if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|exist")) 736 goto out; 737 738 key = 0; 739 verify_map_value_all_cpus(map_fd, &key, values, elem_sz, first_value, nr_cpus); 740 741 out: 742 close(map_fd); 743 free(values); 744 } 745 746 static bool pin_current_cpu(cpu_set_t *old_mask) 747 { 748 cpu_set_t new_mask; 749 int err, cpu; 750 751 err = sched_getaffinity(0, sizeof(*old_mask), old_mask); 752 if (!ASSERT_OK(err, "sched_getaffinity")) 753 return false; 754 755 cpu = sched_getcpu(); 756 if (!ASSERT_GE(cpu, 0, "sched_getcpu")) 757 return false; 758 759 CPU_ZERO(&new_mask); 760 CPU_SET(cpu, &new_mask); 761 762 err = sched_setaffinity(0, sizeof(new_mask), &new_mask); 763 return ASSERT_OK(err, "sched_setaffinity"); 764 } 765 766 static void test_percpu_hash_flags_combination(void) 767 { 768 test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_HASH); 769 } 770 771 static void test_lru_percpu_hash_flags_combination(void) 772 { 773 cpu_set_t old_mask; 774 775 if (!pin_current_cpu(&old_mask)) 776 return; 777 778 test_percpu_map_flags_combination(BPF_MAP_TYPE_LRU_PERCPU_HASH); 779 780 sched_setaffinity(0, sizeof(old_mask), &old_mask); 781 } 782 783 static void test_percpu_array_flags_combination(void) 784 { 785 test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_ARRAY); 786 } 787 788 void test_percpu_alloc(void) 789 { 790 if (test__start_subtest("array")) 791 test_array(); 792 if (test__start_subtest("array_sleepable")) 793 test_array_sleepable(); 794 if (test__start_subtest("cgrp_local_storage")) 795 test_cgrp_local_storage(); 796 if (test__start_subtest("failure_tests")) 797 test_failure(); 798 if (test__start_subtest("cpu_flag_percpu_array")) 799 test_percpu_array_cpu_flag(); 800 if (test__start_subtest("cpu_flag_percpu_hash")) 801 test_percpu_hash_cpu_flag(); 802 if (test__start_subtest("cpu_flag_lru_percpu_hash")) 803 test_lru_percpu_hash_cpu_flag(); 804 if (test__start_subtest("cpu_flag_create_percpu_hash")) 805 test_percpu_hash_cpu_flag_create(); 806 if (test__start_subtest("cpu_flag_create_percpu_hash_malloc")) 807 test_percpu_hash_cpu_flag_create_malloc(); 808 if (test__start_subtest("cpu_flag_create_lru_percpu_hash")) 809 test_lru_percpu_hash_cpu_flag_create(); 810 if (test__start_subtest("cpu_flag_percpu_cgroup_storage")) 811 test_percpu_cgroup_storage_cpu_flag(); 812 if (test__start_subtest("cpu_flag_array")) 813 test_array_cpu_flag(); 814 if (test__start_subtest("cpu_flag_hash")) 815 test_hash_cpu_flag(); 816 if (test__start_subtest("cpu_flag_combination_percpu_hash")) 817 test_percpu_hash_flags_combination(); 818 if (test__start_subtest("cpu_flag_combination_lru_percpu_hash")) 819 test_lru_percpu_hash_flags_combination(); 820 if (test__start_subtest("cpu_flag_combination_percpu_array")) 821 test_percpu_array_flags_combination(); 822 } 823