1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Testsuite for eBPF maps 4 * 5 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com 6 * Copyright (c) 2016 Facebook 7 */ 8 9 #include <stdio.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <string.h> 13 #include <assert.h> 14 #include <stdlib.h> 15 #include <time.h> 16 17 #include <sys/wait.h> 18 #include <sys/socket.h> 19 #include <netinet/in.h> 20 #include <linux/bpf.h> 21 22 #include <bpf/bpf.h> 23 #include <bpf/libbpf.h> 24 25 #include "bpf_util.h" 26 #include "bpf_rlimit.h" 27 #include "test_maps.h" 28 #include "testing_helpers.h" 29 30 #ifndef ENOTSUPP 31 #define ENOTSUPP 524 32 #endif 33 34 static int skips; 35 36 static int map_flags; 37 38 static void test_hashmap(unsigned int task, void *data) 39 { 40 long long key, next_key, first_key, value; 41 int fd; 42 43 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 44 2, map_flags); 45 if (fd < 0) { 46 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 47 exit(1); 48 } 49 50 key = 1; 51 value = 1234; 52 /* Insert key=1 element. */ 53 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 54 55 value = 0; 56 /* BPF_NOEXIST means add new element if it doesn't exist. */ 57 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 58 /* key=1 already exists. */ 59 errno == EEXIST); 60 61 /* -1 is an invalid flag. */ 62 assert(bpf_map_update_elem(fd, &key, &value, -1) < 0 && 63 errno == EINVAL); 64 65 /* Check that key=1 can be found. */ 66 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234); 67 68 key = 2; 69 value = 1234; 70 /* Insert key=2 element. */ 71 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 72 73 /* Check that key=2 matches the value and delete it */ 74 assert(bpf_map_lookup_and_delete_elem(fd, &key, &value) == 0 && value == 1234); 75 76 /* Check that key=2 is not found. */ 77 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 78 79 /* BPF_EXIST means update existing element. */ 80 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) < 0 && 81 /* key=2 is not there. */ 82 errno == ENOENT); 83 84 /* Insert key=2 element. */ 85 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0); 86 87 /* key=1 and key=2 were inserted, check that key=0 cannot be 88 * inserted due to max_entries limit. 89 */ 90 key = 0; 91 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 92 errno == E2BIG); 93 94 /* Update existing element, though the map is full. */ 95 key = 1; 96 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0); 97 key = 2; 98 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 99 key = 3; 100 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 101 errno == E2BIG); 102 103 /* Check that key = 0 doesn't exist. */ 104 key = 0; 105 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 106 107 /* Iterate over two elements. */ 108 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 && 109 (first_key == 1 || first_key == 2)); 110 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 111 (next_key == first_key)); 112 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 113 (next_key == 1 || next_key == 2) && 114 (next_key != first_key)); 115 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 && 116 errno == ENOENT); 117 118 /* Delete both elements. */ 119 key = 1; 120 assert(bpf_map_delete_elem(fd, &key) == 0); 121 key = 2; 122 assert(bpf_map_delete_elem(fd, &key) == 0); 123 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 124 125 key = 0; 126 /* Check that map is empty. */ 127 assert(bpf_map_get_next_key(fd, NULL, &next_key) < 0 && 128 errno == ENOENT); 129 assert(bpf_map_get_next_key(fd, &key, &next_key) < 0 && 130 errno == ENOENT); 131 132 close(fd); 133 } 134 135 static void test_hashmap_sizes(unsigned int task, void *data) 136 { 137 int fd, i, j; 138 139 for (i = 1; i <= 512; i <<= 1) 140 for (j = 1; j <= 1 << 18; j <<= 1) { 141 fd = bpf_create_map(BPF_MAP_TYPE_HASH, i, j, 142 2, map_flags); 143 if (fd < 0) { 144 if (errno == ENOMEM) 145 return; 146 printf("Failed to create hashmap key=%d value=%d '%s'\n", 147 i, j, strerror(errno)); 148 exit(1); 149 } 150 close(fd); 151 usleep(10); /* give kernel time to destroy */ 152 } 153 } 154 155 static void test_hashmap_percpu(unsigned int task, void *data) 156 { 157 unsigned int nr_cpus = bpf_num_possible_cpus(); 158 BPF_DECLARE_PERCPU(long, value); 159 long long key, next_key, first_key; 160 int expected_key_mask = 0; 161 int fd, i; 162 163 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_HASH, sizeof(key), 164 sizeof(bpf_percpu(value, 0)), 2, map_flags); 165 if (fd < 0) { 166 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 167 exit(1); 168 } 169 170 for (i = 0; i < nr_cpus; i++) 171 bpf_percpu(value, i) = i + 100; 172 173 key = 1; 174 /* Insert key=1 element. */ 175 assert(!(expected_key_mask & key)); 176 assert(bpf_map_update_elem(fd, &key, value, BPF_ANY) == 0); 177 178 /* Lookup and delete elem key=1 and check value. */ 179 assert(bpf_map_lookup_and_delete_elem(fd, &key, value) == 0 && 180 bpf_percpu(value,0) == 100); 181 182 for (i = 0; i < nr_cpus; i++) 183 bpf_percpu(value,i) = i + 100; 184 185 /* Insert key=1 element which should not exist. */ 186 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0); 187 expected_key_mask |= key; 188 189 /* BPF_NOEXIST means add new element if it doesn't exist. */ 190 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) < 0 && 191 /* key=1 already exists. */ 192 errno == EEXIST); 193 194 /* -1 is an invalid flag. */ 195 assert(bpf_map_update_elem(fd, &key, value, -1) < 0 && 196 errno == EINVAL); 197 198 /* Check that key=1 can be found. Value could be 0 if the lookup 199 * was run from a different CPU. 200 */ 201 bpf_percpu(value, 0) = 1; 202 assert(bpf_map_lookup_elem(fd, &key, value) == 0 && 203 bpf_percpu(value, 0) == 100); 204 205 key = 2; 206 /* Check that key=2 is not found. */ 207 assert(bpf_map_lookup_elem(fd, &key, value) < 0 && errno == ENOENT); 208 209 /* BPF_EXIST means update existing element. */ 210 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) < 0 && 211 /* key=2 is not there. */ 212 errno == ENOENT); 213 214 /* Insert key=2 element. */ 215 assert(!(expected_key_mask & key)); 216 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0); 217 expected_key_mask |= key; 218 219 /* key=1 and key=2 were inserted, check that key=0 cannot be 220 * inserted due to max_entries limit. 221 */ 222 key = 0; 223 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) < 0 && 224 errno == E2BIG); 225 226 /* Check that key = 0 doesn't exist. */ 227 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 228 229 /* Iterate over two elements. */ 230 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 && 231 ((expected_key_mask & first_key) == first_key)); 232 while (!bpf_map_get_next_key(fd, &key, &next_key)) { 233 if (first_key) { 234 assert(next_key == first_key); 235 first_key = 0; 236 } 237 assert((expected_key_mask & next_key) == next_key); 238 expected_key_mask &= ~next_key; 239 240 assert(bpf_map_lookup_elem(fd, &next_key, value) == 0); 241 242 for (i = 0; i < nr_cpus; i++) 243 assert(bpf_percpu(value, i) == i + 100); 244 245 key = next_key; 246 } 247 assert(errno == ENOENT); 248 249 /* Update with BPF_EXIST. */ 250 key = 1; 251 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0); 252 253 /* Delete both elements. */ 254 key = 1; 255 assert(bpf_map_delete_elem(fd, &key) == 0); 256 key = 2; 257 assert(bpf_map_delete_elem(fd, &key) == 0); 258 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT); 259 260 key = 0; 261 /* Check that map is empty. */ 262 assert(bpf_map_get_next_key(fd, NULL, &next_key) < 0 && 263 errno == ENOENT); 264 assert(bpf_map_get_next_key(fd, &key, &next_key) < 0 && 265 errno == ENOENT); 266 267 close(fd); 268 } 269 270 static int helper_fill_hashmap(int max_entries) 271 { 272 int i, fd, ret; 273 long long key, value; 274 275 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 276 max_entries, map_flags); 277 CHECK(fd < 0, 278 "failed to create hashmap", 279 "err: %s, flags: 0x%x\n", strerror(errno), map_flags); 280 281 for (i = 0; i < max_entries; i++) { 282 key = i; value = key; 283 ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST); 284 CHECK(ret != 0, 285 "can't update hashmap", 286 "err: %s\n", strerror(ret)); 287 } 288 289 return fd; 290 } 291 292 static void test_hashmap_walk(unsigned int task, void *data) 293 { 294 int fd, i, max_entries = 1000; 295 long long key, value, next_key; 296 bool next_key_valid = true; 297 298 fd = helper_fill_hashmap(max_entries); 299 300 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, 301 &next_key) == 0; i++) { 302 key = next_key; 303 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 304 } 305 306 assert(i == max_entries); 307 308 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 309 for (i = 0; next_key_valid; i++) { 310 next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0; 311 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 312 value++; 313 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0); 314 key = next_key; 315 } 316 317 assert(i == max_entries); 318 319 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, 320 &next_key) == 0; i++) { 321 key = next_key; 322 assert(bpf_map_lookup_elem(fd, &key, &value) == 0); 323 assert(value - 1 == key); 324 } 325 326 assert(i == max_entries); 327 close(fd); 328 } 329 330 static void test_hashmap_zero_seed(void) 331 { 332 int i, first, second, old_flags; 333 long long key, next_first, next_second; 334 335 old_flags = map_flags; 336 map_flags |= BPF_F_ZERO_SEED; 337 338 first = helper_fill_hashmap(3); 339 second = helper_fill_hashmap(3); 340 341 for (i = 0; ; i++) { 342 void *key_ptr = !i ? NULL : &key; 343 344 if (bpf_map_get_next_key(first, key_ptr, &next_first) != 0) 345 break; 346 347 CHECK(bpf_map_get_next_key(second, key_ptr, &next_second) != 0, 348 "next_key for second map must succeed", 349 "key_ptr: %p", key_ptr); 350 CHECK(next_first != next_second, 351 "keys must match", 352 "i: %d first: %lld second: %lld\n", i, 353 next_first, next_second); 354 355 key = next_first; 356 } 357 358 map_flags = old_flags; 359 close(first); 360 close(second); 361 } 362 363 static void test_arraymap(unsigned int task, void *data) 364 { 365 int key, next_key, fd; 366 long long value; 367 368 fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(key), sizeof(value), 369 2, 0); 370 if (fd < 0) { 371 printf("Failed to create arraymap '%s'!\n", strerror(errno)); 372 exit(1); 373 } 374 375 key = 1; 376 value = 1234; 377 /* Insert key=1 element. */ 378 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 379 380 value = 0; 381 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 382 errno == EEXIST); 383 384 /* Check that key=1 can be found. */ 385 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234); 386 387 key = 0; 388 /* Check that key=0 is also found and zero initialized. */ 389 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0); 390 391 /* key=0 and key=1 were inserted, check that key=2 cannot be inserted 392 * due to max_entries limit. 393 */ 394 key = 2; 395 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) < 0 && 396 errno == E2BIG); 397 398 /* Check that key = 2 doesn't exist. */ 399 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 400 401 /* Iterate over two elements. */ 402 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 && 403 next_key == 0); 404 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 405 next_key == 0); 406 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 407 next_key == 1); 408 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 && 409 errno == ENOENT); 410 411 /* Delete shouldn't succeed. */ 412 key = 1; 413 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == EINVAL); 414 415 close(fd); 416 } 417 418 static void test_arraymap_percpu(unsigned int task, void *data) 419 { 420 unsigned int nr_cpus = bpf_num_possible_cpus(); 421 BPF_DECLARE_PERCPU(long, values); 422 int key, next_key, fd, i; 423 424 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), 425 sizeof(bpf_percpu(values, 0)), 2, 0); 426 if (fd < 0) { 427 printf("Failed to create arraymap '%s'!\n", strerror(errno)); 428 exit(1); 429 } 430 431 for (i = 0; i < nr_cpus; i++) 432 bpf_percpu(values, i) = i + 100; 433 434 key = 1; 435 /* Insert key=1 element. */ 436 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0); 437 438 bpf_percpu(values, 0) = 0; 439 assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) < 0 && 440 errno == EEXIST); 441 442 /* Check that key=1 can be found. */ 443 assert(bpf_map_lookup_elem(fd, &key, values) == 0 && 444 bpf_percpu(values, 0) == 100); 445 446 key = 0; 447 /* Check that key=0 is also found and zero initialized. */ 448 assert(bpf_map_lookup_elem(fd, &key, values) == 0 && 449 bpf_percpu(values, 0) == 0 && 450 bpf_percpu(values, nr_cpus - 1) == 0); 451 452 /* Check that key=2 cannot be inserted due to max_entries limit. */ 453 key = 2; 454 assert(bpf_map_update_elem(fd, &key, values, BPF_EXIST) < 0 && 455 errno == E2BIG); 456 457 /* Check that key = 2 doesn't exist. */ 458 assert(bpf_map_lookup_elem(fd, &key, values) < 0 && errno == ENOENT); 459 460 /* Iterate over two elements. */ 461 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 && 462 next_key == 0); 463 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 && 464 next_key == 0); 465 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 && 466 next_key == 1); 467 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 && 468 errno == ENOENT); 469 470 /* Delete shouldn't succeed. */ 471 key = 1; 472 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == EINVAL); 473 474 close(fd); 475 } 476 477 static void test_arraymap_percpu_many_keys(void) 478 { 479 unsigned int nr_cpus = bpf_num_possible_cpus(); 480 BPF_DECLARE_PERCPU(long, values); 481 /* nr_keys is not too large otherwise the test stresses percpu 482 * allocator more than anything else 483 */ 484 unsigned int nr_keys = 2000; 485 int key, fd, i; 486 487 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), 488 sizeof(bpf_percpu(values, 0)), nr_keys, 0); 489 if (fd < 0) { 490 printf("Failed to create per-cpu arraymap '%s'!\n", 491 strerror(errno)); 492 exit(1); 493 } 494 495 for (i = 0; i < nr_cpus; i++) 496 bpf_percpu(values, i) = i + 10; 497 498 for (key = 0; key < nr_keys; key++) 499 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0); 500 501 for (key = 0; key < nr_keys; key++) { 502 for (i = 0; i < nr_cpus; i++) 503 bpf_percpu(values, i) = 0; 504 505 assert(bpf_map_lookup_elem(fd, &key, values) == 0); 506 507 for (i = 0; i < nr_cpus; i++) 508 assert(bpf_percpu(values, i) == i + 10); 509 } 510 511 close(fd); 512 } 513 514 static void test_devmap(unsigned int task, void *data) 515 { 516 int fd; 517 __u32 key, value; 518 519 fd = bpf_create_map(BPF_MAP_TYPE_DEVMAP, sizeof(key), sizeof(value), 520 2, 0); 521 if (fd < 0) { 522 printf("Failed to create devmap '%s'!\n", strerror(errno)); 523 exit(1); 524 } 525 526 close(fd); 527 } 528 529 static void test_devmap_hash(unsigned int task, void *data) 530 { 531 int fd; 532 __u32 key, value; 533 534 fd = bpf_create_map(BPF_MAP_TYPE_DEVMAP_HASH, sizeof(key), sizeof(value), 535 2, 0); 536 if (fd < 0) { 537 printf("Failed to create devmap_hash '%s'!\n", strerror(errno)); 538 exit(1); 539 } 540 541 close(fd); 542 } 543 544 static void test_queuemap(unsigned int task, void *data) 545 { 546 const int MAP_SIZE = 32; 547 __u32 vals[MAP_SIZE + MAP_SIZE/2], val; 548 int fd, i; 549 550 /* Fill test values to be used */ 551 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++) 552 vals[i] = rand(); 553 554 /* Invalid key size */ 555 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 4, sizeof(val), MAP_SIZE, 556 map_flags); 557 assert(fd < 0 && errno == EINVAL); 558 559 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 0, sizeof(val), MAP_SIZE, 560 map_flags); 561 /* Queue map does not support BPF_F_NO_PREALLOC */ 562 if (map_flags & BPF_F_NO_PREALLOC) { 563 assert(fd < 0 && errno == EINVAL); 564 return; 565 } 566 if (fd < 0) { 567 printf("Failed to create queuemap '%s'!\n", strerror(errno)); 568 exit(1); 569 } 570 571 /* Push MAP_SIZE elements */ 572 for (i = 0; i < MAP_SIZE; i++) 573 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0); 574 575 /* Check that element cannot be pushed due to max_entries limit */ 576 assert(bpf_map_update_elem(fd, NULL, &val, 0) < 0 && 577 errno == E2BIG); 578 579 /* Peek element */ 580 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[0]); 581 582 /* Replace half elements */ 583 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++) 584 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0); 585 586 /* Pop all elements */ 587 for (i = MAP_SIZE/2; i < MAP_SIZE + MAP_SIZE/2; i++) 588 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 && 589 val == vals[i]); 590 591 /* Check that there are not elements left */ 592 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) < 0 && 593 errno == ENOENT); 594 595 /* Check that non supported functions set errno to EINVAL */ 596 assert(bpf_map_delete_elem(fd, NULL) < 0 && errno == EINVAL); 597 assert(bpf_map_get_next_key(fd, NULL, NULL) < 0 && errno == EINVAL); 598 599 close(fd); 600 } 601 602 static void test_stackmap(unsigned int task, void *data) 603 { 604 const int MAP_SIZE = 32; 605 __u32 vals[MAP_SIZE + MAP_SIZE/2], val; 606 int fd, i; 607 608 /* Fill test values to be used */ 609 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++) 610 vals[i] = rand(); 611 612 /* Invalid key size */ 613 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 4, sizeof(val), MAP_SIZE, 614 map_flags); 615 assert(fd < 0 && errno == EINVAL); 616 617 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 0, sizeof(val), MAP_SIZE, 618 map_flags); 619 /* Stack map does not support BPF_F_NO_PREALLOC */ 620 if (map_flags & BPF_F_NO_PREALLOC) { 621 assert(fd < 0 && errno == EINVAL); 622 return; 623 } 624 if (fd < 0) { 625 printf("Failed to create stackmap '%s'!\n", strerror(errno)); 626 exit(1); 627 } 628 629 /* Push MAP_SIZE elements */ 630 for (i = 0; i < MAP_SIZE; i++) 631 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0); 632 633 /* Check that element cannot be pushed due to max_entries limit */ 634 assert(bpf_map_update_elem(fd, NULL, &val, 0) < 0 && 635 errno == E2BIG); 636 637 /* Peek element */ 638 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[i - 1]); 639 640 /* Replace half elements */ 641 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++) 642 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0); 643 644 /* Pop all elements */ 645 for (i = MAP_SIZE + MAP_SIZE/2 - 1; i >= MAP_SIZE/2; i--) 646 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 && 647 val == vals[i]); 648 649 /* Check that there are not elements left */ 650 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) < 0 && 651 errno == ENOENT); 652 653 /* Check that non supported functions set errno to EINVAL */ 654 assert(bpf_map_delete_elem(fd, NULL) < 0 && errno == EINVAL); 655 assert(bpf_map_get_next_key(fd, NULL, NULL) < 0 && errno == EINVAL); 656 657 close(fd); 658 } 659 660 #include <sys/ioctl.h> 661 #include <arpa/inet.h> 662 #include <sys/select.h> 663 #include <linux/err.h> 664 #define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.o" 665 #define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o" 666 #define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.o" 667 static void test_sockmap(unsigned int tasks, void *data) 668 { 669 struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break; 670 int map_fd_msg = 0, map_fd_rx = 0, map_fd_tx = 0, map_fd_break; 671 int ports[] = {50200, 50201, 50202, 50204}; 672 int err, i, fd, udp, sfd[6] = {0xdeadbeef}; 673 u8 buf[20] = {0x0, 0x5, 0x3, 0x2, 0x1, 0x0}; 674 int parse_prog, verdict_prog, msg_prog; 675 struct sockaddr_in addr; 676 int one = 1, s, sc, rc; 677 struct bpf_object *obj; 678 struct timeval to; 679 __u32 key, value; 680 pid_t pid[tasks]; 681 fd_set w; 682 683 /* Create some sockets to use with sockmap */ 684 for (i = 0; i < 2; i++) { 685 sfd[i] = socket(AF_INET, SOCK_STREAM, 0); 686 if (sfd[i] < 0) 687 goto out; 688 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR, 689 (char *)&one, sizeof(one)); 690 if (err) { 691 printf("failed to setsockopt\n"); 692 goto out; 693 } 694 err = ioctl(sfd[i], FIONBIO, (char *)&one); 695 if (err < 0) { 696 printf("failed to ioctl\n"); 697 goto out; 698 } 699 memset(&addr, 0, sizeof(struct sockaddr_in)); 700 addr.sin_family = AF_INET; 701 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 702 addr.sin_port = htons(ports[i]); 703 err = bind(sfd[i], (struct sockaddr *)&addr, sizeof(addr)); 704 if (err < 0) { 705 printf("failed to bind: err %i: %i:%i\n", 706 err, i, sfd[i]); 707 goto out; 708 } 709 err = listen(sfd[i], 32); 710 if (err < 0) { 711 printf("failed to listen\n"); 712 goto out; 713 } 714 } 715 716 for (i = 2; i < 4; i++) { 717 sfd[i] = socket(AF_INET, SOCK_STREAM, 0); 718 if (sfd[i] < 0) 719 goto out; 720 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR, 721 (char *)&one, sizeof(one)); 722 if (err) { 723 printf("set sock opt\n"); 724 goto out; 725 } 726 memset(&addr, 0, sizeof(struct sockaddr_in)); 727 addr.sin_family = AF_INET; 728 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 729 addr.sin_port = htons(ports[i - 2]); 730 err = connect(sfd[i], (struct sockaddr *)&addr, sizeof(addr)); 731 if (err) { 732 printf("failed to connect\n"); 733 goto out; 734 } 735 } 736 737 738 for (i = 4; i < 6; i++) { 739 sfd[i] = accept(sfd[i - 4], NULL, NULL); 740 if (sfd[i] < 0) { 741 printf("accept failed\n"); 742 goto out; 743 } 744 } 745 746 /* Test sockmap with connected sockets */ 747 fd = bpf_create_map(BPF_MAP_TYPE_SOCKMAP, 748 sizeof(key), sizeof(value), 749 6, 0); 750 if (fd < 0) { 751 if (!bpf_probe_map_type(BPF_MAP_TYPE_SOCKMAP, 0)) { 752 printf("%s SKIP (unsupported map type BPF_MAP_TYPE_SOCKMAP)\n", 753 __func__); 754 skips++; 755 for (i = 0; i < 6; i++) 756 close(sfd[i]); 757 return; 758 } 759 760 printf("Failed to create sockmap %i\n", fd); 761 goto out_sockmap; 762 } 763 764 /* Test update with unsupported UDP socket */ 765 udp = socket(AF_INET, SOCK_DGRAM, 0); 766 i = 0; 767 err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY); 768 if (err) { 769 printf("Failed socket update SOCK_DGRAM '%i:%i'\n", 770 i, udp); 771 goto out_sockmap; 772 } 773 774 /* Test update without programs */ 775 for (i = 0; i < 6; i++) { 776 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 777 if (err) { 778 printf("Failed noprog update sockmap '%i:%i'\n", 779 i, sfd[i]); 780 goto out_sockmap; 781 } 782 } 783 784 /* Test attaching/detaching bad fds */ 785 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_PARSER, 0); 786 if (!err) { 787 printf("Failed invalid parser prog attach\n"); 788 goto out_sockmap; 789 } 790 791 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_VERDICT, 0); 792 if (!err) { 793 printf("Failed invalid verdict prog attach\n"); 794 goto out_sockmap; 795 } 796 797 err = bpf_prog_attach(-1, fd, BPF_SK_MSG_VERDICT, 0); 798 if (!err) { 799 printf("Failed invalid msg verdict prog attach\n"); 800 goto out_sockmap; 801 } 802 803 err = bpf_prog_attach(-1, fd, __MAX_BPF_ATTACH_TYPE, 0); 804 if (!err) { 805 printf("Failed unknown prog attach\n"); 806 goto out_sockmap; 807 } 808 809 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_PARSER); 810 if (!err) { 811 printf("Failed empty parser prog detach\n"); 812 goto out_sockmap; 813 } 814 815 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_VERDICT); 816 if (!err) { 817 printf("Failed empty verdict prog detach\n"); 818 goto out_sockmap; 819 } 820 821 err = bpf_prog_detach(fd, BPF_SK_MSG_VERDICT); 822 if (!err) { 823 printf("Failed empty msg verdict prog detach\n"); 824 goto out_sockmap; 825 } 826 827 err = bpf_prog_detach(fd, __MAX_BPF_ATTACH_TYPE); 828 if (!err) { 829 printf("Detach invalid prog successful\n"); 830 goto out_sockmap; 831 } 832 833 /* Load SK_SKB program and Attach */ 834 err = bpf_prog_test_load(SOCKMAP_PARSE_PROG, 835 BPF_PROG_TYPE_SK_SKB, &obj, &parse_prog); 836 if (err) { 837 printf("Failed to load SK_SKB parse prog\n"); 838 goto out_sockmap; 839 } 840 841 err = bpf_prog_test_load(SOCKMAP_TCP_MSG_PROG, 842 BPF_PROG_TYPE_SK_MSG, &obj, &msg_prog); 843 if (err) { 844 printf("Failed to load SK_SKB msg prog\n"); 845 goto out_sockmap; 846 } 847 848 err = bpf_prog_test_load(SOCKMAP_VERDICT_PROG, 849 BPF_PROG_TYPE_SK_SKB, &obj, &verdict_prog); 850 if (err) { 851 printf("Failed to load SK_SKB verdict prog\n"); 852 goto out_sockmap; 853 } 854 855 bpf_map_rx = bpf_object__find_map_by_name(obj, "sock_map_rx"); 856 if (!bpf_map_rx) { 857 printf("Failed to load map rx from verdict prog\n"); 858 goto out_sockmap; 859 } 860 861 map_fd_rx = bpf_map__fd(bpf_map_rx); 862 if (map_fd_rx < 0) { 863 printf("Failed to get map rx fd\n"); 864 goto out_sockmap; 865 } 866 867 bpf_map_tx = bpf_object__find_map_by_name(obj, "sock_map_tx"); 868 if (!bpf_map_tx) { 869 printf("Failed to load map tx from verdict prog\n"); 870 goto out_sockmap; 871 } 872 873 map_fd_tx = bpf_map__fd(bpf_map_tx); 874 if (map_fd_tx < 0) { 875 printf("Failed to get map tx fd\n"); 876 goto out_sockmap; 877 } 878 879 bpf_map_msg = bpf_object__find_map_by_name(obj, "sock_map_msg"); 880 if (!bpf_map_msg) { 881 printf("Failed to load map msg from msg_verdict prog\n"); 882 goto out_sockmap; 883 } 884 885 map_fd_msg = bpf_map__fd(bpf_map_msg); 886 if (map_fd_msg < 0) { 887 printf("Failed to get map msg fd\n"); 888 goto out_sockmap; 889 } 890 891 bpf_map_break = bpf_object__find_map_by_name(obj, "sock_map_break"); 892 if (!bpf_map_break) { 893 printf("Failed to load map tx from verdict prog\n"); 894 goto out_sockmap; 895 } 896 897 map_fd_break = bpf_map__fd(bpf_map_break); 898 if (map_fd_break < 0) { 899 printf("Failed to get map tx fd\n"); 900 goto out_sockmap; 901 } 902 903 err = bpf_prog_attach(parse_prog, map_fd_break, 904 BPF_SK_SKB_STREAM_PARSER, 0); 905 if (!err) { 906 printf("Allowed attaching SK_SKB program to invalid map\n"); 907 goto out_sockmap; 908 } 909 910 err = bpf_prog_attach(parse_prog, map_fd_rx, 911 BPF_SK_SKB_STREAM_PARSER, 0); 912 if (err) { 913 printf("Failed stream parser bpf prog attach\n"); 914 goto out_sockmap; 915 } 916 917 err = bpf_prog_attach(verdict_prog, map_fd_rx, 918 BPF_SK_SKB_STREAM_VERDICT, 0); 919 if (err) { 920 printf("Failed stream verdict bpf prog attach\n"); 921 goto out_sockmap; 922 } 923 924 err = bpf_prog_attach(msg_prog, map_fd_msg, BPF_SK_MSG_VERDICT, 0); 925 if (err) { 926 printf("Failed msg verdict bpf prog attach\n"); 927 goto out_sockmap; 928 } 929 930 err = bpf_prog_attach(verdict_prog, map_fd_rx, 931 __MAX_BPF_ATTACH_TYPE, 0); 932 if (!err) { 933 printf("Attached unknown bpf prog\n"); 934 goto out_sockmap; 935 } 936 937 /* Test map update elem afterwards fd lives in fd and map_fd */ 938 for (i = 2; i < 6; i++) { 939 err = bpf_map_update_elem(map_fd_rx, &i, &sfd[i], BPF_ANY); 940 if (err) { 941 printf("Failed map_fd_rx update sockmap %i '%i:%i'\n", 942 err, i, sfd[i]); 943 goto out_sockmap; 944 } 945 err = bpf_map_update_elem(map_fd_tx, &i, &sfd[i], BPF_ANY); 946 if (err) { 947 printf("Failed map_fd_tx update sockmap %i '%i:%i'\n", 948 err, i, sfd[i]); 949 goto out_sockmap; 950 } 951 } 952 953 /* Test map delete elem and remove send/recv sockets */ 954 for (i = 2; i < 4; i++) { 955 err = bpf_map_delete_elem(map_fd_rx, &i); 956 if (err) { 957 printf("Failed delete sockmap rx %i '%i:%i'\n", 958 err, i, sfd[i]); 959 goto out_sockmap; 960 } 961 err = bpf_map_delete_elem(map_fd_tx, &i); 962 if (err) { 963 printf("Failed delete sockmap tx %i '%i:%i'\n", 964 err, i, sfd[i]); 965 goto out_sockmap; 966 } 967 } 968 969 /* Put sfd[2] (sending fd below) into msg map to test sendmsg bpf */ 970 i = 0; 971 err = bpf_map_update_elem(map_fd_msg, &i, &sfd[2], BPF_ANY); 972 if (err) { 973 printf("Failed map_fd_msg update sockmap %i\n", err); 974 goto out_sockmap; 975 } 976 977 /* Test map send/recv */ 978 for (i = 0; i < 2; i++) { 979 buf[0] = i; 980 buf[1] = 0x5; 981 sc = send(sfd[2], buf, 20, 0); 982 if (sc < 0) { 983 printf("Failed sockmap send\n"); 984 goto out_sockmap; 985 } 986 987 FD_ZERO(&w); 988 FD_SET(sfd[3], &w); 989 to.tv_sec = 30; 990 to.tv_usec = 0; 991 s = select(sfd[3] + 1, &w, NULL, NULL, &to); 992 if (s == -1) { 993 perror("Failed sockmap select()"); 994 goto out_sockmap; 995 } else if (!s) { 996 printf("Failed sockmap unexpected timeout\n"); 997 goto out_sockmap; 998 } 999 1000 if (!FD_ISSET(sfd[3], &w)) { 1001 printf("Failed sockmap select/recv\n"); 1002 goto out_sockmap; 1003 } 1004 1005 rc = recv(sfd[3], buf, sizeof(buf), 0); 1006 if (rc < 0) { 1007 printf("Failed sockmap recv\n"); 1008 goto out_sockmap; 1009 } 1010 } 1011 1012 /* Negative null entry lookup from datapath should be dropped */ 1013 buf[0] = 1; 1014 buf[1] = 12; 1015 sc = send(sfd[2], buf, 20, 0); 1016 if (sc < 0) { 1017 printf("Failed sockmap send\n"); 1018 goto out_sockmap; 1019 } 1020 1021 /* Push fd into same slot */ 1022 i = 2; 1023 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST); 1024 if (!err) { 1025 printf("Failed allowed sockmap dup slot BPF_NOEXIST\n"); 1026 goto out_sockmap; 1027 } 1028 1029 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 1030 if (err) { 1031 printf("Failed sockmap update new slot BPF_ANY\n"); 1032 goto out_sockmap; 1033 } 1034 1035 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST); 1036 if (err) { 1037 printf("Failed sockmap update new slot BPF_EXIST\n"); 1038 goto out_sockmap; 1039 } 1040 1041 /* Delete the elems without programs */ 1042 for (i = 2; i < 6; i++) { 1043 err = bpf_map_delete_elem(fd, &i); 1044 if (err) { 1045 printf("Failed delete sockmap %i '%i:%i'\n", 1046 err, i, sfd[i]); 1047 } 1048 } 1049 1050 /* Test having multiple maps open and set with programs on same fds */ 1051 err = bpf_prog_attach(parse_prog, fd, 1052 BPF_SK_SKB_STREAM_PARSER, 0); 1053 if (err) { 1054 printf("Failed fd bpf parse prog attach\n"); 1055 goto out_sockmap; 1056 } 1057 err = bpf_prog_attach(verdict_prog, fd, 1058 BPF_SK_SKB_STREAM_VERDICT, 0); 1059 if (err) { 1060 printf("Failed fd bpf verdict prog attach\n"); 1061 goto out_sockmap; 1062 } 1063 1064 for (i = 4; i < 6; i++) { 1065 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY); 1066 if (!err) { 1067 printf("Failed allowed duplicate programs in update ANY sockmap %i '%i:%i'\n", 1068 err, i, sfd[i]); 1069 goto out_sockmap; 1070 } 1071 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST); 1072 if (!err) { 1073 printf("Failed allowed duplicate program in update NOEXIST sockmap %i '%i:%i'\n", 1074 err, i, sfd[i]); 1075 goto out_sockmap; 1076 } 1077 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST); 1078 if (!err) { 1079 printf("Failed allowed duplicate program in update EXIST sockmap %i '%i:%i'\n", 1080 err, i, sfd[i]); 1081 goto out_sockmap; 1082 } 1083 } 1084 1085 /* Test tasks number of forked operations */ 1086 for (i = 0; i < tasks; i++) { 1087 pid[i] = fork(); 1088 if (pid[i] == 0) { 1089 for (i = 0; i < 6; i++) { 1090 bpf_map_delete_elem(map_fd_tx, &i); 1091 bpf_map_delete_elem(map_fd_rx, &i); 1092 bpf_map_update_elem(map_fd_tx, &i, 1093 &sfd[i], BPF_ANY); 1094 bpf_map_update_elem(map_fd_rx, &i, 1095 &sfd[i], BPF_ANY); 1096 } 1097 exit(0); 1098 } else if (pid[i] == -1) { 1099 printf("Couldn't spawn #%d process!\n", i); 1100 exit(1); 1101 } 1102 } 1103 1104 for (i = 0; i < tasks; i++) { 1105 int status; 1106 1107 assert(waitpid(pid[i], &status, 0) == pid[i]); 1108 assert(status == 0); 1109 } 1110 1111 err = bpf_prog_detach2(parse_prog, map_fd_rx, __MAX_BPF_ATTACH_TYPE); 1112 if (!err) { 1113 printf("Detached an invalid prog type.\n"); 1114 goto out_sockmap; 1115 } 1116 1117 err = bpf_prog_detach2(parse_prog, map_fd_rx, BPF_SK_SKB_STREAM_PARSER); 1118 if (err) { 1119 printf("Failed parser prog detach\n"); 1120 goto out_sockmap; 1121 } 1122 1123 err = bpf_prog_detach2(verdict_prog, map_fd_rx, BPF_SK_SKB_STREAM_VERDICT); 1124 if (err) { 1125 printf("Failed parser prog detach\n"); 1126 goto out_sockmap; 1127 } 1128 1129 /* Test map close sockets and empty maps */ 1130 for (i = 0; i < 6; i++) { 1131 bpf_map_delete_elem(map_fd_tx, &i); 1132 bpf_map_delete_elem(map_fd_rx, &i); 1133 close(sfd[i]); 1134 } 1135 close(fd); 1136 close(map_fd_rx); 1137 bpf_object__close(obj); 1138 return; 1139 out: 1140 for (i = 0; i < 6; i++) 1141 close(sfd[i]); 1142 printf("Failed to create sockmap '%i:%s'!\n", i, strerror(errno)); 1143 exit(1); 1144 out_sockmap: 1145 for (i = 0; i < 6; i++) { 1146 if (map_fd_tx) 1147 bpf_map_delete_elem(map_fd_tx, &i); 1148 if (map_fd_rx) 1149 bpf_map_delete_elem(map_fd_rx, &i); 1150 close(sfd[i]); 1151 } 1152 close(fd); 1153 exit(1); 1154 } 1155 1156 #define MAPINMAP_PROG "./test_map_in_map.o" 1157 #define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.o" 1158 static void test_map_in_map(void) 1159 { 1160 struct bpf_object *obj; 1161 struct bpf_map *map; 1162 int mim_fd, fd, err; 1163 int pos = 0; 1164 struct bpf_map_info info = {}; 1165 __u32 len = sizeof(info); 1166 __u32 id = 0; 1167 libbpf_print_fn_t old_print_fn; 1168 1169 obj = bpf_object__open(MAPINMAP_PROG); 1170 1171 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(int), sizeof(int), 1172 2, 0); 1173 if (fd < 0) { 1174 printf("Failed to create hashmap '%s'!\n", strerror(errno)); 1175 exit(1); 1176 } 1177 1178 map = bpf_object__find_map_by_name(obj, "mim_array"); 1179 if (!map) { 1180 printf("Failed to load array of maps from test prog\n"); 1181 goto out_map_in_map; 1182 } 1183 err = bpf_map__set_inner_map_fd(map, fd); 1184 if (err) { 1185 printf("Failed to set inner_map_fd for array of maps\n"); 1186 goto out_map_in_map; 1187 } 1188 1189 map = bpf_object__find_map_by_name(obj, "mim_hash"); 1190 if (!map) { 1191 printf("Failed to load hash of maps from test prog\n"); 1192 goto out_map_in_map; 1193 } 1194 err = bpf_map__set_inner_map_fd(map, fd); 1195 if (err) { 1196 printf("Failed to set inner_map_fd for hash of maps\n"); 1197 goto out_map_in_map; 1198 } 1199 1200 bpf_object__load(obj); 1201 1202 map = bpf_object__find_map_by_name(obj, "mim_array"); 1203 if (!map) { 1204 printf("Failed to load array of maps from test prog\n"); 1205 goto out_map_in_map; 1206 } 1207 mim_fd = bpf_map__fd(map); 1208 if (mim_fd < 0) { 1209 printf("Failed to get descriptor for array of maps\n"); 1210 goto out_map_in_map; 1211 } 1212 1213 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); 1214 if (err) { 1215 printf("Failed to update array of maps\n"); 1216 goto out_map_in_map; 1217 } 1218 1219 map = bpf_object__find_map_by_name(obj, "mim_hash"); 1220 if (!map) { 1221 printf("Failed to load hash of maps from test prog\n"); 1222 goto out_map_in_map; 1223 } 1224 mim_fd = bpf_map__fd(map); 1225 if (mim_fd < 0) { 1226 printf("Failed to get descriptor for hash of maps\n"); 1227 goto out_map_in_map; 1228 } 1229 1230 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); 1231 if (err) { 1232 printf("Failed to update hash of maps\n"); 1233 goto out_map_in_map; 1234 } 1235 1236 close(fd); 1237 fd = -1; 1238 bpf_object__close(obj); 1239 1240 /* Test that failing bpf_object__create_map() destroys the inner map */ 1241 obj = bpf_object__open(MAPINMAP_INVALID_PROG); 1242 err = libbpf_get_error(obj); 1243 if (err) { 1244 printf("Failed to load %s program: %d %d", 1245 MAPINMAP_INVALID_PROG, err, errno); 1246 goto out_map_in_map; 1247 } 1248 1249 map = bpf_object__find_map_by_name(obj, "mim"); 1250 if (!map) { 1251 printf("Failed to load array of maps from test prog\n"); 1252 goto out_map_in_map; 1253 } 1254 1255 old_print_fn = libbpf_set_print(NULL); 1256 1257 err = bpf_object__load(obj); 1258 if (!err) { 1259 printf("Loading obj supposed to fail\n"); 1260 goto out_map_in_map; 1261 } 1262 1263 libbpf_set_print(old_print_fn); 1264 1265 /* Iterate over all maps to check whether the internal map 1266 * ("mim.internal") has been destroyed. 1267 */ 1268 while (true) { 1269 err = bpf_map_get_next_id(id, &id); 1270 if (err) { 1271 if (errno == ENOENT) 1272 break; 1273 printf("Failed to get next map: %d", errno); 1274 goto out_map_in_map; 1275 } 1276 1277 fd = bpf_map_get_fd_by_id(id); 1278 if (fd < 0) { 1279 if (errno == ENOENT) 1280 continue; 1281 printf("Failed to get map by id %u: %d", id, errno); 1282 goto out_map_in_map; 1283 } 1284 1285 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1286 if (err) { 1287 printf("Failed to get map info by fd %d: %d", fd, 1288 errno); 1289 goto out_map_in_map; 1290 } 1291 1292 if (!strcmp(info.name, "mim.inner")) { 1293 printf("Inner map mim.inner was not destroyed\n"); 1294 goto out_map_in_map; 1295 } 1296 } 1297 1298 return; 1299 1300 out_map_in_map: 1301 if (fd >= 0) 1302 close(fd); 1303 exit(1); 1304 } 1305 1306 #define MAP_SIZE (32 * 1024) 1307 1308 static void test_map_large(void) 1309 { 1310 1311 struct bigkey { 1312 int a; 1313 char b[4096]; 1314 long long c; 1315 } key; 1316 int fd, i, value; 1317 1318 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1319 MAP_SIZE, map_flags); 1320 if (fd < 0) { 1321 printf("Failed to create large map '%s'!\n", strerror(errno)); 1322 exit(1); 1323 } 1324 1325 for (i = 0; i < MAP_SIZE; i++) { 1326 key = (struct bigkey) { .c = i }; 1327 value = i; 1328 1329 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0); 1330 } 1331 1332 key.c = -1; 1333 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 1334 errno == E2BIG); 1335 1336 /* Iterate through all elements. */ 1337 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 1338 key.c = -1; 1339 for (i = 0; i < MAP_SIZE; i++) 1340 assert(bpf_map_get_next_key(fd, &key, &key) == 0); 1341 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT); 1342 1343 key.c = 0; 1344 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0); 1345 key.a = 1; 1346 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 1347 1348 close(fd); 1349 } 1350 1351 #define run_parallel(N, FN, DATA) \ 1352 printf("Fork %u tasks to '" #FN "'\n", N); \ 1353 __run_parallel(N, FN, DATA) 1354 1355 static void __run_parallel(unsigned int tasks, 1356 void (*fn)(unsigned int task, void *data), 1357 void *data) 1358 { 1359 pid_t pid[tasks]; 1360 int i; 1361 1362 fflush(stdout); 1363 1364 for (i = 0; i < tasks; i++) { 1365 pid[i] = fork(); 1366 if (pid[i] == 0) { 1367 fn(i, data); 1368 exit(0); 1369 } else if (pid[i] == -1) { 1370 printf("Couldn't spawn #%d process!\n", i); 1371 exit(1); 1372 } 1373 } 1374 1375 for (i = 0; i < tasks; i++) { 1376 int status; 1377 1378 assert(waitpid(pid[i], &status, 0) == pid[i]); 1379 assert(status == 0); 1380 } 1381 } 1382 1383 static void test_map_stress(void) 1384 { 1385 run_parallel(100, test_hashmap, NULL); 1386 run_parallel(100, test_hashmap_percpu, NULL); 1387 run_parallel(100, test_hashmap_sizes, NULL); 1388 run_parallel(100, test_hashmap_walk, NULL); 1389 1390 run_parallel(100, test_arraymap, NULL); 1391 run_parallel(100, test_arraymap_percpu, NULL); 1392 } 1393 1394 #define TASKS 1024 1395 1396 #define DO_UPDATE 1 1397 #define DO_DELETE 0 1398 1399 #define MAP_RETRIES 20 1400 #define MAX_DELAY_US 50000 1401 #define MIN_DELAY_RANGE_US 5000 1402 1403 static int map_update_retriable(int map_fd, const void *key, const void *value, 1404 int flags, int attempts) 1405 { 1406 int delay = rand() % MIN_DELAY_RANGE_US; 1407 1408 while (bpf_map_update_elem(map_fd, key, value, flags)) { 1409 if (!attempts || (errno != EAGAIN && errno != EBUSY)) 1410 return -errno; 1411 1412 if (delay <= MAX_DELAY_US / 2) 1413 delay *= 2; 1414 1415 usleep(delay); 1416 attempts--; 1417 } 1418 1419 return 0; 1420 } 1421 1422 static int map_delete_retriable(int map_fd, const void *key, int attempts) 1423 { 1424 int delay = rand() % MIN_DELAY_RANGE_US; 1425 1426 while (bpf_map_delete_elem(map_fd, key)) { 1427 if (!attempts || (errno != EAGAIN && errno != EBUSY)) 1428 return -errno; 1429 1430 if (delay <= MAX_DELAY_US / 2) 1431 delay *= 2; 1432 1433 usleep(delay); 1434 attempts--; 1435 } 1436 1437 return 0; 1438 } 1439 1440 static void test_update_delete(unsigned int fn, void *data) 1441 { 1442 int do_update = ((int *)data)[1]; 1443 int fd = ((int *)data)[0]; 1444 int i, key, value, err; 1445 1446 for (i = fn; i < MAP_SIZE; i += TASKS) { 1447 key = value = i; 1448 1449 if (do_update) { 1450 err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES); 1451 if (err) 1452 printf("error %d %d\n", err, errno); 1453 assert(err == 0); 1454 err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES); 1455 if (err) 1456 printf("error %d %d\n", err, errno); 1457 assert(err == 0); 1458 } else { 1459 err = map_delete_retriable(fd, &key, MAP_RETRIES); 1460 if (err) 1461 printf("error %d %d\n", err, errno); 1462 assert(err == 0); 1463 } 1464 } 1465 } 1466 1467 static void test_map_parallel(void) 1468 { 1469 int i, fd, key = 0, value = 0; 1470 int data[2]; 1471 1472 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1473 MAP_SIZE, map_flags); 1474 if (fd < 0) { 1475 printf("Failed to create map for parallel test '%s'!\n", 1476 strerror(errno)); 1477 exit(1); 1478 } 1479 1480 /* Use the same fd in children to add elements to this map: 1481 * child_0 adds key=0, key=1024, key=2048, ... 1482 * child_1 adds key=1, key=1025, key=2049, ... 1483 * child_1023 adds key=1023, ... 1484 */ 1485 data[0] = fd; 1486 data[1] = DO_UPDATE; 1487 run_parallel(TASKS, test_update_delete, data); 1488 1489 /* Check that key=0 is already there. */ 1490 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 && 1491 errno == EEXIST); 1492 1493 /* Check that all elements were inserted. */ 1494 assert(bpf_map_get_next_key(fd, NULL, &key) == 0); 1495 key = -1; 1496 for (i = 0; i < MAP_SIZE; i++) 1497 assert(bpf_map_get_next_key(fd, &key, &key) == 0); 1498 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT); 1499 1500 /* Another check for all elements */ 1501 for (i = 0; i < MAP_SIZE; i++) { 1502 key = MAP_SIZE - i - 1; 1503 1504 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && 1505 value == key); 1506 } 1507 1508 /* Now let's delete all elemenets in parallel. */ 1509 data[1] = DO_DELETE; 1510 run_parallel(TASKS, test_update_delete, data); 1511 1512 /* Nothing should be left. */ 1513 key = -1; 1514 assert(bpf_map_get_next_key(fd, NULL, &key) < 0 && errno == ENOENT); 1515 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT); 1516 } 1517 1518 static void test_map_rdonly(void) 1519 { 1520 int fd, key = 0, value = 0; 1521 1522 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1523 MAP_SIZE, map_flags | BPF_F_RDONLY); 1524 if (fd < 0) { 1525 printf("Failed to create map for read only test '%s'!\n", 1526 strerror(errno)); 1527 exit(1); 1528 } 1529 1530 key = 1; 1531 value = 1234; 1532 /* Try to insert key=1 element. */ 1533 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) < 0 && 1534 errno == EPERM); 1535 1536 /* Check that key=1 is not found. */ 1537 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT); 1538 assert(bpf_map_get_next_key(fd, &key, &value) < 0 && errno == ENOENT); 1539 1540 close(fd); 1541 } 1542 1543 static void test_map_wronly_hash(void) 1544 { 1545 int fd, key = 0, value = 0; 1546 1547 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), 1548 MAP_SIZE, map_flags | BPF_F_WRONLY); 1549 if (fd < 0) { 1550 printf("Failed to create map for write only test '%s'!\n", 1551 strerror(errno)); 1552 exit(1); 1553 } 1554 1555 key = 1; 1556 value = 1234; 1557 /* Insert key=1 element. */ 1558 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0); 1559 1560 /* Check that reading elements and keys from the map is not allowed. */ 1561 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == EPERM); 1562 assert(bpf_map_get_next_key(fd, &key, &value) < 0 && errno == EPERM); 1563 1564 close(fd); 1565 } 1566 1567 static void test_map_wronly_stack_or_queue(enum bpf_map_type map_type) 1568 { 1569 int fd, value = 0; 1570 1571 assert(map_type == BPF_MAP_TYPE_QUEUE || 1572 map_type == BPF_MAP_TYPE_STACK); 1573 fd = bpf_create_map(map_type, 0, sizeof(value), MAP_SIZE, 1574 map_flags | BPF_F_WRONLY); 1575 /* Stack/Queue maps do not support BPF_F_NO_PREALLOC */ 1576 if (map_flags & BPF_F_NO_PREALLOC) { 1577 assert(fd < 0 && errno == EINVAL); 1578 return; 1579 } 1580 if (fd < 0) { 1581 printf("Failed to create map '%s'!\n", strerror(errno)); 1582 exit(1); 1583 } 1584 1585 value = 1234; 1586 assert(bpf_map_update_elem(fd, NULL, &value, BPF_ANY) == 0); 1587 1588 /* Peek element should fail */ 1589 assert(bpf_map_lookup_elem(fd, NULL, &value) < 0 && errno == EPERM); 1590 1591 /* Pop element should fail */ 1592 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &value) < 0 && 1593 errno == EPERM); 1594 1595 close(fd); 1596 } 1597 1598 static void test_map_wronly(void) 1599 { 1600 test_map_wronly_hash(); 1601 test_map_wronly_stack_or_queue(BPF_MAP_TYPE_STACK); 1602 test_map_wronly_stack_or_queue(BPF_MAP_TYPE_QUEUE); 1603 } 1604 1605 static void prepare_reuseport_grp(int type, int map_fd, size_t map_elem_size, 1606 __s64 *fds64, __u64 *sk_cookies, 1607 unsigned int n) 1608 { 1609 socklen_t optlen, addrlen; 1610 struct sockaddr_in6 s6; 1611 const __u32 index0 = 0; 1612 const int optval = 1; 1613 unsigned int i; 1614 u64 sk_cookie; 1615 void *value; 1616 __s32 fd32; 1617 __s64 fd64; 1618 int err; 1619 1620 s6.sin6_family = AF_INET6; 1621 s6.sin6_addr = in6addr_any; 1622 s6.sin6_port = 0; 1623 addrlen = sizeof(s6); 1624 optlen = sizeof(sk_cookie); 1625 1626 for (i = 0; i < n; i++) { 1627 fd64 = socket(AF_INET6, type, 0); 1628 CHECK(fd64 == -1, "socket()", 1629 "sock_type:%d fd64:%lld errno:%d\n", 1630 type, fd64, errno); 1631 1632 err = setsockopt(fd64, SOL_SOCKET, SO_REUSEPORT, 1633 &optval, sizeof(optval)); 1634 CHECK(err == -1, "setsockopt(SO_REUSEPORT)", 1635 "err:%d errno:%d\n", err, errno); 1636 1637 /* reuseport_array does not allow unbound sk */ 1638 if (map_elem_size == sizeof(__u64)) 1639 value = &fd64; 1640 else { 1641 assert(map_elem_size == sizeof(__u32)); 1642 fd32 = (__s32)fd64; 1643 value = &fd32; 1644 } 1645 err = bpf_map_update_elem(map_fd, &index0, value, BPF_ANY); 1646 CHECK(err >= 0 || errno != EINVAL, 1647 "reuseport array update unbound sk", 1648 "sock_type:%d err:%d errno:%d\n", 1649 type, err, errno); 1650 1651 err = bind(fd64, (struct sockaddr *)&s6, sizeof(s6)); 1652 CHECK(err == -1, "bind()", 1653 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1654 1655 if (i == 0) { 1656 err = getsockname(fd64, (struct sockaddr *)&s6, 1657 &addrlen); 1658 CHECK(err == -1, "getsockname()", 1659 "sock_type:%d err:%d errno:%d\n", 1660 type, err, errno); 1661 } 1662 1663 err = getsockopt(fd64, SOL_SOCKET, SO_COOKIE, &sk_cookie, 1664 &optlen); 1665 CHECK(err == -1, "getsockopt(SO_COOKIE)", 1666 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1667 1668 if (type == SOCK_STREAM) { 1669 /* 1670 * reuseport_array does not allow 1671 * non-listening tcp sk. 1672 */ 1673 err = bpf_map_update_elem(map_fd, &index0, value, 1674 BPF_ANY); 1675 CHECK(err >= 0 || errno != EINVAL, 1676 "reuseport array update non-listening sk", 1677 "sock_type:%d err:%d errno:%d\n", 1678 type, err, errno); 1679 err = listen(fd64, 0); 1680 CHECK(err == -1, "listen()", 1681 "sock_type:%d, err:%d errno:%d\n", 1682 type, err, errno); 1683 } 1684 1685 fds64[i] = fd64; 1686 sk_cookies[i] = sk_cookie; 1687 } 1688 } 1689 1690 static void test_reuseport_array(void) 1691 { 1692 #define REUSEPORT_FD_IDX(err, last) ({ (err) ? last : !last; }) 1693 1694 const __u32 array_size = 4, index0 = 0, index3 = 3; 1695 int types[2] = { SOCK_STREAM, SOCK_DGRAM }, type; 1696 __u64 grpa_cookies[2], sk_cookie, map_cookie; 1697 __s64 grpa_fds64[2] = { -1, -1 }, fd64 = -1; 1698 const __u32 bad_index = array_size; 1699 int map_fd, err, t, f; 1700 __u32 fds_idx = 0; 1701 int fd; 1702 1703 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 1704 sizeof(__u32), sizeof(__u64), array_size, 0); 1705 CHECK(map_fd < 0, "reuseport array create", 1706 "map_fd:%d, errno:%d\n", map_fd, errno); 1707 1708 /* Test lookup/update/delete with invalid index */ 1709 err = bpf_map_delete_elem(map_fd, &bad_index); 1710 CHECK(err >= 0 || errno != E2BIG, "reuseport array del >=max_entries", 1711 "err:%d errno:%d\n", err, errno); 1712 1713 err = bpf_map_update_elem(map_fd, &bad_index, &fd64, BPF_ANY); 1714 CHECK(err >= 0 || errno != E2BIG, 1715 "reuseport array update >=max_entries", 1716 "err:%d errno:%d\n", err, errno); 1717 1718 err = bpf_map_lookup_elem(map_fd, &bad_index, &map_cookie); 1719 CHECK(err >= 0 || errno != ENOENT, 1720 "reuseport array update >=max_entries", 1721 "err:%d errno:%d\n", err, errno); 1722 1723 /* Test lookup/delete non existence elem */ 1724 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1725 CHECK(err >= 0 || errno != ENOENT, 1726 "reuseport array lookup not-exist elem", 1727 "err:%d errno:%d\n", err, errno); 1728 err = bpf_map_delete_elem(map_fd, &index3); 1729 CHECK(err >= 0 || errno != ENOENT, 1730 "reuseport array del not-exist elem", 1731 "err:%d errno:%d\n", err, errno); 1732 1733 for (t = 0; t < ARRAY_SIZE(types); t++) { 1734 type = types[t]; 1735 1736 prepare_reuseport_grp(type, map_fd, sizeof(__u64), grpa_fds64, 1737 grpa_cookies, ARRAY_SIZE(grpa_fds64)); 1738 1739 /* Test BPF_* update flags */ 1740 /* BPF_EXIST failure case */ 1741 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1742 BPF_EXIST); 1743 CHECK(err >= 0 || errno != ENOENT, 1744 "reuseport array update empty elem BPF_EXIST", 1745 "sock_type:%d err:%d errno:%d\n", 1746 type, err, errno); 1747 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1748 1749 /* BPF_NOEXIST success case */ 1750 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1751 BPF_NOEXIST); 1752 CHECK(err < 0, 1753 "reuseport array update empty elem BPF_NOEXIST", 1754 "sock_type:%d err:%d errno:%d\n", 1755 type, err, errno); 1756 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1757 1758 /* BPF_EXIST success case. */ 1759 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1760 BPF_EXIST); 1761 CHECK(err < 0, 1762 "reuseport array update same elem BPF_EXIST", 1763 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1764 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1765 1766 /* BPF_NOEXIST failure case */ 1767 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1768 BPF_NOEXIST); 1769 CHECK(err >= 0 || errno != EEXIST, 1770 "reuseport array update non-empty elem BPF_NOEXIST", 1771 "sock_type:%d err:%d errno:%d\n", 1772 type, err, errno); 1773 fds_idx = REUSEPORT_FD_IDX(err, fds_idx); 1774 1775 /* BPF_ANY case (always succeed) */ 1776 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx], 1777 BPF_ANY); 1778 CHECK(err < 0, 1779 "reuseport array update same sk with BPF_ANY", 1780 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1781 1782 fd64 = grpa_fds64[fds_idx]; 1783 sk_cookie = grpa_cookies[fds_idx]; 1784 1785 /* The same sk cannot be added to reuseport_array twice */ 1786 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_ANY); 1787 CHECK(err >= 0 || errno != EBUSY, 1788 "reuseport array update same sk with same index", 1789 "sock_type:%d err:%d errno:%d\n", 1790 type, err, errno); 1791 1792 err = bpf_map_update_elem(map_fd, &index0, &fd64, BPF_ANY); 1793 CHECK(err >= 0 || errno != EBUSY, 1794 "reuseport array update same sk with different index", 1795 "sock_type:%d err:%d errno:%d\n", 1796 type, err, errno); 1797 1798 /* Test delete elem */ 1799 err = bpf_map_delete_elem(map_fd, &index3); 1800 CHECK(err < 0, "reuseport array delete sk", 1801 "sock_type:%d err:%d errno:%d\n", 1802 type, err, errno); 1803 1804 /* Add it back with BPF_NOEXIST */ 1805 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); 1806 CHECK(err < 0, 1807 "reuseport array re-add with BPF_NOEXIST after del", 1808 "sock_type:%d err:%d errno:%d\n", type, err, errno); 1809 1810 /* Test cookie */ 1811 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1812 CHECK(err < 0 || sk_cookie != map_cookie, 1813 "reuseport array lookup re-added sk", 1814 "sock_type:%d err:%d errno:%d sk_cookie:0x%llx map_cookie:0x%llxn", 1815 type, err, errno, sk_cookie, map_cookie); 1816 1817 /* Test elem removed by close() */ 1818 for (f = 0; f < ARRAY_SIZE(grpa_fds64); f++) 1819 close(grpa_fds64[f]); 1820 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1821 CHECK(err >= 0 || errno != ENOENT, 1822 "reuseport array lookup after close()", 1823 "sock_type:%d err:%d errno:%d\n", 1824 type, err, errno); 1825 } 1826 1827 /* Test SOCK_RAW */ 1828 fd64 = socket(AF_INET6, SOCK_RAW, IPPROTO_UDP); 1829 CHECK(fd64 == -1, "socket(SOCK_RAW)", "err:%d errno:%d\n", 1830 err, errno); 1831 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST); 1832 CHECK(err >= 0 || errno != ENOTSUPP, "reuseport array update SOCK_RAW", 1833 "err:%d errno:%d\n", err, errno); 1834 close(fd64); 1835 1836 /* Close the 64 bit value map */ 1837 close(map_fd); 1838 1839 /* Test 32 bit fd */ 1840 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, 1841 sizeof(__u32), sizeof(__u32), array_size, 0); 1842 CHECK(map_fd < 0, "reuseport array create", 1843 "map_fd:%d, errno:%d\n", map_fd, errno); 1844 prepare_reuseport_grp(SOCK_STREAM, map_fd, sizeof(__u32), &fd64, 1845 &sk_cookie, 1); 1846 fd = fd64; 1847 err = bpf_map_update_elem(map_fd, &index3, &fd, BPF_NOEXIST); 1848 CHECK(err < 0, "reuseport array update 32 bit fd", 1849 "err:%d errno:%d\n", err, errno); 1850 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie); 1851 CHECK(err >= 0 || errno != ENOSPC, 1852 "reuseport array lookup 32 bit fd", 1853 "err:%d errno:%d\n", err, errno); 1854 close(fd); 1855 close(map_fd); 1856 } 1857 1858 static void run_all_tests(void) 1859 { 1860 test_hashmap(0, NULL); 1861 test_hashmap_percpu(0, NULL); 1862 test_hashmap_walk(0, NULL); 1863 test_hashmap_zero_seed(); 1864 1865 test_arraymap(0, NULL); 1866 test_arraymap_percpu(0, NULL); 1867 1868 test_arraymap_percpu_many_keys(); 1869 1870 test_devmap(0, NULL); 1871 test_devmap_hash(0, NULL); 1872 test_sockmap(0, NULL); 1873 1874 test_map_large(); 1875 test_map_parallel(); 1876 test_map_stress(); 1877 1878 test_map_rdonly(); 1879 test_map_wronly(); 1880 1881 test_reuseport_array(); 1882 1883 test_queuemap(0, NULL); 1884 test_stackmap(0, NULL); 1885 1886 test_map_in_map(); 1887 } 1888 1889 #define DEFINE_TEST(name) extern void test_##name(void); 1890 #include <map_tests/tests.h> 1891 #undef DEFINE_TEST 1892 1893 int main(void) 1894 { 1895 srand(time(NULL)); 1896 1897 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 1898 1899 map_flags = 0; 1900 run_all_tests(); 1901 1902 map_flags = BPF_F_NO_PREALLOC; 1903 run_all_tests(); 1904 1905 #define DEFINE_TEST(name) test_##name(); 1906 #include <map_tests/tests.h> 1907 #undef DEFINE_TEST 1908 1909 printf("test_maps: OK, %d SKIPPED\n", skips); 1910 return 0; 1911 } 1912