1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2016 Facebook 3 */ 4 #define _GNU_SOURCE 5 #include <sched.h> 6 #include <stdio.h> 7 #include <sys/types.h> 8 #include <asm/unistd.h> 9 #include <unistd.h> 10 #include <assert.h> 11 #include <sys/wait.h> 12 #include <stdlib.h> 13 #include <signal.h> 14 #include <string.h> 15 #include <time.h> 16 #include <sys/resource.h> 17 #include <arpa/inet.h> 18 #include <errno.h> 19 20 #include <bpf/bpf.h> 21 #include <bpf/libbpf.h> 22 23 #define TEST_BIT(t) (1U << (t)) 24 #define MAX_NR_CPUS 1024 25 26 static __u64 time_get_ns(void) 27 { 28 struct timespec ts; 29 30 clock_gettime(CLOCK_MONOTONIC, &ts); 31 return ts.tv_sec * 1000000000ull + ts.tv_nsec; 32 } 33 34 enum test_type { 35 HASH_PREALLOC, 36 PERCPU_HASH_PREALLOC, 37 HASH_KMALLOC, 38 PERCPU_HASH_KMALLOC, 39 LRU_HASH_PREALLOC, 40 NOCOMMON_LRU_HASH_PREALLOC, 41 LPM_KMALLOC, 42 HASH_LOOKUP, 43 ARRAY_LOOKUP, 44 INNER_LRU_HASH_PREALLOC, 45 LRU_HASH_LOOKUP, 46 NR_TESTS, 47 }; 48 49 const char *test_map_names[NR_TESTS] = { 50 [HASH_PREALLOC] = "hash_map", 51 [PERCPU_HASH_PREALLOC] = "percpu_hash_map", 52 [HASH_KMALLOC] = "hash_map_alloc", 53 [PERCPU_HASH_KMALLOC] = "percpu_hash_map_alloc", 54 [LRU_HASH_PREALLOC] = "lru_hash_map", 55 [NOCOMMON_LRU_HASH_PREALLOC] = "nocommon_lru_hash_map", 56 [LPM_KMALLOC] = "lpm_trie_map_alloc", 57 [HASH_LOOKUP] = "hash_map", 58 [ARRAY_LOOKUP] = "array_map", 59 [INNER_LRU_HASH_PREALLOC] = "inner_lru_hash_map", 60 [LRU_HASH_LOOKUP] = "lru_hash_lookup_map", 61 }; 62 63 enum map_idx { 64 array_of_lru_hashs_idx, 65 hash_map_alloc_idx, 66 lru_hash_lookup_idx, 67 NR_IDXES, 68 }; 69 70 static int map_fd[NR_IDXES]; 71 72 static int test_flags = ~0; 73 static uint32_t num_map_entries; 74 static uint32_t inner_lru_hash_size; 75 static int lru_hash_lookup_test_entries = 32; 76 static uint32_t max_cnt = 1000000; 77 78 static int check_test_flags(enum test_type t) 79 { 80 return test_flags & TEST_BIT(t); 81 } 82 83 static void test_hash_prealloc(int cpu) 84 { 85 __u64 start_time; 86 int i; 87 88 start_time = time_get_ns(); 89 for (i = 0; i < max_cnt; i++) 90 syscall(__NR_getuid); 91 printf("%d:hash_map_perf pre-alloc %lld events per sec\n", 92 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 93 } 94 95 static int pre_test_lru_hash_lookup(int tasks) 96 { 97 int fd = map_fd[lru_hash_lookup_idx]; 98 uint32_t key; 99 long val = 1; 100 int ret; 101 102 if (num_map_entries > lru_hash_lookup_test_entries) 103 lru_hash_lookup_test_entries = num_map_entries; 104 105 /* Populate the lru_hash_map for LRU_HASH_LOOKUP perf test. 106 * 107 * It is fine that the user requests for a map with 108 * num_map_entries < 32 and some of the later lru hash lookup 109 * may return not found. For LRU map, we are not interested 110 * in such small map performance. 111 */ 112 for (key = 0; key < lru_hash_lookup_test_entries; key++) { 113 ret = bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST); 114 if (ret) 115 return ret; 116 } 117 118 return 0; 119 } 120 121 static void do_test_lru(enum test_type test, int cpu) 122 { 123 static int inner_lru_map_fds[MAX_NR_CPUS]; 124 125 struct sockaddr_in6 in6 = { .sin6_family = AF_INET6 }; 126 const char *test_name; 127 __u64 start_time; 128 int i, ret; 129 130 if (test == INNER_LRU_HASH_PREALLOC && cpu) { 131 /* If CPU is not 0, create inner_lru hash map and insert the fd 132 * value into the array_of_lru_hash map. In case of CPU 0, 133 * 'inner_lru_hash_map' was statically inserted on the map init 134 */ 135 int outer_fd = map_fd[array_of_lru_hashs_idx]; 136 unsigned int mycpu, mynode; 137 138 assert(cpu < MAX_NR_CPUS); 139 140 ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL); 141 assert(!ret); 142 143 inner_lru_map_fds[cpu] = 144 bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH, 145 test_map_names[INNER_LRU_HASH_PREALLOC], 146 sizeof(uint32_t), 147 sizeof(long), 148 inner_lru_hash_size, 0, 149 mynode); 150 if (inner_lru_map_fds[cpu] == -1) { 151 printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n", 152 strerror(errno), errno); 153 exit(1); 154 } 155 156 ret = bpf_map_update_elem(outer_fd, &cpu, 157 &inner_lru_map_fds[cpu], 158 BPF_ANY); 159 if (ret) { 160 printf("cannot update ARRAY_OF_LRU_HASHS with key:%u. %s(%d)\n", 161 cpu, strerror(errno), errno); 162 exit(1); 163 } 164 } 165 166 in6.sin6_addr.s6_addr16[0] = 0xdead; 167 in6.sin6_addr.s6_addr16[1] = 0xbeef; 168 169 if (test == LRU_HASH_PREALLOC) { 170 test_name = "lru_hash_map_perf"; 171 in6.sin6_addr.s6_addr16[2] = 0; 172 } else if (test == NOCOMMON_LRU_HASH_PREALLOC) { 173 test_name = "nocommon_lru_hash_map_perf"; 174 in6.sin6_addr.s6_addr16[2] = 1; 175 } else if (test == INNER_LRU_HASH_PREALLOC) { 176 test_name = "inner_lru_hash_map_perf"; 177 in6.sin6_addr.s6_addr16[2] = 2; 178 } else if (test == LRU_HASH_LOOKUP) { 179 test_name = "lru_hash_lookup_perf"; 180 in6.sin6_addr.s6_addr16[2] = 3; 181 in6.sin6_addr.s6_addr32[3] = 0; 182 } else { 183 assert(0); 184 } 185 186 start_time = time_get_ns(); 187 for (i = 0; i < max_cnt; i++) { 188 ret = connect(-1, (const struct sockaddr *)&in6, sizeof(in6)); 189 assert(ret == -1 && errno == EBADF); 190 if (in6.sin6_addr.s6_addr32[3] < 191 lru_hash_lookup_test_entries - 32) 192 in6.sin6_addr.s6_addr32[3] += 32; 193 else 194 in6.sin6_addr.s6_addr32[3] = 0; 195 } 196 printf("%d:%s pre-alloc %lld events per sec\n", 197 cpu, test_name, 198 max_cnt * 1000000000ll / (time_get_ns() - start_time)); 199 } 200 201 static void test_lru_hash_prealloc(int cpu) 202 { 203 do_test_lru(LRU_HASH_PREALLOC, cpu); 204 } 205 206 static void test_nocommon_lru_hash_prealloc(int cpu) 207 { 208 do_test_lru(NOCOMMON_LRU_HASH_PREALLOC, cpu); 209 } 210 211 static void test_inner_lru_hash_prealloc(int cpu) 212 { 213 do_test_lru(INNER_LRU_HASH_PREALLOC, cpu); 214 } 215 216 static void test_lru_hash_lookup(int cpu) 217 { 218 do_test_lru(LRU_HASH_LOOKUP, cpu); 219 } 220 221 static void test_percpu_hash_prealloc(int cpu) 222 { 223 __u64 start_time; 224 int i; 225 226 start_time = time_get_ns(); 227 for (i = 0; i < max_cnt; i++) 228 syscall(__NR_geteuid); 229 printf("%d:percpu_hash_map_perf pre-alloc %lld events per sec\n", 230 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 231 } 232 233 static void test_hash_kmalloc(int cpu) 234 { 235 __u64 start_time; 236 int i; 237 238 start_time = time_get_ns(); 239 for (i = 0; i < max_cnt; i++) 240 syscall(__NR_getgid); 241 printf("%d:hash_map_perf kmalloc %lld events per sec\n", 242 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 243 } 244 245 static void test_percpu_hash_kmalloc(int cpu) 246 { 247 __u64 start_time; 248 int i; 249 250 start_time = time_get_ns(); 251 for (i = 0; i < max_cnt; i++) 252 syscall(__NR_getegid); 253 printf("%d:percpu_hash_map_perf kmalloc %lld events per sec\n", 254 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 255 } 256 257 static void test_lpm_kmalloc(int cpu) 258 { 259 __u64 start_time; 260 int i; 261 262 start_time = time_get_ns(); 263 for (i = 0; i < max_cnt; i++) 264 syscall(__NR_gettid); 265 printf("%d:lpm_perf kmalloc %lld events per sec\n", 266 cpu, max_cnt * 1000000000ll / (time_get_ns() - start_time)); 267 } 268 269 static void test_hash_lookup(int cpu) 270 { 271 __u64 start_time; 272 int i; 273 274 start_time = time_get_ns(); 275 for (i = 0; i < max_cnt; i++) 276 syscall(__NR_getpgid, 0); 277 printf("%d:hash_lookup %lld lookups per sec\n", 278 cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time)); 279 } 280 281 static void test_array_lookup(int cpu) 282 { 283 __u64 start_time; 284 int i; 285 286 start_time = time_get_ns(); 287 for (i = 0; i < max_cnt; i++) 288 syscall(__NR_getppid, 0); 289 printf("%d:array_lookup %lld lookups per sec\n", 290 cpu, max_cnt * 1000000000ll * 64 / (time_get_ns() - start_time)); 291 } 292 293 typedef int (*pre_test_func)(int tasks); 294 const pre_test_func pre_test_funcs[] = { 295 [LRU_HASH_LOOKUP] = pre_test_lru_hash_lookup, 296 }; 297 298 typedef void (*test_func)(int cpu); 299 const test_func test_funcs[] = { 300 [HASH_PREALLOC] = test_hash_prealloc, 301 [PERCPU_HASH_PREALLOC] = test_percpu_hash_prealloc, 302 [HASH_KMALLOC] = test_hash_kmalloc, 303 [PERCPU_HASH_KMALLOC] = test_percpu_hash_kmalloc, 304 [LRU_HASH_PREALLOC] = test_lru_hash_prealloc, 305 [NOCOMMON_LRU_HASH_PREALLOC] = test_nocommon_lru_hash_prealloc, 306 [LPM_KMALLOC] = test_lpm_kmalloc, 307 [HASH_LOOKUP] = test_hash_lookup, 308 [ARRAY_LOOKUP] = test_array_lookup, 309 [INNER_LRU_HASH_PREALLOC] = test_inner_lru_hash_prealloc, 310 [LRU_HASH_LOOKUP] = test_lru_hash_lookup, 311 }; 312 313 static int pre_test(int tasks) 314 { 315 int i; 316 317 for (i = 0; i < NR_TESTS; i++) { 318 if (pre_test_funcs[i] && check_test_flags(i)) { 319 int ret = pre_test_funcs[i](tasks); 320 321 if (ret) 322 return ret; 323 } 324 } 325 326 return 0; 327 } 328 329 static void loop(int cpu) 330 { 331 cpu_set_t cpuset; 332 int i; 333 334 CPU_ZERO(&cpuset); 335 CPU_SET(cpu, &cpuset); 336 sched_setaffinity(0, sizeof(cpuset), &cpuset); 337 338 for (i = 0; i < NR_TESTS; i++) { 339 if (check_test_flags(i)) 340 test_funcs[i](cpu); 341 } 342 } 343 344 static void run_perf_test(int tasks) 345 { 346 pid_t pid[tasks]; 347 int i; 348 349 assert(!pre_test(tasks)); 350 351 for (i = 0; i < tasks; i++) { 352 pid[i] = fork(); 353 if (pid[i] == 0) { 354 loop(i); 355 exit(0); 356 } else if (pid[i] == -1) { 357 printf("couldn't spawn #%d process\n", i); 358 exit(1); 359 } 360 } 361 for (i = 0; i < tasks; i++) { 362 int status; 363 364 assert(waitpid(pid[i], &status, 0) == pid[i]); 365 assert(status == 0); 366 } 367 } 368 369 static void fill_lpm_trie(void) 370 { 371 struct bpf_lpm_trie_key *key; 372 unsigned long value = 0; 373 unsigned int i; 374 int r; 375 376 key = alloca(sizeof(*key) + 4); 377 key->prefixlen = 32; 378 379 for (i = 0; i < 512; ++i) { 380 key->prefixlen = rand() % 33; 381 key->data[0] = rand() & 0xff; 382 key->data[1] = rand() & 0xff; 383 key->data[2] = rand() & 0xff; 384 key->data[3] = rand() & 0xff; 385 r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], 386 key, &value, 0); 387 assert(!r); 388 } 389 390 key->prefixlen = 32; 391 key->data[0] = 192; 392 key->data[1] = 168; 393 key->data[2] = 0; 394 key->data[3] = 1; 395 value = 128; 396 397 r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0); 398 assert(!r); 399 } 400 401 static void fixup_map(struct bpf_object *obj) 402 { 403 struct bpf_map *map; 404 int i; 405 406 bpf_object__for_each_map(map, obj) { 407 const char *name = bpf_map__name(map); 408 409 /* Only change the max_entries for the enabled test(s) */ 410 for (i = 0; i < NR_TESTS; i++) { 411 if (!strcmp(test_map_names[i], name) && 412 (check_test_flags(i))) { 413 bpf_map__resize(map, num_map_entries); 414 continue; 415 } 416 } 417 } 418 419 inner_lru_hash_size = num_map_entries; 420 } 421 422 int main(int argc, char **argv) 423 { 424 int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); 425 struct bpf_link *links[8]; 426 struct bpf_program *prog; 427 struct bpf_object *obj; 428 struct bpf_map *map; 429 char filename[256]; 430 int i = 0; 431 432 if (argc > 1) 433 test_flags = atoi(argv[1]) ? : test_flags; 434 435 if (argc > 2) 436 nr_cpus = atoi(argv[2]) ? : nr_cpus; 437 438 if (argc > 3) 439 num_map_entries = atoi(argv[3]); 440 441 if (argc > 4) 442 max_cnt = atoi(argv[4]); 443 444 snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); 445 obj = bpf_object__open_file(filename, NULL); 446 if (libbpf_get_error(obj)) { 447 fprintf(stderr, "ERROR: opening BPF object file failed\n"); 448 return 0; 449 } 450 451 map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map"); 452 if (libbpf_get_error(map)) { 453 fprintf(stderr, "ERROR: finding a map in obj file failed\n"); 454 goto cleanup; 455 } 456 457 inner_lru_hash_size = bpf_map__max_entries(map); 458 if (!inner_lru_hash_size) { 459 fprintf(stderr, "ERROR: failed to get map attribute\n"); 460 goto cleanup; 461 } 462 463 /* resize BPF map prior to loading */ 464 if (num_map_entries > 0) 465 fixup_map(obj); 466 467 /* load BPF program */ 468 if (bpf_object__load(obj)) { 469 fprintf(stderr, "ERROR: loading BPF object file failed\n"); 470 goto cleanup; 471 } 472 473 map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs"); 474 map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc"); 475 map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map"); 476 if (map_fd[0] < 0 || map_fd[1] < 0 || map_fd[2] < 0) { 477 fprintf(stderr, "ERROR: finding a map in obj file failed\n"); 478 goto cleanup; 479 } 480 481 bpf_object__for_each_program(prog, obj) { 482 links[i] = bpf_program__attach(prog); 483 if (libbpf_get_error(links[i])) { 484 fprintf(stderr, "ERROR: bpf_program__attach failed\n"); 485 links[i] = NULL; 486 goto cleanup; 487 } 488 i++; 489 } 490 491 fill_lpm_trie(); 492 493 run_perf_test(nr_cpus); 494 495 cleanup: 496 for (i--; i >= 0; i--) 497 bpf_link__destroy(links[i]); 498 499 bpf_object__close(obj); 500 return 0; 501 } 502