1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Testsuite for eBPF verifier 4 * 5 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com 6 * Copyright (c) 2017 Facebook 7 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 8 */ 9 10 #include <endian.h> 11 #include <asm/types.h> 12 #include <linux/types.h> 13 #include <stdint.h> 14 #include <stdio.h> 15 #include <stdlib.h> 16 #include <unistd.h> 17 #include <errno.h> 18 #include <string.h> 19 #include <stddef.h> 20 #include <stdbool.h> 21 #include <sched.h> 22 #include <limits.h> 23 #include <assert.h> 24 25 #include <sys/capability.h> 26 27 #include <linux/unistd.h> 28 #include <linux/filter.h> 29 #include <linux/bpf_perf_event.h> 30 #include <linux/bpf.h> 31 #include <linux/if_ether.h> 32 #include <linux/btf.h> 33 34 #include <bpf/bpf.h> 35 #include <bpf/libbpf.h> 36 37 #ifdef HAVE_GENHDR 38 # include "autoconf.h" 39 #else 40 # if defined(__i386) || defined(__x86_64) || defined(__s390x__) || defined(__aarch64__) 41 # define CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 1 42 # endif 43 #endif 44 #include "bpf_rlimit.h" 45 #include "bpf_rand.h" 46 #include "bpf_util.h" 47 #include "test_btf.h" 48 #include "../../../include/linux/filter.h" 49 50 #define MAX_INSNS BPF_MAXINSNS 51 #define MAX_TEST_INSNS 1000000 52 #define MAX_FIXUPS 8 53 #define MAX_NR_MAPS 18 54 #define MAX_TEST_RUNS 8 55 #define POINTER_VALUE 0xcafe4all 56 #define TEST_DATA_LEN 64 57 58 #define F_NEEDS_EFFICIENT_UNALIGNED_ACCESS (1 << 0) 59 #define F_LOAD_WITH_STRICT_ALIGNMENT (1 << 1) 60 61 #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled" 62 static bool unpriv_disabled = false; 63 static int skips; 64 65 struct bpf_test { 66 const char *descr; 67 struct bpf_insn insns[MAX_INSNS]; 68 struct bpf_insn *fill_insns; 69 int fixup_map_hash_8b[MAX_FIXUPS]; 70 int fixup_map_hash_48b[MAX_FIXUPS]; 71 int fixup_map_hash_16b[MAX_FIXUPS]; 72 int fixup_map_array_48b[MAX_FIXUPS]; 73 int fixup_map_sockmap[MAX_FIXUPS]; 74 int fixup_map_sockhash[MAX_FIXUPS]; 75 int fixup_map_xskmap[MAX_FIXUPS]; 76 int fixup_map_stacktrace[MAX_FIXUPS]; 77 int fixup_prog1[MAX_FIXUPS]; 78 int fixup_prog2[MAX_FIXUPS]; 79 int fixup_map_in_map[MAX_FIXUPS]; 80 int fixup_cgroup_storage[MAX_FIXUPS]; 81 int fixup_percpu_cgroup_storage[MAX_FIXUPS]; 82 int fixup_map_spin_lock[MAX_FIXUPS]; 83 int fixup_map_array_ro[MAX_FIXUPS]; 84 int fixup_map_array_wo[MAX_FIXUPS]; 85 int fixup_map_array_small[MAX_FIXUPS]; 86 int fixup_sk_storage_map[MAX_FIXUPS]; 87 const char *errstr; 88 const char *errstr_unpriv; 89 uint32_t retval, retval_unpriv, insn_processed; 90 int prog_len; 91 enum { 92 UNDEF, 93 ACCEPT, 94 REJECT 95 } result, result_unpriv; 96 enum bpf_prog_type prog_type; 97 uint8_t flags; 98 __u8 data[TEST_DATA_LEN]; 99 void (*fill_helper)(struct bpf_test *self); 100 uint8_t runs; 101 struct { 102 uint32_t retval, retval_unpriv; 103 union { 104 __u8 data[TEST_DATA_LEN]; 105 __u64 data64[TEST_DATA_LEN / 8]; 106 }; 107 } retvals[MAX_TEST_RUNS]; 108 }; 109 110 /* Note we want this to be 64 bit aligned so that the end of our array is 111 * actually the end of the structure. 112 */ 113 #define MAX_ENTRIES 11 114 115 struct test_val { 116 unsigned int index; 117 int foo[MAX_ENTRIES]; 118 }; 119 120 struct other_val { 121 long long foo; 122 long long bar; 123 }; 124 125 static void bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self) 126 { 127 /* test: {skb->data[0], vlan_push} x 51 + {skb->data[0], vlan_pop} x 51 */ 128 #define PUSH_CNT 51 129 /* jump range is limited to 16 bit. PUSH_CNT of ld_abs needs room */ 130 unsigned int len = (1 << 15) - PUSH_CNT * 2 * 5 * 6; 131 struct bpf_insn *insn = self->fill_insns; 132 int i = 0, j, k = 0; 133 134 insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); 135 loop: 136 for (j = 0; j < PUSH_CNT; j++) { 137 insn[i++] = BPF_LD_ABS(BPF_B, 0); 138 /* jump to error label */ 139 insn[i] = BPF_JMP32_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 3); 140 i++; 141 insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6); 142 insn[i++] = BPF_MOV64_IMM(BPF_REG_2, 1); 143 insn[i++] = BPF_MOV64_IMM(BPF_REG_3, 2); 144 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 145 BPF_FUNC_skb_vlan_push), 146 insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 3); 147 i++; 148 } 149 150 for (j = 0; j < PUSH_CNT; j++) { 151 insn[i++] = BPF_LD_ABS(BPF_B, 0); 152 insn[i] = BPF_JMP32_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 3); 153 i++; 154 insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6); 155 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 156 BPF_FUNC_skb_vlan_pop), 157 insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 3); 158 i++; 159 } 160 if (++k < 5) 161 goto loop; 162 163 for (; i < len - 3; i++) 164 insn[i] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0xbef); 165 insn[len - 3] = BPF_JMP_A(1); 166 /* error label */ 167 insn[len - 2] = BPF_MOV32_IMM(BPF_REG_0, 0); 168 insn[len - 1] = BPF_EXIT_INSN(); 169 self->prog_len = len; 170 } 171 172 static void bpf_fill_jump_around_ld_abs(struct bpf_test *self) 173 { 174 struct bpf_insn *insn = self->fill_insns; 175 /* jump range is limited to 16 bit. every ld_abs is replaced by 6 insns, 176 * but on arches like arm, ppc etc, there will be one BPF_ZEXT inserted 177 * to extend the error value of the inlined ld_abs sequence which then 178 * contains 7 insns. so, set the dividend to 7 so the testcase could 179 * work on all arches. 180 */ 181 unsigned int len = (1 << 15) / 7; 182 int i = 0; 183 184 insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); 185 insn[i++] = BPF_LD_ABS(BPF_B, 0); 186 insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 10, len - i - 2); 187 i++; 188 while (i < len - 1) 189 insn[i++] = BPF_LD_ABS(BPF_B, 1); 190 insn[i] = BPF_EXIT_INSN(); 191 self->prog_len = i + 1; 192 } 193 194 static void bpf_fill_rand_ld_dw(struct bpf_test *self) 195 { 196 struct bpf_insn *insn = self->fill_insns; 197 uint64_t res = 0; 198 int i = 0; 199 200 insn[i++] = BPF_MOV32_IMM(BPF_REG_0, 0); 201 while (i < self->retval) { 202 uint64_t val = bpf_semi_rand_get(); 203 struct bpf_insn tmp[2] = { BPF_LD_IMM64(BPF_REG_1, val) }; 204 205 res ^= val; 206 insn[i++] = tmp[0]; 207 insn[i++] = tmp[1]; 208 insn[i++] = BPF_ALU64_REG(BPF_XOR, BPF_REG_0, BPF_REG_1); 209 } 210 insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0); 211 insn[i++] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_1, 32); 212 insn[i++] = BPF_ALU64_REG(BPF_XOR, BPF_REG_0, BPF_REG_1); 213 insn[i] = BPF_EXIT_INSN(); 214 self->prog_len = i + 1; 215 res ^= (res >> 32); 216 self->retval = (uint32_t)res; 217 } 218 219 #define MAX_JMP_SEQ 8192 220 221 /* test the sequence of 8k jumps */ 222 static void bpf_fill_scale1(struct bpf_test *self) 223 { 224 struct bpf_insn *insn = self->fill_insns; 225 int i = 0, k = 0; 226 227 insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); 228 /* test to check that the long sequence of jumps is acceptable */ 229 while (k++ < MAX_JMP_SEQ) { 230 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 231 BPF_FUNC_get_prandom_u32); 232 insn[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, bpf_semi_rand_get(), 2); 233 insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_10); 234 insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 235 -8 * (k % 64 + 1)); 236 } 237 /* every jump adds 1 step to insn_processed, so to stay exactly 238 * within 1m limit add MAX_TEST_INSNS - MAX_JMP_SEQ - 1 MOVs and 1 EXIT 239 */ 240 while (i < MAX_TEST_INSNS - MAX_JMP_SEQ - 1) 241 insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 42); 242 insn[i] = BPF_EXIT_INSN(); 243 self->prog_len = i + 1; 244 self->retval = 42; 245 } 246 247 /* test the sequence of 8k jumps in inner most function (function depth 8)*/ 248 static void bpf_fill_scale2(struct bpf_test *self) 249 { 250 struct bpf_insn *insn = self->fill_insns; 251 int i = 0, k = 0; 252 253 #define FUNC_NEST 7 254 for (k = 0; k < FUNC_NEST; k++) { 255 insn[i++] = BPF_CALL_REL(1); 256 insn[i++] = BPF_EXIT_INSN(); 257 } 258 insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); 259 /* test to check that the long sequence of jumps is acceptable */ 260 k = 0; 261 while (k++ < MAX_JMP_SEQ) { 262 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 263 BPF_FUNC_get_prandom_u32); 264 insn[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, bpf_semi_rand_get(), 2); 265 insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_10); 266 insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 267 -8 * (k % (64 - 4 * FUNC_NEST) + 1)); 268 } 269 /* every jump adds 1 step to insn_processed, so to stay exactly 270 * within 1m limit add MAX_TEST_INSNS - MAX_JMP_SEQ - 1 MOVs and 1 EXIT 271 */ 272 while (i < MAX_TEST_INSNS - MAX_JMP_SEQ - 1) 273 insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 42); 274 insn[i] = BPF_EXIT_INSN(); 275 self->prog_len = i + 1; 276 self->retval = 42; 277 } 278 279 static void bpf_fill_scale(struct bpf_test *self) 280 { 281 switch (self->retval) { 282 case 1: 283 return bpf_fill_scale1(self); 284 case 2: 285 return bpf_fill_scale2(self); 286 default: 287 self->prog_len = 0; 288 break; 289 } 290 } 291 292 /* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */ 293 #define BPF_SK_LOOKUP(func) \ 294 /* struct bpf_sock_tuple tuple = {} */ \ 295 BPF_MOV64_IMM(BPF_REG_2, 0), \ 296 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -8), \ 297 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -16), \ 298 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -24), \ 299 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -32), \ 300 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -40), \ 301 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -48), \ 302 /* sk = func(ctx, &tuple, sizeof tuple, 0, 0) */ \ 303 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), \ 304 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -48), \ 305 BPF_MOV64_IMM(BPF_REG_3, sizeof(struct bpf_sock_tuple)), \ 306 BPF_MOV64_IMM(BPF_REG_4, 0), \ 307 BPF_MOV64_IMM(BPF_REG_5, 0), \ 308 BPF_EMIT_CALL(BPF_FUNC_ ## func) 309 310 /* BPF_DIRECT_PKT_R2 contains 7 instructions, it initializes default return 311 * value into 0 and does necessary preparation for direct packet access 312 * through r2. The allowed access range is 8 bytes. 313 */ 314 #define BPF_DIRECT_PKT_R2 \ 315 BPF_MOV64_IMM(BPF_REG_0, 0), \ 316 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, \ 317 offsetof(struct __sk_buff, data)), \ 318 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, \ 319 offsetof(struct __sk_buff, data_end)), \ 320 BPF_MOV64_REG(BPF_REG_4, BPF_REG_2), \ 321 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 8), \ 322 BPF_JMP_REG(BPF_JLE, BPF_REG_4, BPF_REG_3, 1), \ 323 BPF_EXIT_INSN() 324 325 /* BPF_RAND_UEXT_R7 contains 4 instructions, it initializes R7 into a random 326 * positive u32, and zero-extend it into 64-bit. 327 */ 328 #define BPF_RAND_UEXT_R7 \ 329 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, \ 330 BPF_FUNC_get_prandom_u32), \ 331 BPF_MOV64_REG(BPF_REG_7, BPF_REG_0), \ 332 BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 33), \ 333 BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 33) 334 335 /* BPF_RAND_SEXT_R7 contains 5 instructions, it initializes R7 into a random 336 * negative u32, and sign-extend it into 64-bit. 337 */ 338 #define BPF_RAND_SEXT_R7 \ 339 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, \ 340 BPF_FUNC_get_prandom_u32), \ 341 BPF_MOV64_REG(BPF_REG_7, BPF_REG_0), \ 342 BPF_ALU64_IMM(BPF_OR, BPF_REG_7, 0x80000000), \ 343 BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 32), \ 344 BPF_ALU64_IMM(BPF_ARSH, BPF_REG_7, 32) 345 346 static struct bpf_test tests[] = { 347 #define FILL_ARRAY 348 #include <verifier/tests.h> 349 #undef FILL_ARRAY 350 }; 351 352 static int probe_filter_length(const struct bpf_insn *fp) 353 { 354 int len; 355 356 for (len = MAX_INSNS - 1; len > 0; --len) 357 if (fp[len].code != 0 || fp[len].imm != 0) 358 break; 359 return len + 1; 360 } 361 362 static bool skip_unsupported_map(enum bpf_map_type map_type) 363 { 364 if (!bpf_probe_map_type(map_type, 0)) { 365 printf("SKIP (unsupported map type %d)\n", map_type); 366 skips++; 367 return true; 368 } 369 return false; 370 } 371 372 static int __create_map(uint32_t type, uint32_t size_key, 373 uint32_t size_value, uint32_t max_elem, 374 uint32_t extra_flags) 375 { 376 int fd; 377 378 fd = bpf_create_map(type, size_key, size_value, max_elem, 379 (type == BPF_MAP_TYPE_HASH ? 380 BPF_F_NO_PREALLOC : 0) | extra_flags); 381 if (fd < 0) { 382 if (skip_unsupported_map(type)) 383 return -1; 384 printf("Failed to create hash map '%s'!\n", strerror(errno)); 385 } 386 387 return fd; 388 } 389 390 static int create_map(uint32_t type, uint32_t size_key, 391 uint32_t size_value, uint32_t max_elem) 392 { 393 return __create_map(type, size_key, size_value, max_elem, 0); 394 } 395 396 static void update_map(int fd, int index) 397 { 398 struct test_val value = { 399 .index = (6 + 1) * sizeof(int), 400 .foo[6] = 0xabcdef12, 401 }; 402 403 assert(!bpf_map_update_elem(fd, &index, &value, 0)); 404 } 405 406 static int create_prog_dummy1(enum bpf_prog_type prog_type) 407 { 408 struct bpf_insn prog[] = { 409 BPF_MOV64_IMM(BPF_REG_0, 42), 410 BPF_EXIT_INSN(), 411 }; 412 413 return bpf_load_program(prog_type, prog, 414 ARRAY_SIZE(prog), "GPL", 0, NULL, 0); 415 } 416 417 static int create_prog_dummy2(enum bpf_prog_type prog_type, int mfd, int idx) 418 { 419 struct bpf_insn prog[] = { 420 BPF_MOV64_IMM(BPF_REG_3, idx), 421 BPF_LD_MAP_FD(BPF_REG_2, mfd), 422 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 423 BPF_FUNC_tail_call), 424 BPF_MOV64_IMM(BPF_REG_0, 41), 425 BPF_EXIT_INSN(), 426 }; 427 428 return bpf_load_program(prog_type, prog, 429 ARRAY_SIZE(prog), "GPL", 0, NULL, 0); 430 } 431 432 static int create_prog_array(enum bpf_prog_type prog_type, uint32_t max_elem, 433 int p1key) 434 { 435 int p2key = 1; 436 int mfd, p1fd, p2fd; 437 438 mfd = bpf_create_map(BPF_MAP_TYPE_PROG_ARRAY, sizeof(int), 439 sizeof(int), max_elem, 0); 440 if (mfd < 0) { 441 if (skip_unsupported_map(BPF_MAP_TYPE_PROG_ARRAY)) 442 return -1; 443 printf("Failed to create prog array '%s'!\n", strerror(errno)); 444 return -1; 445 } 446 447 p1fd = create_prog_dummy1(prog_type); 448 p2fd = create_prog_dummy2(prog_type, mfd, p2key); 449 if (p1fd < 0 || p2fd < 0) 450 goto out; 451 if (bpf_map_update_elem(mfd, &p1key, &p1fd, BPF_ANY) < 0) 452 goto out; 453 if (bpf_map_update_elem(mfd, &p2key, &p2fd, BPF_ANY) < 0) 454 goto out; 455 close(p2fd); 456 close(p1fd); 457 458 return mfd; 459 out: 460 close(p2fd); 461 close(p1fd); 462 close(mfd); 463 return -1; 464 } 465 466 static int create_map_in_map(void) 467 { 468 int inner_map_fd, outer_map_fd; 469 470 inner_map_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(int), 471 sizeof(int), 1, 0); 472 if (inner_map_fd < 0) { 473 if (skip_unsupported_map(BPF_MAP_TYPE_ARRAY)) 474 return -1; 475 printf("Failed to create array '%s'!\n", strerror(errno)); 476 return inner_map_fd; 477 } 478 479 outer_map_fd = bpf_create_map_in_map(BPF_MAP_TYPE_ARRAY_OF_MAPS, NULL, 480 sizeof(int), inner_map_fd, 1, 0); 481 if (outer_map_fd < 0) { 482 if (skip_unsupported_map(BPF_MAP_TYPE_ARRAY_OF_MAPS)) 483 return -1; 484 printf("Failed to create array of maps '%s'!\n", 485 strerror(errno)); 486 } 487 488 close(inner_map_fd); 489 490 return outer_map_fd; 491 } 492 493 static int create_cgroup_storage(bool percpu) 494 { 495 enum bpf_map_type type = percpu ? BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE : 496 BPF_MAP_TYPE_CGROUP_STORAGE; 497 int fd; 498 499 fd = bpf_create_map(type, sizeof(struct bpf_cgroup_storage_key), 500 TEST_DATA_LEN, 0, 0); 501 if (fd < 0) { 502 if (skip_unsupported_map(type)) 503 return -1; 504 printf("Failed to create cgroup storage '%s'!\n", 505 strerror(errno)); 506 } 507 508 return fd; 509 } 510 511 /* struct bpf_spin_lock { 512 * int val; 513 * }; 514 * struct val { 515 * int cnt; 516 * struct bpf_spin_lock l; 517 * }; 518 */ 519 static const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l"; 520 static __u32 btf_raw_types[] = { 521 /* int */ 522 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 523 /* struct bpf_spin_lock */ /* [2] */ 524 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4), 525 BTF_MEMBER_ENC(15, 1, 0), /* int val; */ 526 /* struct val */ /* [3] */ 527 BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8), 528 BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */ 529 BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */ 530 }; 531 532 static int load_btf(void) 533 { 534 struct btf_header hdr = { 535 .magic = BTF_MAGIC, 536 .version = BTF_VERSION, 537 .hdr_len = sizeof(struct btf_header), 538 .type_len = sizeof(btf_raw_types), 539 .str_off = sizeof(btf_raw_types), 540 .str_len = sizeof(btf_str_sec), 541 }; 542 void *ptr, *raw_btf; 543 int btf_fd; 544 545 ptr = raw_btf = malloc(sizeof(hdr) + sizeof(btf_raw_types) + 546 sizeof(btf_str_sec)); 547 548 memcpy(ptr, &hdr, sizeof(hdr)); 549 ptr += sizeof(hdr); 550 memcpy(ptr, btf_raw_types, hdr.type_len); 551 ptr += hdr.type_len; 552 memcpy(ptr, btf_str_sec, hdr.str_len); 553 ptr += hdr.str_len; 554 555 btf_fd = bpf_load_btf(raw_btf, ptr - raw_btf, 0, 0, 0); 556 free(raw_btf); 557 if (btf_fd < 0) 558 return -1; 559 return btf_fd; 560 } 561 562 static int create_map_spin_lock(void) 563 { 564 struct bpf_create_map_attr attr = { 565 .name = "test_map", 566 .map_type = BPF_MAP_TYPE_ARRAY, 567 .key_size = 4, 568 .value_size = 8, 569 .max_entries = 1, 570 .btf_key_type_id = 1, 571 .btf_value_type_id = 3, 572 }; 573 int fd, btf_fd; 574 575 btf_fd = load_btf(); 576 if (btf_fd < 0) 577 return -1; 578 attr.btf_fd = btf_fd; 579 fd = bpf_create_map_xattr(&attr); 580 if (fd < 0) 581 printf("Failed to create map with spin_lock\n"); 582 return fd; 583 } 584 585 static int create_sk_storage_map(void) 586 { 587 struct bpf_create_map_attr attr = { 588 .name = "test_map", 589 .map_type = BPF_MAP_TYPE_SK_STORAGE, 590 .key_size = 4, 591 .value_size = 8, 592 .max_entries = 0, 593 .map_flags = BPF_F_NO_PREALLOC, 594 .btf_key_type_id = 1, 595 .btf_value_type_id = 3, 596 }; 597 int fd, btf_fd; 598 599 btf_fd = load_btf(); 600 if (btf_fd < 0) 601 return -1; 602 attr.btf_fd = btf_fd; 603 fd = bpf_create_map_xattr(&attr); 604 close(attr.btf_fd); 605 if (fd < 0) 606 printf("Failed to create sk_storage_map\n"); 607 return fd; 608 } 609 610 static char bpf_vlog[UINT_MAX >> 8]; 611 612 static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type, 613 struct bpf_insn *prog, int *map_fds) 614 { 615 int *fixup_map_hash_8b = test->fixup_map_hash_8b; 616 int *fixup_map_hash_48b = test->fixup_map_hash_48b; 617 int *fixup_map_hash_16b = test->fixup_map_hash_16b; 618 int *fixup_map_array_48b = test->fixup_map_array_48b; 619 int *fixup_map_sockmap = test->fixup_map_sockmap; 620 int *fixup_map_sockhash = test->fixup_map_sockhash; 621 int *fixup_map_xskmap = test->fixup_map_xskmap; 622 int *fixup_map_stacktrace = test->fixup_map_stacktrace; 623 int *fixup_prog1 = test->fixup_prog1; 624 int *fixup_prog2 = test->fixup_prog2; 625 int *fixup_map_in_map = test->fixup_map_in_map; 626 int *fixup_cgroup_storage = test->fixup_cgroup_storage; 627 int *fixup_percpu_cgroup_storage = test->fixup_percpu_cgroup_storage; 628 int *fixup_map_spin_lock = test->fixup_map_spin_lock; 629 int *fixup_map_array_ro = test->fixup_map_array_ro; 630 int *fixup_map_array_wo = test->fixup_map_array_wo; 631 int *fixup_map_array_small = test->fixup_map_array_small; 632 int *fixup_sk_storage_map = test->fixup_sk_storage_map; 633 634 if (test->fill_helper) { 635 test->fill_insns = calloc(MAX_TEST_INSNS, sizeof(struct bpf_insn)); 636 test->fill_helper(test); 637 } 638 639 /* Allocating HTs with 1 elem is fine here, since we only test 640 * for verifier and not do a runtime lookup, so the only thing 641 * that really matters is value size in this case. 642 */ 643 if (*fixup_map_hash_8b) { 644 map_fds[0] = create_map(BPF_MAP_TYPE_HASH, sizeof(long long), 645 sizeof(long long), 1); 646 do { 647 prog[*fixup_map_hash_8b].imm = map_fds[0]; 648 fixup_map_hash_8b++; 649 } while (*fixup_map_hash_8b); 650 } 651 652 if (*fixup_map_hash_48b) { 653 map_fds[1] = create_map(BPF_MAP_TYPE_HASH, sizeof(long long), 654 sizeof(struct test_val), 1); 655 do { 656 prog[*fixup_map_hash_48b].imm = map_fds[1]; 657 fixup_map_hash_48b++; 658 } while (*fixup_map_hash_48b); 659 } 660 661 if (*fixup_map_hash_16b) { 662 map_fds[2] = create_map(BPF_MAP_TYPE_HASH, sizeof(long long), 663 sizeof(struct other_val), 1); 664 do { 665 prog[*fixup_map_hash_16b].imm = map_fds[2]; 666 fixup_map_hash_16b++; 667 } while (*fixup_map_hash_16b); 668 } 669 670 if (*fixup_map_array_48b) { 671 map_fds[3] = create_map(BPF_MAP_TYPE_ARRAY, sizeof(int), 672 sizeof(struct test_val), 1); 673 update_map(map_fds[3], 0); 674 do { 675 prog[*fixup_map_array_48b].imm = map_fds[3]; 676 fixup_map_array_48b++; 677 } while (*fixup_map_array_48b); 678 } 679 680 if (*fixup_prog1) { 681 map_fds[4] = create_prog_array(prog_type, 4, 0); 682 do { 683 prog[*fixup_prog1].imm = map_fds[4]; 684 fixup_prog1++; 685 } while (*fixup_prog1); 686 } 687 688 if (*fixup_prog2) { 689 map_fds[5] = create_prog_array(prog_type, 8, 7); 690 do { 691 prog[*fixup_prog2].imm = map_fds[5]; 692 fixup_prog2++; 693 } while (*fixup_prog2); 694 } 695 696 if (*fixup_map_in_map) { 697 map_fds[6] = create_map_in_map(); 698 do { 699 prog[*fixup_map_in_map].imm = map_fds[6]; 700 fixup_map_in_map++; 701 } while (*fixup_map_in_map); 702 } 703 704 if (*fixup_cgroup_storage) { 705 map_fds[7] = create_cgroup_storage(false); 706 do { 707 prog[*fixup_cgroup_storage].imm = map_fds[7]; 708 fixup_cgroup_storage++; 709 } while (*fixup_cgroup_storage); 710 } 711 712 if (*fixup_percpu_cgroup_storage) { 713 map_fds[8] = create_cgroup_storage(true); 714 do { 715 prog[*fixup_percpu_cgroup_storage].imm = map_fds[8]; 716 fixup_percpu_cgroup_storage++; 717 } while (*fixup_percpu_cgroup_storage); 718 } 719 if (*fixup_map_sockmap) { 720 map_fds[9] = create_map(BPF_MAP_TYPE_SOCKMAP, sizeof(int), 721 sizeof(int), 1); 722 do { 723 prog[*fixup_map_sockmap].imm = map_fds[9]; 724 fixup_map_sockmap++; 725 } while (*fixup_map_sockmap); 726 } 727 if (*fixup_map_sockhash) { 728 map_fds[10] = create_map(BPF_MAP_TYPE_SOCKHASH, sizeof(int), 729 sizeof(int), 1); 730 do { 731 prog[*fixup_map_sockhash].imm = map_fds[10]; 732 fixup_map_sockhash++; 733 } while (*fixup_map_sockhash); 734 } 735 if (*fixup_map_xskmap) { 736 map_fds[11] = create_map(BPF_MAP_TYPE_XSKMAP, sizeof(int), 737 sizeof(int), 1); 738 do { 739 prog[*fixup_map_xskmap].imm = map_fds[11]; 740 fixup_map_xskmap++; 741 } while (*fixup_map_xskmap); 742 } 743 if (*fixup_map_stacktrace) { 744 map_fds[12] = create_map(BPF_MAP_TYPE_STACK_TRACE, sizeof(u32), 745 sizeof(u64), 1); 746 do { 747 prog[*fixup_map_stacktrace].imm = map_fds[12]; 748 fixup_map_stacktrace++; 749 } while (*fixup_map_stacktrace); 750 } 751 if (*fixup_map_spin_lock) { 752 map_fds[13] = create_map_spin_lock(); 753 do { 754 prog[*fixup_map_spin_lock].imm = map_fds[13]; 755 fixup_map_spin_lock++; 756 } while (*fixup_map_spin_lock); 757 } 758 if (*fixup_map_array_ro) { 759 map_fds[14] = __create_map(BPF_MAP_TYPE_ARRAY, sizeof(int), 760 sizeof(struct test_val), 1, 761 BPF_F_RDONLY_PROG); 762 update_map(map_fds[14], 0); 763 do { 764 prog[*fixup_map_array_ro].imm = map_fds[14]; 765 fixup_map_array_ro++; 766 } while (*fixup_map_array_ro); 767 } 768 if (*fixup_map_array_wo) { 769 map_fds[15] = __create_map(BPF_MAP_TYPE_ARRAY, sizeof(int), 770 sizeof(struct test_val), 1, 771 BPF_F_WRONLY_PROG); 772 update_map(map_fds[15], 0); 773 do { 774 prog[*fixup_map_array_wo].imm = map_fds[15]; 775 fixup_map_array_wo++; 776 } while (*fixup_map_array_wo); 777 } 778 if (*fixup_map_array_small) { 779 map_fds[16] = __create_map(BPF_MAP_TYPE_ARRAY, sizeof(int), 780 1, 1, 0); 781 update_map(map_fds[16], 0); 782 do { 783 prog[*fixup_map_array_small].imm = map_fds[16]; 784 fixup_map_array_small++; 785 } while (*fixup_map_array_small); 786 } 787 if (*fixup_sk_storage_map) { 788 map_fds[17] = create_sk_storage_map(); 789 do { 790 prog[*fixup_sk_storage_map].imm = map_fds[17]; 791 fixup_sk_storage_map++; 792 } while (*fixup_sk_storage_map); 793 } 794 } 795 796 static int set_admin(bool admin) 797 { 798 cap_t caps; 799 const cap_value_t cap_val = CAP_SYS_ADMIN; 800 int ret = -1; 801 802 caps = cap_get_proc(); 803 if (!caps) { 804 perror("cap_get_proc"); 805 return -1; 806 } 807 if (cap_set_flag(caps, CAP_EFFECTIVE, 1, &cap_val, 808 admin ? CAP_SET : CAP_CLEAR)) { 809 perror("cap_set_flag"); 810 goto out; 811 } 812 if (cap_set_proc(caps)) { 813 perror("cap_set_proc"); 814 goto out; 815 } 816 ret = 0; 817 out: 818 if (cap_free(caps)) 819 perror("cap_free"); 820 return ret; 821 } 822 823 static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val, 824 void *data, size_t size_data) 825 { 826 __u8 tmp[TEST_DATA_LEN << 2]; 827 __u32 size_tmp = sizeof(tmp); 828 uint32_t retval; 829 int err; 830 831 if (unpriv) 832 set_admin(true); 833 err = bpf_prog_test_run(fd_prog, 1, data, size_data, 834 tmp, &size_tmp, &retval, NULL); 835 if (unpriv) 836 set_admin(false); 837 if (err && errno != 524/*ENOTSUPP*/ && errno != EPERM) { 838 printf("Unexpected bpf_prog_test_run error "); 839 return err; 840 } 841 if (!err && retval != expected_val && 842 expected_val != POINTER_VALUE) { 843 printf("FAIL retval %d != %d ", retval, expected_val); 844 return 1; 845 } 846 847 return 0; 848 } 849 850 static void do_test_single(struct bpf_test *test, bool unpriv, 851 int *passes, int *errors) 852 { 853 int fd_prog, expected_ret, alignment_prevented_execution; 854 int prog_len, prog_type = test->prog_type; 855 struct bpf_insn *prog = test->insns; 856 int run_errs, run_successes; 857 int map_fds[MAX_NR_MAPS]; 858 const char *expected_err; 859 int fixup_skips; 860 __u32 pflags; 861 int i, err; 862 863 for (i = 0; i < MAX_NR_MAPS; i++) 864 map_fds[i] = -1; 865 866 if (!prog_type) 867 prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 868 fixup_skips = skips; 869 do_test_fixup(test, prog_type, prog, map_fds); 870 if (test->fill_insns) { 871 prog = test->fill_insns; 872 prog_len = test->prog_len; 873 } else { 874 prog_len = probe_filter_length(prog); 875 } 876 /* If there were some map skips during fixup due to missing bpf 877 * features, skip this test. 878 */ 879 if (fixup_skips != skips) 880 return; 881 882 pflags = BPF_F_TEST_RND_HI32; 883 if (test->flags & F_LOAD_WITH_STRICT_ALIGNMENT) 884 pflags |= BPF_F_STRICT_ALIGNMENT; 885 if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS) 886 pflags |= BPF_F_ANY_ALIGNMENT; 887 fd_prog = bpf_verify_program(prog_type, prog, prog_len, pflags, 888 "GPL", 0, bpf_vlog, sizeof(bpf_vlog), 4); 889 if (fd_prog < 0 && !bpf_probe_prog_type(prog_type, 0)) { 890 printf("SKIP (unsupported program type %d)\n", prog_type); 891 skips++; 892 goto close_fds; 893 } 894 895 expected_ret = unpriv && test->result_unpriv != UNDEF ? 896 test->result_unpriv : test->result; 897 expected_err = unpriv && test->errstr_unpriv ? 898 test->errstr_unpriv : test->errstr; 899 900 alignment_prevented_execution = 0; 901 902 if (expected_ret == ACCEPT) { 903 if (fd_prog < 0) { 904 printf("FAIL\nFailed to load prog '%s'!\n", 905 strerror(errno)); 906 goto fail_log; 907 } 908 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 909 if (fd_prog >= 0 && 910 (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)) 911 alignment_prevented_execution = 1; 912 #endif 913 } else { 914 if (fd_prog >= 0) { 915 printf("FAIL\nUnexpected success to load!\n"); 916 goto fail_log; 917 } 918 if (!strstr(bpf_vlog, expected_err)) { 919 printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n", 920 expected_err, bpf_vlog); 921 goto fail_log; 922 } 923 } 924 925 if (test->insn_processed) { 926 uint32_t insn_processed; 927 char *proc; 928 929 proc = strstr(bpf_vlog, "processed "); 930 insn_processed = atoi(proc + 10); 931 if (test->insn_processed != insn_processed) { 932 printf("FAIL\nUnexpected insn_processed %u vs %u\n", 933 insn_processed, test->insn_processed); 934 goto fail_log; 935 } 936 } 937 938 run_errs = 0; 939 run_successes = 0; 940 if (!alignment_prevented_execution && fd_prog >= 0) { 941 uint32_t expected_val; 942 int i; 943 944 if (!test->runs) { 945 expected_val = unpriv && test->retval_unpriv ? 946 test->retval_unpriv : test->retval; 947 948 err = do_prog_test_run(fd_prog, unpriv, expected_val, 949 test->data, sizeof(test->data)); 950 if (err) 951 run_errs++; 952 else 953 run_successes++; 954 } 955 956 for (i = 0; i < test->runs; i++) { 957 if (unpriv && test->retvals[i].retval_unpriv) 958 expected_val = test->retvals[i].retval_unpriv; 959 else 960 expected_val = test->retvals[i].retval; 961 962 err = do_prog_test_run(fd_prog, unpriv, expected_val, 963 test->retvals[i].data, 964 sizeof(test->retvals[i].data)); 965 if (err) { 966 printf("(run %d/%d) ", i + 1, test->runs); 967 run_errs++; 968 } else { 969 run_successes++; 970 } 971 } 972 } 973 974 if (!run_errs) { 975 (*passes)++; 976 if (run_successes > 1) 977 printf("%d cases ", run_successes); 978 printf("OK"); 979 if (alignment_prevented_execution) 980 printf(" (NOTE: not executed due to unknown alignment)"); 981 printf("\n"); 982 } else { 983 printf("\n"); 984 goto fail_log; 985 } 986 close_fds: 987 if (test->fill_insns) 988 free(test->fill_insns); 989 close(fd_prog); 990 for (i = 0; i < MAX_NR_MAPS; i++) 991 close(map_fds[i]); 992 sched_yield(); 993 return; 994 fail_log: 995 (*errors)++; 996 printf("%s", bpf_vlog); 997 goto close_fds; 998 } 999 1000 static bool is_admin(void) 1001 { 1002 cap_t caps; 1003 cap_flag_value_t sysadmin = CAP_CLEAR; 1004 const cap_value_t cap_val = CAP_SYS_ADMIN; 1005 1006 #ifdef CAP_IS_SUPPORTED 1007 if (!CAP_IS_SUPPORTED(CAP_SETFCAP)) { 1008 perror("cap_get_flag"); 1009 return false; 1010 } 1011 #endif 1012 caps = cap_get_proc(); 1013 if (!caps) { 1014 perror("cap_get_proc"); 1015 return false; 1016 } 1017 if (cap_get_flag(caps, cap_val, CAP_EFFECTIVE, &sysadmin)) 1018 perror("cap_get_flag"); 1019 if (cap_free(caps)) 1020 perror("cap_free"); 1021 return (sysadmin == CAP_SET); 1022 } 1023 1024 static void get_unpriv_disabled() 1025 { 1026 char buf[2]; 1027 FILE *fd; 1028 1029 fd = fopen("/proc/sys/"UNPRIV_SYSCTL, "r"); 1030 if (!fd) { 1031 perror("fopen /proc/sys/"UNPRIV_SYSCTL); 1032 unpriv_disabled = true; 1033 return; 1034 } 1035 if (fgets(buf, 2, fd) == buf && atoi(buf)) 1036 unpriv_disabled = true; 1037 fclose(fd); 1038 } 1039 1040 static bool test_as_unpriv(struct bpf_test *test) 1041 { 1042 return !test->prog_type || 1043 test->prog_type == BPF_PROG_TYPE_SOCKET_FILTER || 1044 test->prog_type == BPF_PROG_TYPE_CGROUP_SKB; 1045 } 1046 1047 static int do_test(bool unpriv, unsigned int from, unsigned int to) 1048 { 1049 int i, passes = 0, errors = 0; 1050 1051 for (i = from; i < to; i++) { 1052 struct bpf_test *test = &tests[i]; 1053 1054 /* Program types that are not supported by non-root we 1055 * skip right away. 1056 */ 1057 if (test_as_unpriv(test) && unpriv_disabled) { 1058 printf("#%d/u %s SKIP\n", i, test->descr); 1059 skips++; 1060 } else if (test_as_unpriv(test)) { 1061 if (!unpriv) 1062 set_admin(false); 1063 printf("#%d/u %s ", i, test->descr); 1064 do_test_single(test, true, &passes, &errors); 1065 if (!unpriv) 1066 set_admin(true); 1067 } 1068 1069 if (unpriv) { 1070 printf("#%d/p %s SKIP\n", i, test->descr); 1071 skips++; 1072 } else { 1073 printf("#%d/p %s ", i, test->descr); 1074 do_test_single(test, false, &passes, &errors); 1075 } 1076 } 1077 1078 printf("Summary: %d PASSED, %d SKIPPED, %d FAILED\n", passes, 1079 skips, errors); 1080 return errors ? EXIT_FAILURE : EXIT_SUCCESS; 1081 } 1082 1083 int main(int argc, char **argv) 1084 { 1085 unsigned int from = 0, to = ARRAY_SIZE(tests); 1086 bool unpriv = !is_admin(); 1087 1088 if (argc == 3) { 1089 unsigned int l = atoi(argv[argc - 2]); 1090 unsigned int u = atoi(argv[argc - 1]); 1091 1092 if (l < to && u < to) { 1093 from = l; 1094 to = u + 1; 1095 } 1096 } else if (argc == 2) { 1097 unsigned int t = atoi(argv[argc - 1]); 1098 1099 if (t < to) { 1100 from = t; 1101 to = t + 1; 1102 } 1103 } 1104 1105 get_unpriv_disabled(); 1106 if (unpriv && unpriv_disabled) { 1107 printf("Cannot run as unprivileged user with sysctl %s.\n", 1108 UNPRIV_SYSCTL); 1109 return EXIT_FAILURE; 1110 } 1111 1112 bpf_semi_rand_init(); 1113 return do_test(unpriv, from, to); 1114 } 1115