1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <bpf/bpf.h> 4 #include <bpf/btf.h> 5 #include <test_progs.h> 6 7 #if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) || \ 8 (defined(__riscv) && __riscv_xlen == 64) 9 static int map_create(__u32 map_type, __u32 max_entries) 10 { 11 const char *map_name = "insn_array"; 12 __u32 key_size = 4; 13 __u32 value_size = sizeof(struct bpf_insn_array_value); 14 15 return bpf_map_create(map_type, map_name, key_size, value_size, max_entries, NULL); 16 } 17 18 static int prog_load(struct bpf_insn *insns, __u32 insn_cnt, int *fd_array, __u32 fd_array_cnt) 19 { 20 LIBBPF_OPTS(bpf_prog_load_opts, opts); 21 22 opts.fd_array = fd_array; 23 opts.fd_array_cnt = fd_array_cnt; 24 25 return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, &opts); 26 } 27 28 static void __check_success(struct bpf_insn *insns, __u32 insn_cnt, __u32 *map_in, __u32 *map_out) 29 { 30 struct bpf_insn_array_value val = {}; 31 int prog_fd = -1, map_fd, i; 32 33 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, insn_cnt); 34 if (!ASSERT_GE(map_fd, 0, "map_create")) 35 return; 36 37 for (i = 0; i < insn_cnt; i++) { 38 val.orig_off = map_in[i]; 39 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 40 goto cleanup; 41 } 42 43 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 44 goto cleanup; 45 46 prog_fd = prog_load(insns, insn_cnt, &map_fd, 1); 47 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 48 goto cleanup; 49 50 for (i = 0; i < insn_cnt; i++) { 51 char buf[64]; 52 53 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 54 goto cleanup; 55 56 snprintf(buf, sizeof(buf), "val.xlated_off should be equal map_out[%d]", i); 57 ASSERT_EQ(val.xlated_off, map_out[i], buf); 58 } 59 60 cleanup: 61 close(prog_fd); 62 close(map_fd); 63 } 64 65 /* 66 * Load a program, which will not be anyhow mangled by the verifier. Add an 67 * insn_array map pointing to every instruction. Check that it hasn't changed 68 * after the program load. 69 */ 70 static void check_one_to_one_mapping(void) 71 { 72 struct bpf_insn insns[] = { 73 BPF_MOV64_IMM(BPF_REG_0, 4), 74 BPF_MOV64_IMM(BPF_REG_0, 3), 75 BPF_MOV64_IMM(BPF_REG_0, 2), 76 BPF_MOV64_IMM(BPF_REG_0, 1), 77 BPF_MOV64_IMM(BPF_REG_0, 0), 78 BPF_EXIT_INSN(), 79 }; 80 __u32 map_in[] = {0, 1, 2, 3, 4, 5}; 81 __u32 map_out[] = {0, 1, 2, 3, 4, 5}; 82 83 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 84 } 85 86 /* 87 * Load a program with two patches (get jiffies, for simplicity). Add an 88 * insn_array map pointing to every instruction. Check how it was changed 89 * after the program load. 90 */ 91 static void check_simple(void) 92 { 93 struct bpf_insn insns[] = { 94 BPF_MOV64_IMM(BPF_REG_0, 2), 95 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 96 BPF_MOV64_IMM(BPF_REG_0, 1), 97 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 98 BPF_MOV64_IMM(BPF_REG_0, 0), 99 BPF_EXIT_INSN(), 100 }; 101 __u32 map_in[] = {0, 1, 2, 3, 4, 5}; 102 __u32 map_out[] = {0, 1, 4, 5, 8, 9}; 103 104 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 105 } 106 107 /* 108 * Verifier can delete code in two cases: nops & dead code. From insn 109 * array's point of view, the two cases are the same, so test using 110 * the simplest method: by loading some nops 111 */ 112 static void check_deletions(void) 113 { 114 struct bpf_insn insns[] = { 115 BPF_MOV64_IMM(BPF_REG_0, 2), 116 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 117 BPF_MOV64_IMM(BPF_REG_0, 1), 118 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 119 BPF_MOV64_IMM(BPF_REG_0, 0), 120 BPF_EXIT_INSN(), 121 }; 122 __u32 map_in[] = {0, 1, 2, 3, 4, 5}; 123 __u32 map_out[] = {0, -1, 1, -1, 2, 3}; 124 125 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 126 } 127 128 /* 129 * Same test as check_deletions, but also add code which adds instructions 130 */ 131 static void check_deletions_with_functions(void) 132 { 133 struct bpf_insn insns[] = { 134 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 135 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 136 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 137 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), 138 BPF_MOV64_IMM(BPF_REG_0, 1), 139 BPF_EXIT_INSN(), 140 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 141 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 142 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 143 BPF_MOV64_IMM(BPF_REG_0, 2), 144 BPF_EXIT_INSN(), 145 }; 146 __u32 map_in[] = { 0, 1, 2, 3, 4, 5, /* func */ 6, 7, 8, 9, 10}; 147 __u32 map_out[] = {-1, 0, -1, 3, 4, 5, /* func */ -1, 6, -1, 9, 10}; 148 149 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 150 } 151 152 /* 153 * Try to load a program with a map which points to outside of the program 154 */ 155 static void check_out_of_bounds_index(void) 156 { 157 struct bpf_insn insns[] = { 158 BPF_MOV64_IMM(BPF_REG_0, 4), 159 BPF_MOV64_IMM(BPF_REG_0, 3), 160 BPF_MOV64_IMM(BPF_REG_0, 2), 161 BPF_MOV64_IMM(BPF_REG_0, 1), 162 BPF_MOV64_IMM(BPF_REG_0, 0), 163 BPF_EXIT_INSN(), 164 }; 165 int prog_fd, map_fd; 166 struct bpf_insn_array_value val = {}; 167 int key; 168 169 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 170 if (!ASSERT_GE(map_fd, 0, "map_create")) 171 return; 172 173 key = 0; 174 val.orig_off = ARRAY_SIZE(insns); /* too big */ 175 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) 176 goto cleanup; 177 178 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 179 goto cleanup; 180 181 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 182 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) { 183 close(prog_fd); 184 goto cleanup; 185 } 186 187 cleanup: 188 close(map_fd); 189 } 190 191 /* 192 * Try to load a program with a map which points to the middle of 16-bit insn 193 */ 194 static void check_mid_insn_index(void) 195 { 196 struct bpf_insn insns[] = { 197 BPF_LD_IMM64(BPF_REG_0, 0), /* 2 x 8 */ 198 BPF_EXIT_INSN(), 199 }; 200 int prog_fd, map_fd; 201 struct bpf_insn_array_value val = {}; 202 int key; 203 204 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 205 if (!ASSERT_GE(map_fd, 0, "map_create")) 206 return; 207 208 key = 0; 209 val.orig_off = 1; /* middle of 16-byte instruction */ 210 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) 211 goto cleanup; 212 213 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 214 goto cleanup; 215 216 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 217 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) { 218 close(prog_fd); 219 goto cleanup; 220 } 221 222 cleanup: 223 close(map_fd); 224 } 225 226 static void check_incorrect_index(void) 227 { 228 check_out_of_bounds_index(); 229 check_mid_insn_index(); 230 } 231 232 static int set_bpf_jit_harden(char *level) 233 { 234 char old_level; 235 int err = -1; 236 int fd = -1; 237 238 fd = open("/proc/sys/net/core/bpf_jit_harden", O_RDWR | O_NONBLOCK); 239 if (fd < 0) { 240 ASSERT_FAIL("open .../bpf_jit_harden returned %d (errno=%d)", fd, errno); 241 return -1; 242 } 243 244 err = read(fd, &old_level, 1); 245 if (err != 1) { 246 ASSERT_FAIL("read from .../bpf_jit_harden returned %d (errno=%d)", err, errno); 247 err = -1; 248 goto end; 249 } 250 251 lseek(fd, 0, SEEK_SET); 252 253 err = write(fd, level, 1); 254 if (err != 1) { 255 ASSERT_FAIL("write to .../bpf_jit_harden returned %d (errno=%d)", err, errno); 256 err = -1; 257 goto end; 258 } 259 260 err = 0; 261 *level = old_level; 262 end: 263 if (fd >= 0) 264 close(fd); 265 return err; 266 } 267 268 static void check_blindness(void) 269 { 270 struct bpf_insn insns[] = { 271 BPF_MOV64_IMM(BPF_REG_0, 4), 272 BPF_MOV64_IMM(BPF_REG_0, 3), 273 BPF_MOV64_IMM(BPF_REG_0, 2), 274 BPF_MOV64_IMM(BPF_REG_0, 1), 275 BPF_EXIT_INSN(), 276 }; 277 int prog_fd = -1, map_fd; 278 struct bpf_insn_array_value val = {}; 279 char bpf_jit_harden = '@'; /* non-exizsting value */ 280 int i; 281 282 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, ARRAY_SIZE(insns)); 283 if (!ASSERT_GE(map_fd, 0, "map_create")) 284 return; 285 286 for (i = 0; i < ARRAY_SIZE(insns); i++) { 287 val.orig_off = i; 288 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 289 goto cleanup; 290 } 291 292 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 293 goto cleanup; 294 295 bpf_jit_harden = '2'; 296 if (set_bpf_jit_harden(&bpf_jit_harden)) { 297 bpf_jit_harden = '@'; /* open, read or write failed => no write was done */ 298 goto cleanup; 299 } 300 301 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 302 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 303 goto cleanup; 304 305 for (i = 0; i < ARRAY_SIZE(insns); i++) { 306 char fmt[32]; 307 308 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 309 goto cleanup; 310 311 snprintf(fmt, sizeof(fmt), "val should be equal 3*%d", i); 312 ASSERT_EQ(val.xlated_off, i * 3, fmt); 313 } 314 315 cleanup: 316 /* restore the old one */ 317 if (bpf_jit_harden != '@') 318 set_bpf_jit_harden(&bpf_jit_harden); 319 320 close(prog_fd); 321 close(map_fd); 322 } 323 324 /* Once map was initialized, it should be frozen */ 325 static void check_load_unfrozen_map(void) 326 { 327 struct bpf_insn insns[] = { 328 BPF_MOV64_IMM(BPF_REG_0, 0), 329 BPF_EXIT_INSN(), 330 }; 331 int prog_fd = -1, map_fd; 332 struct bpf_insn_array_value val = {}; 333 int i; 334 335 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, ARRAY_SIZE(insns)); 336 if (!ASSERT_GE(map_fd, 0, "map_create")) 337 return; 338 339 for (i = 0; i < ARRAY_SIZE(insns); i++) { 340 val.orig_off = i; 341 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 342 goto cleanup; 343 } 344 345 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 346 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) 347 goto cleanup; 348 349 /* correctness: now freeze the map, the program should load fine */ 350 351 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 352 goto cleanup; 353 354 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 355 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 356 goto cleanup; 357 358 for (i = 0; i < ARRAY_SIZE(insns); i++) { 359 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 360 goto cleanup; 361 362 ASSERT_EQ(val.xlated_off, i, "val should be equal i"); 363 } 364 365 cleanup: 366 close(prog_fd); 367 close(map_fd); 368 } 369 370 /* Map can be used only by one BPF program */ 371 static void check_no_map_reuse(void) 372 { 373 struct bpf_insn insns[] = { 374 BPF_MOV64_IMM(BPF_REG_0, 0), 375 BPF_EXIT_INSN(), 376 }; 377 int prog_fd = -1, map_fd, extra_fd = -1; 378 struct bpf_insn_array_value val = {}; 379 int i; 380 381 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, ARRAY_SIZE(insns)); 382 if (!ASSERT_GE(map_fd, 0, "map_create")) 383 return; 384 385 for (i = 0; i < ARRAY_SIZE(insns); i++) { 386 val.orig_off = i; 387 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 388 goto cleanup; 389 } 390 391 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 392 goto cleanup; 393 394 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 395 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 396 goto cleanup; 397 398 for (i = 0; i < ARRAY_SIZE(insns); i++) { 399 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 400 goto cleanup; 401 402 ASSERT_EQ(val.xlated_off, i, "val should be equal i"); 403 } 404 405 extra_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 406 if (!ASSERT_EQ(extra_fd, -EBUSY, "program should have been rejected (extra_fd != -EBUSY)")) 407 goto cleanup; 408 409 /* correctness: check that prog is still loadable without fd_array */ 410 extra_fd = prog_load(insns, ARRAY_SIZE(insns), NULL, 0); 411 if (!ASSERT_GE(extra_fd, 0, "bpf(BPF_PROG_LOAD): expected no error")) 412 goto cleanup; 413 414 cleanup: 415 close(extra_fd); 416 close(prog_fd); 417 close(map_fd); 418 } 419 420 static void check_bpf_no_lookup(void) 421 { 422 struct bpf_insn insns[] = { 423 BPF_LD_MAP_FD(BPF_REG_1, 0), 424 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0), 425 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), 426 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8), 427 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 428 BPF_EXIT_INSN(), 429 }; 430 int prog_fd = -1, map_fd; 431 432 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 433 if (!ASSERT_GE(map_fd, 0, "map_create")) 434 return; 435 436 insns[0].imm = map_fd; 437 438 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 439 goto cleanup; 440 441 prog_fd = prog_load(insns, ARRAY_SIZE(insns), NULL, 0); 442 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) 443 goto cleanup; 444 445 /* correctness: check that prog is still loadable with normal map */ 446 close(map_fd); 447 map_fd = map_create(BPF_MAP_TYPE_ARRAY, 1); 448 insns[0].imm = map_fd; 449 prog_fd = prog_load(insns, ARRAY_SIZE(insns), NULL, 0); 450 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 451 goto cleanup; 452 453 cleanup: 454 close(prog_fd); 455 close(map_fd); 456 } 457 458 #define GOTOX_LOG_SZ (256 * 1024) 459 460 /* Load and return the fd, which the caller owns. */ 461 static int gotox_prog_load_funcs_fd(struct bpf_insn *insns, __u32 insn_cnt, 462 int *fd_array, __u32 fd_array_cnt, char *log, 463 int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) 464 { 465 LIBBPF_OPTS(bpf_prog_load_opts, opts); 466 467 log[0] = 0; 468 opts.fd_array = fd_array; 469 opts.fd_array_cnt = fd_array_cnt; 470 opts.log_buf = log; 471 opts.log_size = GOTOX_LOG_SZ; 472 opts.log_level = 1; 473 if (fi_cnt) { 474 opts.prog_btf_fd = btf_fd; 475 opts.func_info = fi; 476 opts.func_info_cnt = fi_cnt; 477 opts.func_info_rec_size = sizeof(*fi); 478 } 479 480 return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, &opts); 481 } 482 483 /* Load for the outcome only: 0 or the negative errno. */ 484 static int gotox_prog_load_funcs(struct bpf_insn *insns, __u32 insn_cnt, 485 int *fd_array, __u32 fd_array_cnt, char *log, 486 int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) 487 { 488 int prog_fd; 489 490 prog_fd = gotox_prog_load_funcs_fd(insns, insn_cnt, fd_array, fd_array_cnt, log, 491 btf_fd, fi, fi_cnt); 492 if (prog_fd < 0) 493 return prog_fd; 494 close(prog_fd); 495 return 0; 496 } 497 498 static int gotox_prog_load(struct bpf_insn *insns, __u32 insn_cnt, 499 int *fd_array, __u32 fd_array_cnt, char *log) 500 { 501 return gotox_prog_load_funcs(insns, insn_cnt, fd_array, fd_array_cnt, log, 502 -1, NULL, 0); 503 } 504 505 static int gotox_jt_create_offs(const __u32 *offs, __u32 cnt) 506 { 507 struct bpf_insn_array_value val = {}; 508 int map_fd; 509 __u32 i; 510 511 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, cnt); 512 if (!ASSERT_GE(map_fd, 0, "map_create")) 513 return map_fd; 514 515 for (i = 0; i < cnt; i++) { 516 val.orig_off = offs[i]; 517 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, 518 "bpf_map_update_elem")) 519 goto err; 520 } 521 522 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 523 goto err; 524 525 return map_fd; 526 err: 527 close(map_fd); 528 return -1; 529 } 530 531 #define GOTOX_SUB_START 4 532 #define GOTOX_MAIN_TGT 2 533 #define GOTOX_SUB_TGT 8 534 #define GOTOX_TWO_INSN_CNT 10 535 536 static void gotox_two_subprogs_fill(struct bpf_insn *insns, __u32 jt_idx, __u32 jt_off) 537 { 538 insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); 539 insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 540 GOTOX_SUB_START - 1 - 1); 541 insns[GOTOX_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 542 insns[3] = BPF_EXIT_INSN(); 543 544 /* r1 = &jt[0], by index 'jt_idx' into fd_array */ 545 insns[GOTOX_SUB_START] = (struct bpf_insn) { 546 .code = BPF_LD | BPF_DW | BPF_IMM, 547 .dst_reg = BPF_REG_1, 548 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 549 .imm = jt_idx, 550 }; 551 insns[GOTOX_SUB_START + 1] = (struct bpf_insn) { .imm = 0 }; 552 insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, jt_off * 8); 553 insns[7] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 554 insns[GOTOX_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); 555 insns[9] = BPF_EXIT_INSN(); 556 } 557 558 /* 559 * Load 'insns' with the two jump tables 'jt_a' and 'jt_b', in this order in 560 * fd_array, and check the outcome. 'fi' describes the subprograms when given. 561 */ 562 static void check_gotox_two_maps(struct bpf_insn *insns, __u32 insn_cnt, 563 const __u32 *jt_a, __u32 a_cnt, 564 const __u32 *jt_b, __u32 b_cnt, 565 int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt, 566 int expect_err, const char *expect_msg) 567 { 568 int map_fd[2] = { -1, -1 }; 569 char *log; 570 int err; 571 572 log = calloc(1, GOTOX_LOG_SZ); 573 if (!ASSERT_OK_PTR(log, "calloc log")) 574 return; 575 576 map_fd[0] = gotox_jt_create_offs(jt_a, a_cnt); 577 if (map_fd[0] < 0) 578 goto free_log; 579 map_fd[1] = gotox_jt_create_offs(jt_b, b_cnt); 580 if (map_fd[1] < 0) 581 goto close_maps; 582 583 if (fi) 584 err = gotox_prog_load_funcs(insns, insn_cnt, map_fd, 2, log, 585 btf_fd, fi, fi_cnt); 586 else 587 err = gotox_prog_load(insns, insn_cnt, map_fd, 2, log); 588 if (expect_err) 589 ASSERT_EQ(err, expect_err, "program should have been rejected"); 590 else 591 ASSERT_EQ(err, 0, "bpf(BPF_PROG_LOAD)"); 592 if (expect_msg) 593 ASSERT_HAS_SUBSTR(log, expect_msg, "verifier log"); 594 595 close_maps: 596 close(map_fd[0]); 597 if (map_fd[1] >= 0) 598 close(map_fd[1]); 599 free_log: 600 free(log); 601 } 602 603 static void check_gotox_target_other_subprog(void) 604 { 605 const __u32 jt_main[] = { GOTOX_MAIN_TGT }; 606 const __u32 jt_sub[] = { GOTOX_SUB_TGT }; 607 struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; 608 609 gotox_two_subprogs_fill(insns, 0, 0); 610 611 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 612 jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, 613 -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); 614 } 615 616 static void check_gotox_jt_per_subprog(void) 617 { 618 const __u32 jt_main[] = { GOTOX_MAIN_TGT }; 619 const __u32 jt_sub[] = { GOTOX_SUB_TGT }; 620 struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; 621 622 gotox_two_subprogs_fill(insns, 1, 0); 623 624 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 625 jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, 626 0, NULL); 627 } 628 629 #define GOTOX_DEAD_MAIN_LEN 7 630 #define GOTOX_DEAD_SUB_LEN 6 631 #define GOTOX_DEAD_INSN_CNT (GOTOX_DEAD_MAIN_LEN + 3 * GOTOX_DEAD_SUB_LEN) 632 #define GOTOX_DEAD_LIVE_CNT 10 633 634 /* r1 = &jt[0], by index 'jt_idx' into fd_array; r1 = *r1; gotox r1; r0 = 0; exit */ 635 static void gotox_sub_fill(struct bpf_insn *insns, __u32 start, __u32 jt_idx) 636 { 637 insns[start] = (struct bpf_insn) { 638 .code = BPF_LD | BPF_DW | BPF_IMM, 639 .dst_reg = BPF_REG_1, 640 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 641 .imm = jt_idx, 642 }; 643 insns[start + 1] = (struct bpf_insn) { .imm = 0 }; 644 insns[start + 2] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 645 insns[start + 3] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 646 insns[start + 4] = BPF_MOV64_IMM(BPF_REG_0, 0); 647 insns[start + 5] = BPF_EXIT_INSN(); 648 } 649 650 /* 651 * Two subprograms which are only called from dead code sit in front of the 652 * live one, and all three own a jump table. Dead code removal drops both in 653 * one go, which compacts subprog_info[] along with the tables. 654 */ 655 static void check_gotox_jt_dead_subprogs(void) 656 { 657 const __u32 sub_start[3] = { GOTOX_DEAD_MAIN_LEN, 658 GOTOX_DEAD_MAIN_LEN + GOTOX_DEAD_SUB_LEN, 659 GOTOX_DEAD_MAIN_LEN + 2 * GOTOX_DEAD_SUB_LEN }; 660 struct bpf_insn insns[GOTOX_DEAD_INSN_CNT]; 661 int map_fd[3] = { -1, -1, -1 }; 662 struct bpf_prog_info info = {}; 663 __u32 len = sizeof(info), tgt, i; 664 int prog_fd; 665 char *log; 666 667 log = calloc(1, GOTOX_LOG_SZ); 668 if (!ASSERT_OK_PTR(log, "calloc log")) 669 return; 670 671 insns[0] = BPF_MOV64_IMM(BPF_REG_1, 0); 672 insns[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2); 673 insns[2] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 674 sub_start[0] - 2 - 1); 675 insns[3] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 676 sub_start[1] - 3 - 1); 677 insns[4] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 678 sub_start[2] - 4 - 1); 679 insns[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 680 insns[6] = BPF_EXIT_INSN(); 681 682 for (i = 0; i < ARRAY_SIZE(sub_start); i++) { 683 gotox_sub_fill(insns, sub_start[i], i); 684 tgt = sub_start[i] + 4; 685 map_fd[i] = gotox_jt_create_offs(&tgt, 1); 686 if (map_fd[i] < 0) 687 goto close_maps; 688 } 689 690 prog_fd = gotox_prog_load_funcs_fd(insns, ARRAY_SIZE(insns), map_fd, 691 ARRAY_SIZE(map_fd), log, -1, NULL, 0); 692 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 693 goto close_maps; 694 695 /* Both dead subprograms and the calls to them must be gone. */ 696 if (ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), 697 "bpf_prog_get_info_by_fd")) 698 ASSERT_EQ(info.xlated_prog_len / sizeof(struct bpf_insn), 699 GOTOX_DEAD_LIVE_CNT, "xlated insn count"); 700 close(prog_fd); 701 close_maps: 702 for (i = 0; i < ARRAY_SIZE(map_fd); i++) 703 if (map_fd[i] >= 0) 704 close(map_fd[i]); 705 free(log); 706 } 707 708 #define GOTOX_FWD_GOTOX 11 709 #define GOTOX_FWD_OWN_TGT 12 710 #define GOTOX_FWD_SUB_START 14 711 #define GOTOX_FWD_INSN_CNT 16 712 713 static void gotox_from_main_fill(struct bpf_insn *insns) 714 { 715 insns[0] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); 716 insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 717 GOTOX_FWD_SUB_START - 1 - 1); 718 insns[2] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, 719 offsetof(struct xdp_md, ingress_ifindex)); 720 insns[3] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 4); 721 722 /* r1 = &jt_leaves[0], by index 1 into fd_array */ 723 insns[4] = (struct bpf_insn) { 724 .code = BPF_LD | BPF_DW | BPF_IMM, 725 .dst_reg = BPF_REG_1, 726 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 727 .imm = 1, 728 }; 729 insns[5] = (struct bpf_insn) { .imm = 0 }; 730 insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 731 insns[7] = BPF_JMP_A(3); 732 733 /* r1 = &jt_own[0], by index 0 into fd_array */ 734 insns[8] = (struct bpf_insn) { 735 .code = BPF_LD | BPF_DW | BPF_IMM, 736 .dst_reg = BPF_REG_1, 737 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 738 .imm = 0, 739 }; 740 insns[9] = (struct bpf_insn) { .imm = 0 }; 741 insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 742 743 insns[GOTOX_FWD_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 744 insns[GOTOX_FWD_OWN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 745 insns[13] = BPF_EXIT_INSN(); 746 insns[GOTOX_FWD_SUB_START] = BPF_MOV64_IMM(BPF_REG_0, 1); 747 insns[15] = BPF_EXIT_INSN(); 748 } 749 750 static void check_gotox_target_subprog_from_main(void) 751 { 752 const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; 753 const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; 754 struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; 755 756 gotox_from_main_fill(insns); 757 758 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), 759 jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, 760 -EINVAL, "indirect jump from insn 11 to 14 leaves the subprog [0,14)"); 761 } 762 763 /* 764 * The only map of the subprog holding the gotox reaches past that subprog, so 765 * the subprog is left without a jump table at all. 766 */ 767 static void check_gotox_jt_spans_subprogs(void) 768 { 769 const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; 770 const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; 771 struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; 772 773 gotox_from_main_fill(insns); 774 775 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_span, ARRAY_SIZE(jt_span), 776 jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, 777 -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); 778 } 779 780 /* 781 * The subprog holding the gotox has a well formed jump table of its own and 782 * also collects a map that reaches past its end. The spanning map is still 783 * rejected, even though the subprog is not left without a table. 784 */ 785 static void check_gotox_jt_spans_with_own_table(void) 786 { 787 const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; 788 const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; 789 struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; 790 791 gotox_from_main_fill(insns); 792 793 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), 794 jt_span, ARRAY_SIZE(jt_span), -1, NULL, 0, 795 -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); 796 } 797 798 #define GOTOX_SLICE_SUB_START 6 799 #define GOTOX_SLICE_GOTOX 14 800 #define GOTOX_SLICE_SUB_TGT 15 801 #define GOTOX_SLICE_INSN_CNT 17 802 803 static void gotox_slice_fill(struct bpf_insn *insns) 804 { 805 insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); 806 insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 807 GOTOX_SLICE_SUB_START - 1 - 1); 808 insns[2] = BPF_MOV64_IMM(BPF_REG_0, 0); 809 insns[3] = BPF_MOV64_IMM(BPF_REG_0, 0); 810 insns[4] = BPF_MOV64_IMM(BPF_REG_0, 0); 811 insns[5] = BPF_EXIT_INSN(); 812 813 insns[GOTOX_SLICE_SUB_START] = 814 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 815 offsetof(struct xdp_md, ingress_ifindex)); 816 insns[7] = BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 1); 817 insns[8] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1); 818 insns[9] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 819 820 /* r1 = &jt_main[0], by index 0 into fd_array */ 821 insns[10] = (struct bpf_insn) { 822 .code = BPF_LD | BPF_DW | BPF_IMM, 823 .dst_reg = BPF_REG_1, 824 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 825 .imm = 0, 826 }; 827 insns[11] = (struct bpf_insn) { .imm = 0 }; 828 insns[12] = BPF_ALU64_REG(BPF_ADD, BPF_REG_1, BPF_REG_2); 829 insns[13] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 830 831 insns[GOTOX_SLICE_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 832 insns[GOTOX_SLICE_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); 833 insns[16] = BPF_EXIT_INSN(); 834 } 835 836 static void check_gotox_index_slice_other_subprog(void) 837 { 838 const __u32 jt_main[] = { 2, 3, 4 }; 839 const __u32 jt_sub[] = { GOTOX_SLICE_SUB_TGT }; 840 struct bpf_insn insns[GOTOX_SLICE_INSN_CNT]; 841 842 gotox_slice_fill(insns); 843 844 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 845 jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, 846 -EINVAL, "indirect jump from insn 14 to 3 leaves the subprog [6,17)"); 847 } 848 849 static int gotox_btf_create(const __u32 *starts, const __u8 *linkage, __u32 cnt, 850 struct bpf_func_info *fi, struct btf **pbtf) 851 { 852 int int_id, proto_id, id; 853 struct btf *btf; 854 char name[24]; 855 __u32 i; 856 857 btf = btf__new_empty(); 858 if (!ASSERT_OK_PTR(btf, "btf__new_empty")) 859 return -1; 860 861 int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); 862 if (!ASSERT_GT(int_id, 0, "btf__add_int")) 863 goto err; 864 865 proto_id = btf__add_func_proto(btf, int_id); 866 if (!ASSERT_GT(proto_id, 0, "btf__add_func_proto")) 867 goto err; 868 869 for (i = 0; i < cnt; i++) { 870 snprintf(name, sizeof(name), "gotox_f%u", i); 871 id = btf__add_func(btf, name, linkage[i], proto_id); 872 if (!ASSERT_GT(id, 0, "btf__add_func")) 873 goto err; 874 fi[i].insn_off = starts[i]; 875 fi[i].type_id = id; 876 } 877 878 if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) 879 goto err; 880 881 *pbtf = btf; 882 return btf__fd(btf); 883 err: 884 btf__free(btf); 885 return -1; 886 } 887 888 static void check_gotox_target_other_global_subprog(void) 889 { 890 const __u32 starts[] = { 0, GOTOX_SUB_START }; 891 const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_GLOBAL }; 892 const __u32 jt_main[] = { GOTOX_MAIN_TGT }; 893 const __u32 jt_sub[] = { GOTOX_SUB_TGT }; 894 struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; 895 struct bpf_func_info fi[2]; 896 struct btf *btf = NULL; 897 int btf_fd; 898 899 gotox_two_subprogs_fill(insns, 0, 0); 900 901 btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); 902 if (btf_fd < 0) 903 return; 904 905 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 906 jt_sub, ARRAY_SIZE(jt_sub), btf_fd, fi, ARRAY_SIZE(fi), 907 -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); 908 btf__free(btf); 909 } 910 911 #define GOTOX_CB_MAIN_TGT 6 912 #define GOTOX_CB_START 8 913 #define GOTOX_CB_GOTOX 11 914 #define GOTOX_CB_TGT 12 915 #define GOTOX_CB_INSN_CNT 14 916 917 static void gotox_callback_fill(struct bpf_insn *insns) 918 { 919 insns[0] = BPF_MOV64_IMM(BPF_REG_1, 1); 920 /* r2 = &callback */ 921 insns[1] = (struct bpf_insn) { 922 .code = BPF_LD | BPF_DW | BPF_IMM, 923 .dst_reg = BPF_REG_2, 924 .src_reg = BPF_PSEUDO_FUNC, 925 .imm = GOTOX_CB_START - 1 - 1, 926 }; 927 insns[2] = (struct bpf_insn) { .imm = 0 }; 928 insns[3] = BPF_MOV64_IMM(BPF_REG_3, 0); 929 insns[4] = BPF_MOV64_IMM(BPF_REG_4, 0); 930 insns[5] = BPF_EMIT_CALL(BPF_FUNC_loop); 931 insns[GOTOX_CB_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 932 insns[7] = BPF_EXIT_INSN(); 933 934 /* r1 = &jt_main[0], by index 0 into fd_array */ 935 insns[GOTOX_CB_START] = (struct bpf_insn) { 936 .code = BPF_LD | BPF_DW | BPF_IMM, 937 .dst_reg = BPF_REG_1, 938 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 939 .imm = 0, 940 }; 941 insns[9] = (struct bpf_insn) { .imm = 0 }; 942 insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 943 insns[GOTOX_CB_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 944 insns[GOTOX_CB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 945 insns[13] = BPF_EXIT_INSN(); 946 } 947 948 static void check_gotox_callback_leaves_subprog(void) 949 { 950 const __u32 starts[] = { 0, GOTOX_CB_START }; 951 const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_STATIC }; 952 const __u32 jt_main[] = { GOTOX_CB_MAIN_TGT }; 953 const __u32 jt_cb[] = { GOTOX_CB_TGT }; 954 struct bpf_insn insns[GOTOX_CB_INSN_CNT]; 955 struct bpf_func_info fi[2]; 956 struct btf *btf = NULL; 957 int btf_fd; 958 959 gotox_callback_fill(insns); 960 961 btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); 962 if (btf_fd < 0) 963 return; 964 965 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 966 jt_cb, ARRAY_SIZE(jt_cb), btf_fd, fi, ARRAY_SIZE(fi), 967 -EINVAL, "indirect jump from insn 11 to 6 leaves the subprog [8,14)"); 968 btf__free(btf); 969 } 970 971 static void check_bpf_side(void) 972 { 973 check_bpf_no_lookup(); 974 } 975 976 /* Return the xlated index of the first insn equal to 'insn', or -1. */ 977 static int xlated_insn_index(int prog_fd, const struct bpf_insn *insn) 978 { 979 struct bpf_insn *xlated; 980 int idx = -1; 981 __u32 cnt, i; 982 983 if (!ASSERT_OK(get_xlated_program(prog_fd, &xlated, &cnt), "get_xlated_program")) 984 return -1; 985 for (i = 0; i < cnt; i++) 986 if (!memcmp(&xlated[i], insn, sizeof(*insn))) { 987 idx = i; 988 break; 989 } 990 free(xlated); 991 return idx; 992 } 993 994 /* 995 * Load 'insns' with a jump table naming insn 0, which a prologue prepend 996 * pushes down, and check that the slot followed it. 997 */ 998 static void check_gotox_target_prepend(enum bpf_prog_type prog_type, 999 struct bpf_insn *insns, __u32 insn_cnt) 1000 { 1001 LIBBPF_OPTS(bpf_prog_load_opts, opts); 1002 struct bpf_insn_array_value val = {}; 1003 int prog_fd = -1, map_fd, idx; 1004 __u32 key = 0; 1005 1006 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 1007 if (!ASSERT_GE(map_fd, 0, "map_create")) 1008 return; 1009 1010 val.orig_off = 0; 1011 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) 1012 goto cleanup; 1013 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 1014 goto cleanup; 1015 1016 opts.fd_array = &map_fd; 1017 opts.fd_array_cnt = 1; 1018 prog_fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insn_cnt, &opts); 1019 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 1020 goto cleanup; 1021 1022 idx = xlated_insn_index(prog_fd, &insns[0]); 1023 if (!ASSERT_GT(idx, 0, "insn 0 was prepended to")) 1024 goto cleanup; 1025 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &key, &val), 0, "bpf_map_lookup_elem")) 1026 goto cleanup; 1027 ASSERT_EQ(val.xlated_off, (__u32)idx, "gotox target followed insn 0 past the prepend"); 1028 cleanup: 1029 if (prog_fd >= 0) 1030 close(prog_fd); 1031 close(map_fd); 1032 } 1033 1034 static void check_gotox_target_prologue_shift(void) 1035 { 1036 struct bpf_insn insns[] = { 1037 /* insn 0: gotox target and subprog start */ 1038 BPF_MOV64_IMM(BPF_REG_0, 0), 1039 /* may_goto +4 -> exit block, bounds the loop */ 1040 BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 4, 0), 1041 /* r1 = &jt[0] (insns 2 and 3) */ 1042 BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 0), 1043 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0), /* r1 = ips[0] */ 1044 /* insn 5: gotox r1 -> insn 0 */ 1045 BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0), 1046 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), /* insn 6: exit block */ 1047 BPF_EXIT_INSN(), 1048 }; 1049 1050 check_gotox_target_prepend(BPF_PROG_TYPE_XDP, insns, ARRAY_SIZE(insns)); 1051 } 1052 1053 static void check_gotox_target_ctx_prologue_shift(void) 1054 { 1055 struct bpf_insn insns[] = { 1056 /* insn 0: gotox target and subprog start */ 1057 BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, -8), 1058 BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, 0x5a5a, 12), /* second visit exits */ 1059 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0x5a5a), 1060 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), /* r6 = ctx */ 1061 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, offsetof(struct __sk_buff, data)), 1062 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_6, offsetof(struct __sk_buff, data_end)), 1063 BPF_MOV64_REG(BPF_REG_4, BPF_REG_2), 1064 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 1), 1065 BPF_JMP_REG(BPF_JGT, BPF_REG_4, BPF_REG_3, 5), 1066 /* insn 9: direct packet write -> tc unclone prologue prepended at insn 0 */ 1067 BPF_ST_MEM(BPF_B, BPF_REG_2, 0, 0), 1068 /* r5 = &jt[0] (insns 10 and 11) */ 1069 BPF_LD_IMM64_RAW(BPF_REG_5, BPF_PSEUDO_MAP_IDX_VALUE, 0), 1070 BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_5, 0), /* r5 = ips[0] */ 1071 /* insn 13: gotox r5 -> insn 0 */ 1072 BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_5, 0, 0, 0), 1073 BPF_MOV64_IMM(BPF_REG_0, 0), /* insn 14: exit block */ 1074 BPF_EXIT_INSN(), 1075 }; 1076 1077 check_gotox_target_prepend(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns)); 1078 } 1079 1080 static void __test_bpf_insn_array(void) 1081 { 1082 /* Test if offsets are adjusted properly */ 1083 if (test__start_subtest("one2one")) 1084 check_one_to_one_mapping(); 1085 if (test__start_subtest("simple")) 1086 check_simple(); 1087 if (test__start_subtest("deletions")) 1088 check_deletions(); 1089 if (test__start_subtest("deletions-with-functions")) 1090 check_deletions_with_functions(); 1091 if (test__start_subtest("blindness")) 1092 check_blindness(); 1093 /* Check all kinds of operations and related restrictions */ 1094 if (test__start_subtest("incorrect-index")) 1095 check_incorrect_index(); 1096 if (test__start_subtest("load-unfrozen-map")) 1097 check_load_unfrozen_map(); 1098 if (test__start_subtest("no-map-reuse")) 1099 check_no_map_reuse(); 1100 if (test__start_subtest("bpf-side-ops")) 1101 check_bpf_side(); 1102 if (test__start_subtest("gotox-jt-spans-subprogs")) 1103 check_gotox_jt_spans_subprogs(); 1104 if (test__start_subtest("gotox-jt-spans-with-own-table")) 1105 check_gotox_jt_spans_with_own_table(); 1106 if (test__start_subtest("gotox-target-other-subprog")) 1107 check_gotox_target_other_subprog(); 1108 if (test__start_subtest("gotox-jt-per-subprog")) 1109 check_gotox_jt_per_subprog(); 1110 if (test__start_subtest("gotox-jt-dead-subprogs")) 1111 check_gotox_jt_dead_subprogs(); 1112 if (test__start_subtest("gotox-target-subprog-from-main")) 1113 check_gotox_target_subprog_from_main(); 1114 if (test__start_subtest("gotox-index-slice-other-subprog")) 1115 check_gotox_index_slice_other_subprog(); 1116 if (test__start_subtest("gotox-target-other-global-subprog")) 1117 check_gotox_target_other_global_subprog(); 1118 if (test__start_subtest("gotox-callback-leaves-subprog")) 1119 check_gotox_callback_leaves_subprog(); 1120 if (test__start_subtest("gotox-target-prologue-shift")) 1121 check_gotox_target_prologue_shift(); 1122 if (test__start_subtest("gotox-target-ctx-prologue-shift")) 1123 check_gotox_target_ctx_prologue_shift(); 1124 } 1125 #else 1126 static void __test_bpf_insn_array(void) 1127 { 1128 test__skip(); 1129 } 1130 #endif 1131 1132 void test_bpf_insn_array(void) 1133 { 1134 __test_bpf_insn_array(); 1135 } 1136