1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <bpf/bpf.h> 4 #include <bpf/btf.h> 5 #include <test_progs.h> 6 7 #if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) 8 static int map_create(__u32 map_type, __u32 max_entries) 9 { 10 const char *map_name = "insn_array"; 11 __u32 key_size = 4; 12 __u32 value_size = sizeof(struct bpf_insn_array_value); 13 14 return bpf_map_create(map_type, map_name, key_size, value_size, max_entries, NULL); 15 } 16 17 static int prog_load(struct bpf_insn *insns, __u32 insn_cnt, int *fd_array, __u32 fd_array_cnt) 18 { 19 LIBBPF_OPTS(bpf_prog_load_opts, opts); 20 21 opts.fd_array = fd_array; 22 opts.fd_array_cnt = fd_array_cnt; 23 24 return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, &opts); 25 } 26 27 static void __check_success(struct bpf_insn *insns, __u32 insn_cnt, __u32 *map_in, __u32 *map_out) 28 { 29 struct bpf_insn_array_value val = {}; 30 int prog_fd = -1, map_fd, i; 31 32 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, insn_cnt); 33 if (!ASSERT_GE(map_fd, 0, "map_create")) 34 return; 35 36 for (i = 0; i < insn_cnt; i++) { 37 val.orig_off = map_in[i]; 38 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 39 goto cleanup; 40 } 41 42 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 43 goto cleanup; 44 45 prog_fd = prog_load(insns, insn_cnt, &map_fd, 1); 46 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 47 goto cleanup; 48 49 for (i = 0; i < insn_cnt; i++) { 50 char buf[64]; 51 52 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 53 goto cleanup; 54 55 snprintf(buf, sizeof(buf), "val.xlated_off should be equal map_out[%d]", i); 56 ASSERT_EQ(val.xlated_off, map_out[i], buf); 57 } 58 59 cleanup: 60 close(prog_fd); 61 close(map_fd); 62 } 63 64 /* 65 * Load a program, which will not be anyhow mangled by the verifier. Add an 66 * insn_array map pointing to every instruction. Check that it hasn't changed 67 * after the program load. 68 */ 69 static void check_one_to_one_mapping(void) 70 { 71 struct bpf_insn insns[] = { 72 BPF_MOV64_IMM(BPF_REG_0, 4), 73 BPF_MOV64_IMM(BPF_REG_0, 3), 74 BPF_MOV64_IMM(BPF_REG_0, 2), 75 BPF_MOV64_IMM(BPF_REG_0, 1), 76 BPF_MOV64_IMM(BPF_REG_0, 0), 77 BPF_EXIT_INSN(), 78 }; 79 __u32 map_in[] = {0, 1, 2, 3, 4, 5}; 80 __u32 map_out[] = {0, 1, 2, 3, 4, 5}; 81 82 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 83 } 84 85 /* 86 * Load a program with two patches (get jiffies, for simplicity). Add an 87 * insn_array map pointing to every instruction. Check how it was changed 88 * after the program load. 89 */ 90 static void check_simple(void) 91 { 92 struct bpf_insn insns[] = { 93 BPF_MOV64_IMM(BPF_REG_0, 2), 94 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 95 BPF_MOV64_IMM(BPF_REG_0, 1), 96 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 97 BPF_MOV64_IMM(BPF_REG_0, 0), 98 BPF_EXIT_INSN(), 99 }; 100 __u32 map_in[] = {0, 1, 2, 3, 4, 5}; 101 __u32 map_out[] = {0, 1, 4, 5, 8, 9}; 102 103 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 104 } 105 106 /* 107 * Verifier can delete code in two cases: nops & dead code. From insn 108 * array's point of view, the two cases are the same, so test using 109 * the simplest method: by loading some nops 110 */ 111 static void check_deletions(void) 112 { 113 struct bpf_insn insns[] = { 114 BPF_MOV64_IMM(BPF_REG_0, 2), 115 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 116 BPF_MOV64_IMM(BPF_REG_0, 1), 117 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 118 BPF_MOV64_IMM(BPF_REG_0, 0), 119 BPF_EXIT_INSN(), 120 }; 121 __u32 map_in[] = {0, 1, 2, 3, 4, 5}; 122 __u32 map_out[] = {0, -1, 1, -1, 2, 3}; 123 124 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 125 } 126 127 /* 128 * Same test as check_deletions, but also add code which adds instructions 129 */ 130 static void check_deletions_with_functions(void) 131 { 132 struct bpf_insn insns[] = { 133 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 134 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 135 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 136 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), 137 BPF_MOV64_IMM(BPF_REG_0, 1), 138 BPF_EXIT_INSN(), 139 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 140 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), 141 BPF_JMP_IMM(BPF_JA, 0, 0, 0), /* nop */ 142 BPF_MOV64_IMM(BPF_REG_0, 2), 143 BPF_EXIT_INSN(), 144 }; 145 __u32 map_in[] = { 0, 1, 2, 3, 4, 5, /* func */ 6, 7, 8, 9, 10}; 146 __u32 map_out[] = {-1, 0, -1, 3, 4, 5, /* func */ -1, 6, -1, 9, 10}; 147 148 __check_success(insns, ARRAY_SIZE(insns), map_in, map_out); 149 } 150 151 /* 152 * Try to load a program with a map which points to outside of the program 153 */ 154 static void check_out_of_bounds_index(void) 155 { 156 struct bpf_insn insns[] = { 157 BPF_MOV64_IMM(BPF_REG_0, 4), 158 BPF_MOV64_IMM(BPF_REG_0, 3), 159 BPF_MOV64_IMM(BPF_REG_0, 2), 160 BPF_MOV64_IMM(BPF_REG_0, 1), 161 BPF_MOV64_IMM(BPF_REG_0, 0), 162 BPF_EXIT_INSN(), 163 }; 164 int prog_fd, map_fd; 165 struct bpf_insn_array_value val = {}; 166 int key; 167 168 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 169 if (!ASSERT_GE(map_fd, 0, "map_create")) 170 return; 171 172 key = 0; 173 val.orig_off = ARRAY_SIZE(insns); /* too big */ 174 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) 175 goto cleanup; 176 177 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 178 goto cleanup; 179 180 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 181 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) { 182 close(prog_fd); 183 goto cleanup; 184 } 185 186 cleanup: 187 close(map_fd); 188 } 189 190 /* 191 * Try to load a program with a map which points to the middle of 16-bit insn 192 */ 193 static void check_mid_insn_index(void) 194 { 195 struct bpf_insn insns[] = { 196 BPF_LD_IMM64(BPF_REG_0, 0), /* 2 x 8 */ 197 BPF_EXIT_INSN(), 198 }; 199 int prog_fd, map_fd; 200 struct bpf_insn_array_value val = {}; 201 int key; 202 203 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 204 if (!ASSERT_GE(map_fd, 0, "map_create")) 205 return; 206 207 key = 0; 208 val.orig_off = 1; /* middle of 16-byte instruction */ 209 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) 210 goto cleanup; 211 212 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 213 goto cleanup; 214 215 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 216 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) { 217 close(prog_fd); 218 goto cleanup; 219 } 220 221 cleanup: 222 close(map_fd); 223 } 224 225 static void check_incorrect_index(void) 226 { 227 check_out_of_bounds_index(); 228 check_mid_insn_index(); 229 } 230 231 static int set_bpf_jit_harden(char *level) 232 { 233 char old_level; 234 int err = -1; 235 int fd = -1; 236 237 fd = open("/proc/sys/net/core/bpf_jit_harden", O_RDWR | O_NONBLOCK); 238 if (fd < 0) { 239 ASSERT_FAIL("open .../bpf_jit_harden returned %d (errno=%d)", fd, errno); 240 return -1; 241 } 242 243 err = read(fd, &old_level, 1); 244 if (err != 1) { 245 ASSERT_FAIL("read from .../bpf_jit_harden returned %d (errno=%d)", err, errno); 246 err = -1; 247 goto end; 248 } 249 250 lseek(fd, 0, SEEK_SET); 251 252 err = write(fd, level, 1); 253 if (err != 1) { 254 ASSERT_FAIL("write to .../bpf_jit_harden returned %d (errno=%d)", err, errno); 255 err = -1; 256 goto end; 257 } 258 259 err = 0; 260 *level = old_level; 261 end: 262 if (fd >= 0) 263 close(fd); 264 return err; 265 } 266 267 static void check_blindness(void) 268 { 269 struct bpf_insn insns[] = { 270 BPF_MOV64_IMM(BPF_REG_0, 4), 271 BPF_MOV64_IMM(BPF_REG_0, 3), 272 BPF_MOV64_IMM(BPF_REG_0, 2), 273 BPF_MOV64_IMM(BPF_REG_0, 1), 274 BPF_EXIT_INSN(), 275 }; 276 int prog_fd = -1, map_fd; 277 struct bpf_insn_array_value val = {}; 278 char bpf_jit_harden = '@'; /* non-exizsting value */ 279 int i; 280 281 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, ARRAY_SIZE(insns)); 282 if (!ASSERT_GE(map_fd, 0, "map_create")) 283 return; 284 285 for (i = 0; i < ARRAY_SIZE(insns); i++) { 286 val.orig_off = i; 287 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 288 goto cleanup; 289 } 290 291 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 292 goto cleanup; 293 294 bpf_jit_harden = '2'; 295 if (set_bpf_jit_harden(&bpf_jit_harden)) { 296 bpf_jit_harden = '@'; /* open, read or write failed => no write was done */ 297 goto cleanup; 298 } 299 300 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 301 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 302 goto cleanup; 303 304 for (i = 0; i < ARRAY_SIZE(insns); i++) { 305 char fmt[32]; 306 307 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 308 goto cleanup; 309 310 snprintf(fmt, sizeof(fmt), "val should be equal 3*%d", i); 311 ASSERT_EQ(val.xlated_off, i * 3, fmt); 312 } 313 314 cleanup: 315 /* restore the old one */ 316 if (bpf_jit_harden != '@') 317 set_bpf_jit_harden(&bpf_jit_harden); 318 319 close(prog_fd); 320 close(map_fd); 321 } 322 323 /* Once map was initialized, it should be frozen */ 324 static void check_load_unfrozen_map(void) 325 { 326 struct bpf_insn insns[] = { 327 BPF_MOV64_IMM(BPF_REG_0, 0), 328 BPF_EXIT_INSN(), 329 }; 330 int prog_fd = -1, map_fd; 331 struct bpf_insn_array_value val = {}; 332 int i; 333 334 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, ARRAY_SIZE(insns)); 335 if (!ASSERT_GE(map_fd, 0, "map_create")) 336 return; 337 338 for (i = 0; i < ARRAY_SIZE(insns); i++) { 339 val.orig_off = i; 340 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 341 goto cleanup; 342 } 343 344 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 345 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) 346 goto cleanup; 347 348 /* correctness: now freeze the map, the program should load fine */ 349 350 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 351 goto cleanup; 352 353 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 354 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 355 goto cleanup; 356 357 for (i = 0; i < ARRAY_SIZE(insns); i++) { 358 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 359 goto cleanup; 360 361 ASSERT_EQ(val.xlated_off, i, "val should be equal i"); 362 } 363 364 cleanup: 365 close(prog_fd); 366 close(map_fd); 367 } 368 369 /* Map can be used only by one BPF program */ 370 static void check_no_map_reuse(void) 371 { 372 struct bpf_insn insns[] = { 373 BPF_MOV64_IMM(BPF_REG_0, 0), 374 BPF_EXIT_INSN(), 375 }; 376 int prog_fd = -1, map_fd, extra_fd = -1; 377 struct bpf_insn_array_value val = {}; 378 int i; 379 380 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, ARRAY_SIZE(insns)); 381 if (!ASSERT_GE(map_fd, 0, "map_create")) 382 return; 383 384 for (i = 0; i < ARRAY_SIZE(insns); i++) { 385 val.orig_off = i; 386 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, "bpf_map_update_elem")) 387 goto cleanup; 388 } 389 390 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 391 goto cleanup; 392 393 prog_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 394 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 395 goto cleanup; 396 397 for (i = 0; i < ARRAY_SIZE(insns); i++) { 398 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &i, &val), 0, "bpf_map_lookup_elem")) 399 goto cleanup; 400 401 ASSERT_EQ(val.xlated_off, i, "val should be equal i"); 402 } 403 404 extra_fd = prog_load(insns, ARRAY_SIZE(insns), &map_fd, 1); 405 if (!ASSERT_EQ(extra_fd, -EBUSY, "program should have been rejected (extra_fd != -EBUSY)")) 406 goto cleanup; 407 408 /* correctness: check that prog is still loadable without fd_array */ 409 extra_fd = prog_load(insns, ARRAY_SIZE(insns), NULL, 0); 410 if (!ASSERT_GE(extra_fd, 0, "bpf(BPF_PROG_LOAD): expected no error")) 411 goto cleanup; 412 413 cleanup: 414 close(extra_fd); 415 close(prog_fd); 416 close(map_fd); 417 } 418 419 static void check_bpf_no_lookup(void) 420 { 421 struct bpf_insn insns[] = { 422 BPF_LD_MAP_FD(BPF_REG_1, 0), 423 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0), 424 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), 425 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8), 426 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 427 BPF_EXIT_INSN(), 428 }; 429 int prog_fd = -1, map_fd; 430 431 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 432 if (!ASSERT_GE(map_fd, 0, "map_create")) 433 return; 434 435 insns[0].imm = map_fd; 436 437 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 438 goto cleanup; 439 440 prog_fd = prog_load(insns, ARRAY_SIZE(insns), NULL, 0); 441 if (!ASSERT_EQ(prog_fd, -EINVAL, "program should have been rejected (prog_fd != -EINVAL)")) 442 goto cleanup; 443 444 /* correctness: check that prog is still loadable with normal map */ 445 close(map_fd); 446 map_fd = map_create(BPF_MAP_TYPE_ARRAY, 1); 447 insns[0].imm = map_fd; 448 prog_fd = prog_load(insns, ARRAY_SIZE(insns), NULL, 0); 449 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 450 goto cleanup; 451 452 cleanup: 453 close(prog_fd); 454 close(map_fd); 455 } 456 457 #define GOTOX_LOG_SZ (256 * 1024) 458 459 /* Load and return the fd, which the caller owns. */ 460 static int gotox_prog_load_funcs_fd(struct bpf_insn *insns, __u32 insn_cnt, 461 int *fd_array, __u32 fd_array_cnt, char *log, 462 int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) 463 { 464 LIBBPF_OPTS(bpf_prog_load_opts, opts); 465 466 log[0] = 0; 467 opts.fd_array = fd_array; 468 opts.fd_array_cnt = fd_array_cnt; 469 opts.log_buf = log; 470 opts.log_size = GOTOX_LOG_SZ; 471 opts.log_level = 1; 472 if (fi_cnt) { 473 opts.prog_btf_fd = btf_fd; 474 opts.func_info = fi; 475 opts.func_info_cnt = fi_cnt; 476 opts.func_info_rec_size = sizeof(*fi); 477 } 478 479 return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, &opts); 480 } 481 482 /* Load for the outcome only: 0 or the negative errno. */ 483 static int gotox_prog_load_funcs(struct bpf_insn *insns, __u32 insn_cnt, 484 int *fd_array, __u32 fd_array_cnt, char *log, 485 int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) 486 { 487 int prog_fd; 488 489 prog_fd = gotox_prog_load_funcs_fd(insns, insn_cnt, fd_array, fd_array_cnt, log, 490 btf_fd, fi, fi_cnt); 491 if (prog_fd < 0) 492 return prog_fd; 493 close(prog_fd); 494 return 0; 495 } 496 497 static int gotox_prog_load(struct bpf_insn *insns, __u32 insn_cnt, 498 int *fd_array, __u32 fd_array_cnt, char *log) 499 { 500 return gotox_prog_load_funcs(insns, insn_cnt, fd_array, fd_array_cnt, log, 501 -1, NULL, 0); 502 } 503 504 static int gotox_jt_create_offs(const __u32 *offs, __u32 cnt) 505 { 506 struct bpf_insn_array_value val = {}; 507 int map_fd; 508 __u32 i; 509 510 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, cnt); 511 if (!ASSERT_GE(map_fd, 0, "map_create")) 512 return map_fd; 513 514 for (i = 0; i < cnt; i++) { 515 val.orig_off = offs[i]; 516 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, 517 "bpf_map_update_elem")) 518 goto err; 519 } 520 521 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 522 goto err; 523 524 return map_fd; 525 err: 526 close(map_fd); 527 return -1; 528 } 529 530 #define GOTOX_SUB_START 4 531 #define GOTOX_MAIN_TGT 2 532 #define GOTOX_SUB_TGT 8 533 #define GOTOX_TWO_INSN_CNT 10 534 535 static void gotox_two_subprogs_fill(struct bpf_insn *insns, __u32 jt_idx, __u32 jt_off) 536 { 537 insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); 538 insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 539 GOTOX_SUB_START - 1 - 1); 540 insns[GOTOX_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 541 insns[3] = BPF_EXIT_INSN(); 542 543 /* r1 = &jt[0], by index 'jt_idx' into fd_array */ 544 insns[GOTOX_SUB_START] = (struct bpf_insn) { 545 .code = BPF_LD | BPF_DW | BPF_IMM, 546 .dst_reg = BPF_REG_1, 547 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 548 .imm = jt_idx, 549 }; 550 insns[GOTOX_SUB_START + 1] = (struct bpf_insn) { .imm = 0 }; 551 insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, jt_off * 8); 552 insns[7] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 553 insns[GOTOX_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); 554 insns[9] = BPF_EXIT_INSN(); 555 } 556 557 /* 558 * Load 'insns' with the two jump tables 'jt_a' and 'jt_b', in this order in 559 * fd_array, and check the outcome. 'fi' describes the subprograms when given. 560 */ 561 static void check_gotox_two_maps(struct bpf_insn *insns, __u32 insn_cnt, 562 const __u32 *jt_a, __u32 a_cnt, 563 const __u32 *jt_b, __u32 b_cnt, 564 int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt, 565 int expect_err, const char *expect_msg) 566 { 567 int map_fd[2] = { -1, -1 }; 568 char *log; 569 int err; 570 571 log = calloc(1, GOTOX_LOG_SZ); 572 if (!ASSERT_OK_PTR(log, "calloc log")) 573 return; 574 575 map_fd[0] = gotox_jt_create_offs(jt_a, a_cnt); 576 if (map_fd[0] < 0) 577 goto free_log; 578 map_fd[1] = gotox_jt_create_offs(jt_b, b_cnt); 579 if (map_fd[1] < 0) 580 goto close_maps; 581 582 if (fi) 583 err = gotox_prog_load_funcs(insns, insn_cnt, map_fd, 2, log, 584 btf_fd, fi, fi_cnt); 585 else 586 err = gotox_prog_load(insns, insn_cnt, map_fd, 2, log); 587 if (expect_err) 588 ASSERT_EQ(err, expect_err, "program should have been rejected"); 589 else 590 ASSERT_EQ(err, 0, "bpf(BPF_PROG_LOAD)"); 591 if (expect_msg) 592 ASSERT_HAS_SUBSTR(log, expect_msg, "verifier log"); 593 594 close_maps: 595 close(map_fd[0]); 596 if (map_fd[1] >= 0) 597 close(map_fd[1]); 598 free_log: 599 free(log); 600 } 601 602 static void check_gotox_target_other_subprog(void) 603 { 604 const __u32 jt_main[] = { GOTOX_MAIN_TGT }; 605 const __u32 jt_sub[] = { GOTOX_SUB_TGT }; 606 struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; 607 608 gotox_two_subprogs_fill(insns, 0, 0); 609 610 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 611 jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, 612 -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); 613 } 614 615 static void check_gotox_jt_per_subprog(void) 616 { 617 const __u32 jt_main[] = { GOTOX_MAIN_TGT }; 618 const __u32 jt_sub[] = { GOTOX_SUB_TGT }; 619 struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; 620 621 gotox_two_subprogs_fill(insns, 1, 0); 622 623 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 624 jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, 625 0, NULL); 626 } 627 628 #define GOTOX_DEAD_MAIN_LEN 7 629 #define GOTOX_DEAD_SUB_LEN 6 630 #define GOTOX_DEAD_INSN_CNT (GOTOX_DEAD_MAIN_LEN + 3 * GOTOX_DEAD_SUB_LEN) 631 #define GOTOX_DEAD_LIVE_CNT 10 632 633 /* r1 = &jt[0], by index 'jt_idx' into fd_array; r1 = *r1; gotox r1; r0 = 0; exit */ 634 static void gotox_sub_fill(struct bpf_insn *insns, __u32 start, __u32 jt_idx) 635 { 636 insns[start] = (struct bpf_insn) { 637 .code = BPF_LD | BPF_DW | BPF_IMM, 638 .dst_reg = BPF_REG_1, 639 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 640 .imm = jt_idx, 641 }; 642 insns[start + 1] = (struct bpf_insn) { .imm = 0 }; 643 insns[start + 2] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 644 insns[start + 3] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 645 insns[start + 4] = BPF_MOV64_IMM(BPF_REG_0, 0); 646 insns[start + 5] = BPF_EXIT_INSN(); 647 } 648 649 /* 650 * Two subprograms which are only called from dead code sit in front of the 651 * live one, and all three own a jump table. Dead code removal drops both in 652 * one go, which compacts subprog_info[] along with the tables. 653 */ 654 static void check_gotox_jt_dead_subprogs(void) 655 { 656 const __u32 sub_start[3] = { GOTOX_DEAD_MAIN_LEN, 657 GOTOX_DEAD_MAIN_LEN + GOTOX_DEAD_SUB_LEN, 658 GOTOX_DEAD_MAIN_LEN + 2 * GOTOX_DEAD_SUB_LEN }; 659 struct bpf_insn insns[GOTOX_DEAD_INSN_CNT]; 660 int map_fd[3] = { -1, -1, -1 }; 661 struct bpf_prog_info info = {}; 662 __u32 len = sizeof(info), tgt, i; 663 int prog_fd; 664 char *log; 665 666 log = calloc(1, GOTOX_LOG_SZ); 667 if (!ASSERT_OK_PTR(log, "calloc log")) 668 return; 669 670 insns[0] = BPF_MOV64_IMM(BPF_REG_1, 0); 671 insns[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2); 672 insns[2] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 673 sub_start[0] - 2 - 1); 674 insns[3] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 675 sub_start[1] - 3 - 1); 676 insns[4] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 677 sub_start[2] - 4 - 1); 678 insns[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 679 insns[6] = BPF_EXIT_INSN(); 680 681 for (i = 0; i < ARRAY_SIZE(sub_start); i++) { 682 gotox_sub_fill(insns, sub_start[i], i); 683 tgt = sub_start[i] + 4; 684 map_fd[i] = gotox_jt_create_offs(&tgt, 1); 685 if (map_fd[i] < 0) 686 goto close_maps; 687 } 688 689 prog_fd = gotox_prog_load_funcs_fd(insns, ARRAY_SIZE(insns), map_fd, 690 ARRAY_SIZE(map_fd), log, -1, NULL, 0); 691 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 692 goto close_maps; 693 694 /* Both dead subprograms and the calls to them must be gone. */ 695 if (ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), 696 "bpf_prog_get_info_by_fd")) 697 ASSERT_EQ(info.xlated_prog_len / sizeof(struct bpf_insn), 698 GOTOX_DEAD_LIVE_CNT, "xlated insn count"); 699 close(prog_fd); 700 close_maps: 701 for (i = 0; i < ARRAY_SIZE(map_fd); i++) 702 if (map_fd[i] >= 0) 703 close(map_fd[i]); 704 free(log); 705 } 706 707 #define GOTOX_FWD_GOTOX 11 708 #define GOTOX_FWD_OWN_TGT 12 709 #define GOTOX_FWD_SUB_START 14 710 #define GOTOX_FWD_INSN_CNT 16 711 712 static void gotox_from_main_fill(struct bpf_insn *insns) 713 { 714 insns[0] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); 715 insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 716 GOTOX_FWD_SUB_START - 1 - 1); 717 insns[2] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, 718 offsetof(struct xdp_md, ingress_ifindex)); 719 insns[3] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 4); 720 721 /* r1 = &jt_leaves[0], by index 1 into fd_array */ 722 insns[4] = (struct bpf_insn) { 723 .code = BPF_LD | BPF_DW | BPF_IMM, 724 .dst_reg = BPF_REG_1, 725 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 726 .imm = 1, 727 }; 728 insns[5] = (struct bpf_insn) { .imm = 0 }; 729 insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 730 insns[7] = BPF_JMP_A(3); 731 732 /* r1 = &jt_own[0], by index 0 into fd_array */ 733 insns[8] = (struct bpf_insn) { 734 .code = BPF_LD | BPF_DW | BPF_IMM, 735 .dst_reg = BPF_REG_1, 736 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 737 .imm = 0, 738 }; 739 insns[9] = (struct bpf_insn) { .imm = 0 }; 740 insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 741 742 insns[GOTOX_FWD_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 743 insns[GOTOX_FWD_OWN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 744 insns[13] = BPF_EXIT_INSN(); 745 insns[GOTOX_FWD_SUB_START] = BPF_MOV64_IMM(BPF_REG_0, 1); 746 insns[15] = BPF_EXIT_INSN(); 747 } 748 749 static void check_gotox_target_subprog_from_main(void) 750 { 751 const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; 752 const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; 753 struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; 754 755 gotox_from_main_fill(insns); 756 757 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), 758 jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, 759 -EINVAL, "indirect jump from insn 11 to 14 leaves the subprog [0,14)"); 760 } 761 762 /* 763 * The only map of the subprog holding the gotox reaches past that subprog, so 764 * the subprog is left without a jump table at all. 765 */ 766 static void check_gotox_jt_spans_subprogs(void) 767 { 768 const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; 769 const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; 770 struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; 771 772 gotox_from_main_fill(insns); 773 774 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_span, ARRAY_SIZE(jt_span), 775 jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, 776 -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); 777 } 778 779 /* 780 * The subprog holding the gotox has a well formed jump table of its own and 781 * also collects a map that reaches past its end. The spanning map is still 782 * rejected, even though the subprog is not left without a table. 783 */ 784 static void check_gotox_jt_spans_with_own_table(void) 785 { 786 const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; 787 const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; 788 struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; 789 790 gotox_from_main_fill(insns); 791 792 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), 793 jt_span, ARRAY_SIZE(jt_span), -1, NULL, 0, 794 -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); 795 } 796 797 #define GOTOX_SLICE_SUB_START 6 798 #define GOTOX_SLICE_GOTOX 14 799 #define GOTOX_SLICE_SUB_TGT 15 800 #define GOTOX_SLICE_INSN_CNT 17 801 802 static void gotox_slice_fill(struct bpf_insn *insns) 803 { 804 insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); 805 insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 806 GOTOX_SLICE_SUB_START - 1 - 1); 807 insns[2] = BPF_MOV64_IMM(BPF_REG_0, 0); 808 insns[3] = BPF_MOV64_IMM(BPF_REG_0, 0); 809 insns[4] = BPF_MOV64_IMM(BPF_REG_0, 0); 810 insns[5] = BPF_EXIT_INSN(); 811 812 insns[GOTOX_SLICE_SUB_START] = 813 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 814 offsetof(struct xdp_md, ingress_ifindex)); 815 insns[7] = BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 1); 816 insns[8] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1); 817 insns[9] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 818 819 /* r1 = &jt_main[0], by index 0 into fd_array */ 820 insns[10] = (struct bpf_insn) { 821 .code = BPF_LD | BPF_DW | BPF_IMM, 822 .dst_reg = BPF_REG_1, 823 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 824 .imm = 0, 825 }; 826 insns[11] = (struct bpf_insn) { .imm = 0 }; 827 insns[12] = BPF_ALU64_REG(BPF_ADD, BPF_REG_1, BPF_REG_2); 828 insns[13] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 829 830 insns[GOTOX_SLICE_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 831 insns[GOTOX_SLICE_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); 832 insns[16] = BPF_EXIT_INSN(); 833 } 834 835 static void check_gotox_index_slice_other_subprog(void) 836 { 837 const __u32 jt_main[] = { 2, 3, 4 }; 838 const __u32 jt_sub[] = { GOTOX_SLICE_SUB_TGT }; 839 struct bpf_insn insns[GOTOX_SLICE_INSN_CNT]; 840 841 gotox_slice_fill(insns); 842 843 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 844 jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, 845 -EINVAL, "indirect jump from insn 14 to 3 leaves the subprog [6,17)"); 846 } 847 848 static int gotox_btf_create(const __u32 *starts, const __u8 *linkage, __u32 cnt, 849 struct bpf_func_info *fi, struct btf **pbtf) 850 { 851 int int_id, proto_id, id; 852 struct btf *btf; 853 char name[24]; 854 __u32 i; 855 856 btf = btf__new_empty(); 857 if (!ASSERT_OK_PTR(btf, "btf__new_empty")) 858 return -1; 859 860 int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); 861 if (!ASSERT_GT(int_id, 0, "btf__add_int")) 862 goto err; 863 864 proto_id = btf__add_func_proto(btf, int_id); 865 if (!ASSERT_GT(proto_id, 0, "btf__add_func_proto")) 866 goto err; 867 868 for (i = 0; i < cnt; i++) { 869 snprintf(name, sizeof(name), "gotox_f%u", i); 870 id = btf__add_func(btf, name, linkage[i], proto_id); 871 if (!ASSERT_GT(id, 0, "btf__add_func")) 872 goto err; 873 fi[i].insn_off = starts[i]; 874 fi[i].type_id = id; 875 } 876 877 if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) 878 goto err; 879 880 *pbtf = btf; 881 return btf__fd(btf); 882 err: 883 btf__free(btf); 884 return -1; 885 } 886 887 static void check_gotox_target_other_global_subprog(void) 888 { 889 const __u32 starts[] = { 0, GOTOX_SUB_START }; 890 const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_GLOBAL }; 891 const __u32 jt_main[] = { GOTOX_MAIN_TGT }; 892 const __u32 jt_sub[] = { GOTOX_SUB_TGT }; 893 struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; 894 struct bpf_func_info fi[2]; 895 struct btf *btf = NULL; 896 int btf_fd; 897 898 gotox_two_subprogs_fill(insns, 0, 0); 899 900 btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); 901 if (btf_fd < 0) 902 return; 903 904 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 905 jt_sub, ARRAY_SIZE(jt_sub), btf_fd, fi, ARRAY_SIZE(fi), 906 -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); 907 btf__free(btf); 908 } 909 910 #define GOTOX_CB_MAIN_TGT 6 911 #define GOTOX_CB_START 8 912 #define GOTOX_CB_GOTOX 11 913 #define GOTOX_CB_TGT 12 914 #define GOTOX_CB_INSN_CNT 14 915 916 static void gotox_callback_fill(struct bpf_insn *insns) 917 { 918 insns[0] = BPF_MOV64_IMM(BPF_REG_1, 1); 919 /* r2 = &callback */ 920 insns[1] = (struct bpf_insn) { 921 .code = BPF_LD | BPF_DW | BPF_IMM, 922 .dst_reg = BPF_REG_2, 923 .src_reg = BPF_PSEUDO_FUNC, 924 .imm = GOTOX_CB_START - 1 - 1, 925 }; 926 insns[2] = (struct bpf_insn) { .imm = 0 }; 927 insns[3] = BPF_MOV64_IMM(BPF_REG_3, 0); 928 insns[4] = BPF_MOV64_IMM(BPF_REG_4, 0); 929 insns[5] = BPF_EMIT_CALL(BPF_FUNC_loop); 930 insns[GOTOX_CB_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 931 insns[7] = BPF_EXIT_INSN(); 932 933 /* r1 = &jt_main[0], by index 0 into fd_array */ 934 insns[GOTOX_CB_START] = (struct bpf_insn) { 935 .code = BPF_LD | BPF_DW | BPF_IMM, 936 .dst_reg = BPF_REG_1, 937 .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, 938 .imm = 0, 939 }; 940 insns[9] = (struct bpf_insn) { .imm = 0 }; 941 insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 942 insns[GOTOX_CB_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); 943 insns[GOTOX_CB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); 944 insns[13] = BPF_EXIT_INSN(); 945 } 946 947 static void check_gotox_callback_leaves_subprog(void) 948 { 949 const __u32 starts[] = { 0, GOTOX_CB_START }; 950 const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_STATIC }; 951 const __u32 jt_main[] = { GOTOX_CB_MAIN_TGT }; 952 const __u32 jt_cb[] = { GOTOX_CB_TGT }; 953 struct bpf_insn insns[GOTOX_CB_INSN_CNT]; 954 struct bpf_func_info fi[2]; 955 struct btf *btf = NULL; 956 int btf_fd; 957 958 gotox_callback_fill(insns); 959 960 btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); 961 if (btf_fd < 0) 962 return; 963 964 check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), 965 jt_cb, ARRAY_SIZE(jt_cb), btf_fd, fi, ARRAY_SIZE(fi), 966 -EINVAL, "indirect jump from insn 11 to 6 leaves the subprog [8,14)"); 967 btf__free(btf); 968 } 969 970 static void check_bpf_side(void) 971 { 972 check_bpf_no_lookup(); 973 } 974 975 /* Return the xlated index of the first insn equal to 'insn', or -1. */ 976 static int xlated_insn_index(int prog_fd, const struct bpf_insn *insn) 977 { 978 struct bpf_insn *xlated; 979 int idx = -1; 980 __u32 cnt, i; 981 982 if (!ASSERT_OK(get_xlated_program(prog_fd, &xlated, &cnt), "get_xlated_program")) 983 return -1; 984 for (i = 0; i < cnt; i++) 985 if (!memcmp(&xlated[i], insn, sizeof(*insn))) { 986 idx = i; 987 break; 988 } 989 free(xlated); 990 return idx; 991 } 992 993 /* 994 * Load 'insns' with a jump table naming insn 0, which a prologue prepend 995 * pushes down, and check that the slot followed it. 996 */ 997 static void check_gotox_target_prepend(enum bpf_prog_type prog_type, 998 struct bpf_insn *insns, __u32 insn_cnt) 999 { 1000 LIBBPF_OPTS(bpf_prog_load_opts, opts); 1001 struct bpf_insn_array_value val = {}; 1002 int prog_fd = -1, map_fd, idx; 1003 __u32 key = 0; 1004 1005 map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); 1006 if (!ASSERT_GE(map_fd, 0, "map_create")) 1007 return; 1008 1009 val.orig_off = 0; 1010 if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) 1011 goto cleanup; 1012 if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) 1013 goto cleanup; 1014 1015 opts.fd_array = &map_fd; 1016 opts.fd_array_cnt = 1; 1017 prog_fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insn_cnt, &opts); 1018 if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) 1019 goto cleanup; 1020 1021 idx = xlated_insn_index(prog_fd, &insns[0]); 1022 if (!ASSERT_GT(idx, 0, "insn 0 was prepended to")) 1023 goto cleanup; 1024 if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &key, &val), 0, "bpf_map_lookup_elem")) 1025 goto cleanup; 1026 ASSERT_EQ(val.xlated_off, (__u32)idx, "gotox target followed insn 0 past the prepend"); 1027 cleanup: 1028 if (prog_fd >= 0) 1029 close(prog_fd); 1030 close(map_fd); 1031 } 1032 1033 static void check_gotox_target_prologue_shift(void) 1034 { 1035 struct bpf_insn insns[] = { 1036 /* insn 0: gotox target and subprog start */ 1037 BPF_MOV64_IMM(BPF_REG_0, 0), 1038 /* may_goto +4 -> exit block, bounds the loop */ 1039 BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 4, 0), 1040 /* r1 = &jt[0] (insns 2 and 3) */ 1041 BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 0), 1042 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0), /* r1 = ips[0] */ 1043 /* insn 5: gotox r1 -> insn 0 */ 1044 BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0), 1045 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), /* insn 6: exit block */ 1046 BPF_EXIT_INSN(), 1047 }; 1048 1049 check_gotox_target_prepend(BPF_PROG_TYPE_XDP, insns, ARRAY_SIZE(insns)); 1050 } 1051 1052 static void check_gotox_target_ctx_prologue_shift(void) 1053 { 1054 struct bpf_insn insns[] = { 1055 /* insn 0: gotox target and subprog start */ 1056 BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, -8), 1057 BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, 0x5a5a, 12), /* second visit exits */ 1058 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0x5a5a), 1059 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), /* r6 = ctx */ 1060 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, offsetof(struct __sk_buff, data)), 1061 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_6, offsetof(struct __sk_buff, data_end)), 1062 BPF_MOV64_REG(BPF_REG_4, BPF_REG_2), 1063 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 1), 1064 BPF_JMP_REG(BPF_JGT, BPF_REG_4, BPF_REG_3, 5), 1065 /* insn 9: direct packet write -> tc unclone prologue prepended at insn 0 */ 1066 BPF_ST_MEM(BPF_B, BPF_REG_2, 0, 0), 1067 /* r5 = &jt[0] (insns 10 and 11) */ 1068 BPF_LD_IMM64_RAW(BPF_REG_5, BPF_PSEUDO_MAP_IDX_VALUE, 0), 1069 BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_5, 0), /* r5 = ips[0] */ 1070 /* insn 13: gotox r5 -> insn 0 */ 1071 BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_5, 0, 0, 0), 1072 BPF_MOV64_IMM(BPF_REG_0, 0), /* insn 14: exit block */ 1073 BPF_EXIT_INSN(), 1074 }; 1075 1076 check_gotox_target_prepend(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns)); 1077 } 1078 1079 static void __test_bpf_insn_array(void) 1080 { 1081 /* Test if offsets are adjusted properly */ 1082 if (test__start_subtest("one2one")) 1083 check_one_to_one_mapping(); 1084 if (test__start_subtest("simple")) 1085 check_simple(); 1086 if (test__start_subtest("deletions")) 1087 check_deletions(); 1088 if (test__start_subtest("deletions-with-functions")) 1089 check_deletions_with_functions(); 1090 if (test__start_subtest("blindness")) 1091 check_blindness(); 1092 /* Check all kinds of operations and related restrictions */ 1093 if (test__start_subtest("incorrect-index")) 1094 check_incorrect_index(); 1095 if (test__start_subtest("load-unfrozen-map")) 1096 check_load_unfrozen_map(); 1097 if (test__start_subtest("no-map-reuse")) 1098 check_no_map_reuse(); 1099 if (test__start_subtest("bpf-side-ops")) 1100 check_bpf_side(); 1101 if (test__start_subtest("gotox-jt-spans-subprogs")) 1102 check_gotox_jt_spans_subprogs(); 1103 if (test__start_subtest("gotox-jt-spans-with-own-table")) 1104 check_gotox_jt_spans_with_own_table(); 1105 if (test__start_subtest("gotox-target-other-subprog")) 1106 check_gotox_target_other_subprog(); 1107 if (test__start_subtest("gotox-jt-per-subprog")) 1108 check_gotox_jt_per_subprog(); 1109 if (test__start_subtest("gotox-jt-dead-subprogs")) 1110 check_gotox_jt_dead_subprogs(); 1111 if (test__start_subtest("gotox-target-subprog-from-main")) 1112 check_gotox_target_subprog_from_main(); 1113 if (test__start_subtest("gotox-index-slice-other-subprog")) 1114 check_gotox_index_slice_other_subprog(); 1115 if (test__start_subtest("gotox-target-other-global-subprog")) 1116 check_gotox_target_other_global_subprog(); 1117 if (test__start_subtest("gotox-callback-leaves-subprog")) 1118 check_gotox_callback_leaves_subprog(); 1119 if (test__start_subtest("gotox-target-prologue-shift")) 1120 check_gotox_target_prologue_shift(); 1121 if (test__start_subtest("gotox-target-ctx-prologue-shift")) 1122 check_gotox_target_ctx_prologue_shift(); 1123 } 1124 #else 1125 static void __test_bpf_insn_array(void) 1126 { 1127 test__skip(); 1128 } 1129 #endif 1130 1131 void test_bpf_insn_array(void) 1132 { 1133 __test_bpf_insn_array(); 1134 } 1135