1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * BPF Jit compiler for s390. 4 * 5 * Minimum build requirements: 6 * 7 * - HAVE_MARCH_Z196_FEATURES: laal, laalg 8 * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj 9 * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf 10 * - 64BIT 11 * 12 * Copyright IBM Corp. 2012,2015 13 * 14 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 15 * Michael Holzheu <holzheu@linux.vnet.ibm.com> 16 */ 17 18 #define KMSG_COMPONENT "bpf_jit" 19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 20 21 #include <linux/netdevice.h> 22 #include <linux/filter.h> 23 #include <linux/init.h> 24 #include <linux/bpf.h> 25 #include <linux/mm.h> 26 #include <linux/kernel.h> 27 #include <asm/cacheflush.h> 28 #include <asm/extable.h> 29 #include <asm/dis.h> 30 #include <asm/facility.h> 31 #include <asm/nospec-branch.h> 32 #include <asm/set_memory.h> 33 #include <asm/text-patching.h> 34 #include <asm/unwind.h> 35 36 struct bpf_jit { 37 u32 seen; /* Flags to remember seen eBPF instructions */ 38 u16 seen_regs; /* Mask to remember which registers are used */ 39 u32 *addrs; /* Array with relative instruction addresses */ 40 u8 *prg_buf; /* Start of program */ 41 int size; /* Size of program and literal pool */ 42 int size_prg; /* Size of program */ 43 int prg; /* Current position in program */ 44 int lit32_start; /* Start of 32-bit literal pool */ 45 int lit32; /* Current position in 32-bit literal pool */ 46 int lit64_start; /* Start of 64-bit literal pool */ 47 int lit64; /* Current position in 64-bit literal pool */ 48 int base_ip; /* Base address for literal pool */ 49 int exit_ip; /* Address of exit */ 50 int tail_call_start; /* Tail call start offset */ 51 int excnt; /* Number of exception table entries */ 52 int prologue_plt_ret; /* Return address for prologue hotpatch PLT */ 53 int prologue_plt; /* Start of prologue hotpatch PLT */ 54 int kern_arena; /* Pool offset of kernel arena address */ 55 u64 user_arena; /* User arena address */ 56 u32 frame_off; /* Offset of struct bpf_prog from %r15 */ 57 }; 58 59 #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */ 60 #define SEEN_LITERAL BIT(1) /* code uses literals */ 61 #define SEEN_FUNC BIT(2) /* calls C functions */ 62 #define SEEN_STACK (SEEN_FUNC | SEEN_MEM) 63 64 #define NVREGS 0xffc0 /* %r6-%r15 */ 65 66 /* 67 * s390 registers 68 */ 69 #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ 70 #define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ 71 #define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */ 72 #define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */ 73 #define REG_0 REG_W0 /* Register 0 */ 74 #define REG_1 REG_W1 /* Register 1 */ 75 #define REG_2 BPF_REG_1 /* Register 2 */ 76 #define REG_3 BPF_REG_2 /* Register 3 */ 77 #define REG_4 BPF_REG_3 /* Register 4 */ 78 #define REG_7 BPF_REG_6 /* Register 7 */ 79 #define REG_8 BPF_REG_7 /* Register 8 */ 80 #define REG_14 BPF_REG_0 /* Register 14 */ 81 82 /* 83 * Mapping of BPF registers to s390 registers 84 */ 85 static const int reg2hex[] = { 86 /* Return code */ 87 [BPF_REG_0] = 14, 88 /* Function parameters */ 89 [BPF_REG_1] = 2, 90 [BPF_REG_2] = 3, 91 [BPF_REG_3] = 4, 92 [BPF_REG_4] = 5, 93 [BPF_REG_5] = 6, 94 /* Call saved registers */ 95 [BPF_REG_6] = 7, 96 [BPF_REG_7] = 8, 97 [BPF_REG_8] = 9, 98 [BPF_REG_9] = 10, 99 /* BPF stack pointer */ 100 [BPF_REG_FP] = 13, 101 /* Register for blinding */ 102 [BPF_REG_AX] = 12, 103 /* Work registers for s390x backend */ 104 [REG_W0] = 0, 105 [REG_W1] = 1, 106 [REG_L] = 11, 107 [REG_15] = 15, 108 }; 109 110 static inline u32 reg(u32 dst_reg, u32 src_reg) 111 { 112 return reg2hex[dst_reg] << 4 | reg2hex[src_reg]; 113 } 114 115 static inline u32 reg_high(u32 reg) 116 { 117 return reg2hex[reg] << 4; 118 } 119 120 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 121 { 122 u32 r1 = reg2hex[b1]; 123 124 if (r1 >= 6 && r1 <= 15) 125 jit->seen_regs |= (1 << r1); 126 } 127 128 static s32 off_to_pcrel(struct bpf_jit *jit, u32 off) 129 { 130 return off - jit->prg; 131 } 132 133 static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr) 134 { 135 if (jit->prg_buf) 136 return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg); 137 return 0; 138 } 139 140 #define REG_SET_SEEN(b1) \ 141 ({ \ 142 reg_set_seen(jit, b1); \ 143 }) 144 145 /* 146 * EMIT macros for code generation 147 */ 148 149 #define _EMIT2(op) \ 150 ({ \ 151 if (jit->prg_buf) \ 152 *(u16 *) (jit->prg_buf + jit->prg) = (op); \ 153 jit->prg += 2; \ 154 }) 155 156 #define EMIT2(op, b1, b2) \ 157 ({ \ 158 _EMIT2((op) | reg(b1, b2)); \ 159 REG_SET_SEEN(b1); \ 160 REG_SET_SEEN(b2); \ 161 }) 162 163 #define _EMIT4(op) \ 164 ({ \ 165 if (jit->prg_buf) \ 166 *(u32 *) (jit->prg_buf + jit->prg) = (op); \ 167 jit->prg += 4; \ 168 }) 169 170 #define EMIT4(op, b1, b2) \ 171 ({ \ 172 _EMIT4((op) | reg(b1, b2)); \ 173 REG_SET_SEEN(b1); \ 174 REG_SET_SEEN(b2); \ 175 }) 176 177 #define EMIT4_RRF(op, b1, b2, b3) \ 178 ({ \ 179 _EMIT4((op) | reg_high(b3) << 8 | reg(b1, b2)); \ 180 REG_SET_SEEN(b1); \ 181 REG_SET_SEEN(b2); \ 182 REG_SET_SEEN(b3); \ 183 }) 184 185 #define _EMIT4_DISP(op, disp) \ 186 ({ \ 187 unsigned int __disp = (disp) & 0xfff; \ 188 _EMIT4((op) | __disp); \ 189 }) 190 191 #define EMIT4_DISP(op, b1, b2, disp) \ 192 ({ \ 193 _EMIT4_DISP((op) | reg_high(b1) << 16 | \ 194 reg_high(b2) << 8, (disp)); \ 195 REG_SET_SEEN(b1); \ 196 REG_SET_SEEN(b2); \ 197 }) 198 199 #define EMIT4_IMM(op, b1, imm) \ 200 ({ \ 201 unsigned int __imm = (imm) & 0xffff; \ 202 _EMIT4((op) | reg_high(b1) << 16 | __imm); \ 203 REG_SET_SEEN(b1); \ 204 }) 205 206 #define EMIT4_PCREL(op, pcrel) \ 207 ({ \ 208 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 209 _EMIT4((op) | __pcrel); \ 210 }) 211 212 #define EMIT4_PCREL_RIC(op, mask, target) \ 213 ({ \ 214 int __rel = off_to_pcrel(jit, target) / 2; \ 215 _EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \ 216 }) 217 218 #define _EMIT6(op1, op2) \ 219 ({ \ 220 if (jit->prg_buf) { \ 221 *(u32 *) (jit->prg_buf + jit->prg) = (op1); \ 222 *(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \ 223 } \ 224 jit->prg += 6; \ 225 }) 226 227 #define _EMIT6_DISP(op1, op2, disp) \ 228 ({ \ 229 unsigned int __disp = (disp) & 0xfff; \ 230 _EMIT6((op1) | __disp, op2); \ 231 }) 232 233 #define _EMIT6_DISP_LH(op1, op2, disp) \ 234 ({ \ 235 u32 _disp = (u32) (disp); \ 236 unsigned int __disp_h = _disp & 0xff000; \ 237 unsigned int __disp_l = _disp & 0x00fff; \ 238 _EMIT6((op1) | __disp_l, (op2) | __disp_h >> 4); \ 239 }) 240 241 #define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \ 242 ({ \ 243 _EMIT6_DISP_LH((op1) | reg(b1, b2) << 16 | \ 244 reg_high(b3) << 8, op2, disp); \ 245 REG_SET_SEEN(b1); \ 246 REG_SET_SEEN(b2); \ 247 REG_SET_SEEN(b3); \ 248 }) 249 250 #define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \ 251 ({ \ 252 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 253 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \ 254 (op2) | (mask) << 12); \ 255 REG_SET_SEEN(b1); \ 256 REG_SET_SEEN(b2); \ 257 }) 258 259 #define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \ 260 ({ \ 261 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 262 _EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \ 263 (rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \ 264 REG_SET_SEEN(b1); \ 265 BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \ 266 }) 267 268 #define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \ 269 ({ \ 270 int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\ 271 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\ 272 REG_SET_SEEN(b1); \ 273 REG_SET_SEEN(b2); \ 274 }) 275 276 static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel) 277 { 278 u32 pc32dbl = (s32)(pcrel / 2); 279 280 _EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff); 281 } 282 283 static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel) 284 { 285 emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel); 286 REG_SET_SEEN(b); 287 } 288 289 #define EMIT6_PCREL_RILB(op, b, target) \ 290 emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target)) 291 292 #define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \ 293 emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr)) 294 295 static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel) 296 { 297 emit6_pcrel_ril(jit, op | mask << 20, pcrel); 298 } 299 300 #define EMIT6_PCREL_RILC(op, mask, target) \ 301 emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target)) 302 303 #define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \ 304 emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr)) 305 306 #define _EMIT6_IMM(op, imm) \ 307 ({ \ 308 unsigned int __imm = (imm); \ 309 _EMIT6((op) | (__imm >> 16), __imm & 0xffff); \ 310 }) 311 312 #define EMIT6_IMM(op, b1, imm) \ 313 ({ \ 314 _EMIT6_IMM((op) | reg_high(b1) << 16, imm); \ 315 REG_SET_SEEN(b1); \ 316 }) 317 318 #define _EMIT_CONST_U32(val) \ 319 ({ \ 320 unsigned int ret; \ 321 ret = jit->lit32; \ 322 if (jit->prg_buf) \ 323 *(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\ 324 jit->lit32 += 4; \ 325 ret; \ 326 }) 327 328 #define EMIT_CONST_U32(val) \ 329 ({ \ 330 jit->seen |= SEEN_LITERAL; \ 331 _EMIT_CONST_U32(val) - jit->base_ip; \ 332 }) 333 334 #define _EMIT_CONST_U64(val) \ 335 ({ \ 336 unsigned int ret; \ 337 ret = jit->lit64; \ 338 if (jit->prg_buf) \ 339 *(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\ 340 jit->lit64 += 8; \ 341 ret; \ 342 }) 343 344 #define EMIT_CONST_U64(val) \ 345 ({ \ 346 jit->seen |= SEEN_LITERAL; \ 347 _EMIT_CONST_U64(val) - jit->base_ip; \ 348 }) 349 350 #define EMIT_ZERO(b1) \ 351 ({ \ 352 if (!fp->aux->verifier_zext) { \ 353 /* llgfr %dst,%dst (zero extend to 64 bit) */ \ 354 EMIT4(0xb9160000, b1, b1); \ 355 REG_SET_SEEN(b1); \ 356 } \ 357 }) 358 359 /* 360 * Return whether this is the first pass. The first pass is special, since we 361 * don't know any sizes yet, and thus must be conservative. 362 */ 363 static bool is_first_pass(struct bpf_jit *jit) 364 { 365 return jit->size == 0; 366 } 367 368 /* 369 * Return whether this is the code generation pass. The code generation pass is 370 * special, since we should change as little as possible. 371 */ 372 static bool is_codegen_pass(struct bpf_jit *jit) 373 { 374 return jit->prg_buf; 375 } 376 377 /* 378 * Return whether "rel" can be encoded as a short PC-relative offset 379 */ 380 static bool is_valid_rel(int rel) 381 { 382 return rel >= -65536 && rel <= 65534; 383 } 384 385 /* 386 * Return whether "off" can be reached using a short PC-relative offset 387 */ 388 static bool can_use_rel(struct bpf_jit *jit, int off) 389 { 390 return is_valid_rel(off - jit->prg); 391 } 392 393 /* 394 * Return whether given displacement can be encoded using 395 * Long-Displacement Facility 396 */ 397 static bool is_valid_ldisp(int disp) 398 { 399 return disp >= -524288 && disp <= 524287; 400 } 401 402 /* 403 * Return whether the next 32-bit literal pool entry can be referenced using 404 * Long-Displacement Facility 405 */ 406 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit) 407 { 408 return is_valid_ldisp(jit->lit32 - jit->base_ip); 409 } 410 411 /* 412 * Return whether the next 64-bit literal pool entry can be referenced using 413 * Long-Displacement Facility 414 */ 415 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit) 416 { 417 return is_valid_ldisp(jit->lit64 - jit->base_ip); 418 } 419 420 /* 421 * Fill whole space with illegal instructions 422 */ 423 static void jit_fill_hole(void *area, unsigned int size) 424 { 425 memset(area, 0, size); 426 } 427 428 /* 429 * Caller-allocated part of the frame. 430 * Thanks to packed stack, its otherwise unused initial part can be used for 431 * the BPF stack and for the next frame. 432 */ 433 struct prog_frame { 434 u64 unused[8]; 435 /* BPF stack starts here and grows towards 0 */ 436 u32 tail_call_cnt; 437 u32 pad; 438 u64 r6[10]; /* r6 - r15 */ 439 u64 backchain; 440 } __packed; 441 442 /* 443 * Save registers from "rs" (register start) to "re" (register end) on stack 444 */ 445 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) 446 { 447 u32 off = offsetof(struct prog_frame, r6) + (rs - 6) * 8; 448 449 if (rs == re) 450 /* stg %rs,off(%r15) */ 451 _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); 452 else 453 /* stmg %rs,%re,off(%r15) */ 454 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); 455 } 456 457 /* 458 * Restore registers from "rs" (register start) to "re" (register end) on stack 459 */ 460 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re) 461 { 462 u32 off = jit->frame_off + offsetof(struct prog_frame, r6) + (rs - 6) * 8; 463 464 if (rs == re) 465 /* lg %rs,off(%r15) */ 466 _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); 467 else 468 /* lmg %rs,%re,off(%r15) */ 469 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); 470 } 471 472 /* 473 * Return first seen register (from start) 474 */ 475 static int get_start(u16 seen_regs, int start) 476 { 477 int i; 478 479 for (i = start; i <= 15; i++) { 480 if (seen_regs & (1 << i)) 481 return i; 482 } 483 return 0; 484 } 485 486 /* 487 * Return last seen register (from start) (gap >= 2) 488 */ 489 static int get_end(u16 seen_regs, int start) 490 { 491 int i; 492 493 for (i = start; i < 15; i++) { 494 if (!(seen_regs & (3 << i))) 495 return i - 1; 496 } 497 return (seen_regs & (1 << 15)) ? 15 : 14; 498 } 499 500 #define REGS_SAVE 1 501 #define REGS_RESTORE 0 502 /* 503 * Save and restore clobbered registers (6-15) on stack. 504 * We save/restore registers in chunks with gap >= 2 registers. 505 */ 506 static void save_restore_regs(struct bpf_jit *jit, int op, u16 extra_regs) 507 { 508 u16 seen_regs = jit->seen_regs | extra_regs; 509 const int last = 15, save_restore_size = 6; 510 int re = 6, rs; 511 512 if (is_first_pass(jit)) { 513 /* 514 * We don't know yet which registers are used. Reserve space 515 * conservatively. 516 */ 517 jit->prg += (last - re + 1) * save_restore_size; 518 return; 519 } 520 521 do { 522 rs = get_start(seen_regs, re); 523 if (!rs) 524 break; 525 re = get_end(seen_regs, rs + 1); 526 if (op == REGS_SAVE) 527 save_regs(jit, rs, re); 528 else 529 restore_regs(jit, rs, re); 530 re++; 531 } while (re <= last); 532 } 533 534 static void bpf_skip(struct bpf_jit *jit, int size) 535 { 536 if (size >= 6 && !is_valid_rel(size)) { 537 /* brcl 0xf,size */ 538 EMIT6_PCREL_RILC(0xc0040000, 0xf, size); 539 size -= 6; 540 } else if (size >= 4 && is_valid_rel(size)) { 541 /* brc 0xf,size */ 542 EMIT4_PCREL(0xa7f40000, size); 543 size -= 4; 544 } 545 while (size >= 2) { 546 /* bcr 0,%0 */ 547 _EMIT2(0x0700); 548 size -= 2; 549 } 550 } 551 552 /* 553 * PLT for hotpatchable calls. The calling convention is the same as for the 554 * ftrace hotpatch trampolines: %r0 is return address, %r1 is clobbered. 555 */ 556 struct bpf_plt { 557 char code[16]; 558 void *ret; 559 void *target; 560 } __packed; 561 extern const struct bpf_plt bpf_plt; 562 asm( 563 ".pushsection .rodata\n" 564 " .balign 8\n" 565 "bpf_plt:\n" 566 " lgrl %r0,bpf_plt_ret\n" 567 " lgrl %r1,bpf_plt_target\n" 568 " br %r1\n" 569 " .balign 8\n" 570 "bpf_plt_ret: .quad 0\n" 571 "bpf_plt_target: .quad 0\n" 572 " .popsection\n" 573 ); 574 575 static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target) 576 { 577 memcpy(plt, &bpf_plt, sizeof(*plt)); 578 plt->ret = ret; 579 plt->target = target; 580 } 581 582 /* 583 * Emit function prologue 584 * 585 * Save registers and create stack frame if necessary. 586 * Stack frame layout is described by struct prog_frame. 587 */ 588 static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) 589 { 590 BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD); 591 592 /* No-op for hotpatching */ 593 /* brcl 0,prologue_plt */ 594 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); 595 jit->prologue_plt_ret = jit->prg; 596 597 if (!bpf_is_subprog(fp)) { 598 /* Initialize the tail call counter in the main program. */ 599 /* xc tail_call_cnt(4,%r15),tail_call_cnt(%r15) */ 600 _EMIT6(0xd703f000 | offsetof(struct prog_frame, tail_call_cnt), 601 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); 602 } else { 603 /* 604 * Skip the tail call counter initialization in subprograms. 605 * Insert nops in order to have tail_call_start at a 606 * predictable offset. 607 */ 608 bpf_skip(jit, 6); 609 } 610 /* Tail calls have to skip above initialization */ 611 jit->tail_call_start = jit->prg; 612 if (fp->aux->exception_cb) { 613 /* 614 * Switch stack, the new address is in the 2nd parameter. 615 * 616 * Arrange the restoration of %r6-%r15 in the epilogue. 617 * Do not restore them now, the prog does not need them. 618 */ 619 /* lgr %r15,%r3 */ 620 EMIT4(0xb9040000, REG_15, REG_3); 621 jit->seen_regs |= NVREGS; 622 } else { 623 /* Save registers */ 624 save_restore_regs(jit, REGS_SAVE, 625 fp->aux->exception_boundary ? NVREGS : 0); 626 } 627 /* Setup literal pool */ 628 if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) { 629 if (!is_first_pass(jit) && 630 is_valid_ldisp(jit->size - (jit->prg + 2))) { 631 /* basr %l,0 */ 632 EMIT2(0x0d00, REG_L, REG_0); 633 jit->base_ip = jit->prg; 634 } else { 635 /* larl %l,lit32_start */ 636 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start); 637 jit->base_ip = jit->lit32_start; 638 } 639 } 640 /* Setup stack and backchain */ 641 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) { 642 /* lgr %w1,%r15 (backchain) */ 643 EMIT4(0xb9040000, REG_W1, REG_15); 644 /* la %bfp,unused_end(%r15) (BPF frame pointer) */ 645 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, 646 offsetofend(struct prog_frame, unused)); 647 /* aghi %r15,-frame_off */ 648 EMIT4_IMM(0xa70b0000, REG_15, -jit->frame_off); 649 /* stg %w1,backchain(%r15) */ 650 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, 651 REG_15, 652 offsetof(struct prog_frame, backchain)); 653 } 654 } 655 656 /* 657 * Jump using a register either directly or via an expoline thunk 658 */ 659 #define EMIT_JUMP_REG(reg) do { \ 660 if (nospec_uses_trampoline()) \ 661 /* brcl 0xf,__s390_indirect_jump_rN */ \ 662 EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \ 663 __s390_indirect_jump_r ## reg); \ 664 else \ 665 /* br %rN */ \ 666 _EMIT2(0x07f0 | reg); \ 667 } while (0) 668 669 /* 670 * Call r1 either directly or via __s390_indirect_jump_r1 thunk 671 */ 672 static void call_r1(struct bpf_jit *jit) 673 { 674 if (nospec_uses_trampoline()) 675 /* brasl %r14,__s390_indirect_jump_r1 */ 676 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 677 __s390_indirect_jump_r1); 678 else 679 /* basr %r14,%r1 */ 680 EMIT2(0x0d00, REG_14, REG_1); 681 } 682 683 /* 684 * Function epilogue 685 */ 686 static void bpf_jit_epilogue(struct bpf_jit *jit) 687 { 688 jit->exit_ip = jit->prg; 689 /* Load exit code: lgr %r2,%b0 */ 690 EMIT4(0xb9040000, REG_2, BPF_REG_0); 691 /* Restore registers */ 692 save_restore_regs(jit, REGS_RESTORE, 0); 693 EMIT_JUMP_REG(14); 694 695 jit->prg = ALIGN(jit->prg, 8); 696 jit->prologue_plt = jit->prg; 697 if (jit->prg_buf) 698 bpf_jit_plt((struct bpf_plt *)(jit->prg_buf + jit->prg), 699 jit->prg_buf + jit->prologue_plt_ret, NULL); 700 jit->prg += sizeof(struct bpf_plt); 701 } 702 703 bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) 704 { 705 regs->psw.addr = extable_fixup(x); 706 if (x->data != -1) 707 regs->gprs[x->data] = 0; 708 return true; 709 } 710 711 /* 712 * A single BPF probe instruction 713 */ 714 struct bpf_jit_probe { 715 int prg; /* JITed instruction offset */ 716 int nop_prg; /* JITed nop offset */ 717 int reg; /* Register to clear on exception */ 718 int arena_reg; /* Register to use for arena addressing */ 719 }; 720 721 static void bpf_jit_probe_init(struct bpf_jit_probe *probe) 722 { 723 probe->prg = -1; 724 probe->nop_prg = -1; 725 probe->reg = -1; 726 probe->arena_reg = REG_0; 727 } 728 729 /* 730 * Handlers of certain exceptions leave psw.addr pointing to the instruction 731 * directly after the failing one. Therefore, create two exception table 732 * entries and also add a nop in case two probing instructions come directly 733 * after each other. 734 */ 735 static void bpf_jit_probe_emit_nop(struct bpf_jit *jit, 736 struct bpf_jit_probe *probe) 737 { 738 if (probe->prg == -1 || probe->nop_prg != -1) 739 /* The probe is not armed or nop is already emitted. */ 740 return; 741 742 probe->nop_prg = jit->prg; 743 /* bcr 0,%0 */ 744 _EMIT2(0x0700); 745 } 746 747 static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, 748 struct bpf_jit_probe *probe) 749 { 750 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 751 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 752 BPF_MODE(insn->code) != BPF_PROBE_MEM32) 753 return; 754 755 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { 756 /* lgrl %r1,kern_arena */ 757 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 758 probe->arena_reg = REG_W1; 759 } 760 probe->prg = jit->prg; 761 probe->reg = reg2hex[insn->dst_reg]; 762 } 763 764 static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, 765 struct bpf_jit_probe *probe) 766 { 767 if (BPF_MODE(insn->code) != BPF_PROBE_MEM32) 768 return; 769 770 /* lgrl %r1,kern_arena */ 771 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 772 probe->arena_reg = REG_W1; 773 probe->prg = jit->prg; 774 } 775 776 static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, 777 struct bpf_insn *insn, 778 struct bpf_jit_probe *probe) 779 { 780 if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 781 return; 782 783 /* lgrl %r1,kern_arena */ 784 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 785 /* agr %r1,%dst */ 786 EMIT4(0xb9080000, REG_W1, insn->dst_reg); 787 probe->arena_reg = REG_W1; 788 probe->prg = jit->prg; 789 } 790 791 static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, 792 struct bpf_jit_probe *probe) 793 { 794 struct exception_table_entry *ex; 795 int i, prg; 796 s64 delta; 797 u8 *insn; 798 799 if (probe->prg == -1) 800 /* The probe is not armed. */ 801 return 0; 802 bpf_jit_probe_emit_nop(jit, probe); 803 if (!fp->aux->extable) 804 /* Do nothing during early JIT passes. */ 805 return 0; 806 insn = jit->prg_buf + probe->prg; 807 if (WARN_ON_ONCE(probe->prg + insn_length(*insn) != probe->nop_prg)) 808 /* JIT bug - gap between probe and nop instructions. */ 809 return -1; 810 for (i = 0; i < 2; i++) { 811 if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) 812 /* Verifier bug - not enough entries. */ 813 return -1; 814 ex = &fp->aux->extable[jit->excnt]; 815 /* Add extable entries for probe and nop instructions. */ 816 prg = i == 0 ? probe->prg : probe->nop_prg; 817 delta = jit->prg_buf + prg - (u8 *)&ex->insn; 818 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 819 /* JIT bug - code and extable must be close. */ 820 return -1; 821 ex->insn = delta; 822 /* 823 * Land on the current instruction. Note that the extable 824 * infrastructure ignores the fixup field; it is handled by 825 * ex_handler_bpf(). 826 */ 827 delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; 828 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 829 /* JIT bug - landing pad and extable must be close. */ 830 return -1; 831 ex->fixup = delta; 832 ex->type = EX_TYPE_BPF; 833 ex->data = probe->reg; 834 jit->excnt++; 835 } 836 return 0; 837 } 838 839 /* 840 * Sign-extend the register if necessary 841 */ 842 static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) 843 { 844 if (!(flags & BTF_FMODEL_SIGNED_ARG)) 845 return 0; 846 847 switch (size) { 848 case 1: 849 /* lgbr %r,%r */ 850 EMIT4(0xb9060000, r, r); 851 return 0; 852 case 2: 853 /* lghr %r,%r */ 854 EMIT4(0xb9070000, r, r); 855 return 0; 856 case 4: 857 /* lgfr %r,%r */ 858 EMIT4(0xb9140000, r, r); 859 return 0; 860 case 8: 861 return 0; 862 default: 863 return -1; 864 } 865 } 866 867 /* 868 * Compile one eBPF instruction into s390x code 869 * 870 * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of 871 * stack space for the large switch statement. 872 */ 873 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 874 int i, bool extra_pass) 875 { 876 struct bpf_insn *insn = &fp->insnsi[i]; 877 s32 branch_oc_off = insn->off; 878 u32 dst_reg = insn->dst_reg; 879 u32 src_reg = insn->src_reg; 880 struct bpf_jit_probe probe; 881 int last, insn_count = 1; 882 u32 *addrs = jit->addrs; 883 s32 imm = insn->imm; 884 s16 off = insn->off; 885 unsigned int mask; 886 int err; 887 888 bpf_jit_probe_init(&probe); 889 890 switch (insn->code) { 891 /* 892 * BPF_MOV 893 */ 894 case BPF_ALU | BPF_MOV | BPF_X: 895 switch (insn->off) { 896 case 0: /* DST = (u32) SRC */ 897 /* llgfr %dst,%src */ 898 EMIT4(0xb9160000, dst_reg, src_reg); 899 if (insn_is_zext(&insn[1])) 900 insn_count = 2; 901 break; 902 case 8: /* DST = (u32)(s8) SRC */ 903 /* lbr %dst,%src */ 904 EMIT4(0xb9260000, dst_reg, src_reg); 905 /* llgfr %dst,%dst */ 906 EMIT4(0xb9160000, dst_reg, dst_reg); 907 break; 908 case 16: /* DST = (u32)(s16) SRC */ 909 /* lhr %dst,%src */ 910 EMIT4(0xb9270000, dst_reg, src_reg); 911 /* llgfr %dst,%dst */ 912 EMIT4(0xb9160000, dst_reg, dst_reg); 913 break; 914 } 915 break; 916 case BPF_ALU64 | BPF_MOV | BPF_X: 917 if (insn_is_cast_user(insn)) { 918 int patch_brc; 919 920 /* ltgr %dst,%src */ 921 EMIT4(0xb9020000, dst_reg, src_reg); 922 /* brc 8,0f */ 923 patch_brc = jit->prg; 924 EMIT4_PCREL_RIC(0xa7040000, 8, 0); 925 /* iihf %dst,user_arena>>32 */ 926 EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32); 927 /* 0: */ 928 if (jit->prg_buf) 929 *(u16 *)(jit->prg_buf + patch_brc + 2) = 930 (jit->prg - patch_brc) >> 1; 931 break; 932 } 933 switch (insn->off) { 934 case 0: /* DST = SRC */ 935 /* lgr %dst,%src */ 936 EMIT4(0xb9040000, dst_reg, src_reg); 937 break; 938 case 8: /* DST = (s8) SRC */ 939 /* lgbr %dst,%src */ 940 EMIT4(0xb9060000, dst_reg, src_reg); 941 break; 942 case 16: /* DST = (s16) SRC */ 943 /* lghr %dst,%src */ 944 EMIT4(0xb9070000, dst_reg, src_reg); 945 break; 946 case 32: /* DST = (s32) SRC */ 947 /* lgfr %dst,%src */ 948 EMIT4(0xb9140000, dst_reg, src_reg); 949 break; 950 } 951 break; 952 case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */ 953 /* llilf %dst,imm */ 954 EMIT6_IMM(0xc00f0000, dst_reg, imm); 955 if (insn_is_zext(&insn[1])) 956 insn_count = 2; 957 break; 958 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */ 959 /* lgfi %dst,imm */ 960 EMIT6_IMM(0xc0010000, dst_reg, imm); 961 break; 962 /* 963 * BPF_LD 64 964 */ 965 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 966 { 967 /* 16 byte instruction that uses two 'struct bpf_insn' */ 968 u64 imm64; 969 970 imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; 971 /* lgrl %dst,imm */ 972 EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64)); 973 insn_count = 2; 974 break; 975 } 976 /* 977 * BPF_ADD 978 */ 979 case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */ 980 /* ar %dst,%src */ 981 EMIT2(0x1a00, dst_reg, src_reg); 982 EMIT_ZERO(dst_reg); 983 break; 984 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */ 985 /* agr %dst,%src */ 986 EMIT4(0xb9080000, dst_reg, src_reg); 987 break; 988 case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */ 989 if (imm != 0) { 990 /* alfi %dst,imm */ 991 EMIT6_IMM(0xc20b0000, dst_reg, imm); 992 } 993 EMIT_ZERO(dst_reg); 994 break; 995 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */ 996 if (!imm) 997 break; 998 /* agfi %dst,imm */ 999 EMIT6_IMM(0xc2080000, dst_reg, imm); 1000 break; 1001 /* 1002 * BPF_SUB 1003 */ 1004 case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */ 1005 /* sr %dst,%src */ 1006 EMIT2(0x1b00, dst_reg, src_reg); 1007 EMIT_ZERO(dst_reg); 1008 break; 1009 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */ 1010 /* sgr %dst,%src */ 1011 EMIT4(0xb9090000, dst_reg, src_reg); 1012 break; 1013 case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */ 1014 if (imm != 0) { 1015 /* alfi %dst,-imm */ 1016 EMIT6_IMM(0xc20b0000, dst_reg, -imm); 1017 } 1018 EMIT_ZERO(dst_reg); 1019 break; 1020 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */ 1021 if (!imm) 1022 break; 1023 if (imm == -0x80000000) { 1024 /* algfi %dst,0x80000000 */ 1025 EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000); 1026 } else { 1027 /* agfi %dst,-imm */ 1028 EMIT6_IMM(0xc2080000, dst_reg, -imm); 1029 } 1030 break; 1031 /* 1032 * BPF_MUL 1033 */ 1034 case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */ 1035 /* msr %dst,%src */ 1036 EMIT4(0xb2520000, dst_reg, src_reg); 1037 EMIT_ZERO(dst_reg); 1038 break; 1039 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */ 1040 /* msgr %dst,%src */ 1041 EMIT4(0xb90c0000, dst_reg, src_reg); 1042 break; 1043 case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */ 1044 if (imm != 1) { 1045 /* msfi %r5,imm */ 1046 EMIT6_IMM(0xc2010000, dst_reg, imm); 1047 } 1048 EMIT_ZERO(dst_reg); 1049 break; 1050 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */ 1051 if (imm == 1) 1052 break; 1053 /* msgfi %dst,imm */ 1054 EMIT6_IMM(0xc2000000, dst_reg, imm); 1055 break; 1056 /* 1057 * BPF_DIV / BPF_MOD 1058 */ 1059 case BPF_ALU | BPF_DIV | BPF_X: 1060 case BPF_ALU | BPF_MOD | BPF_X: 1061 { 1062 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1063 1064 switch (off) { 1065 case 0: /* dst = (u32) dst {/,%} (u32) src */ 1066 /* xr %w0,%w0 */ 1067 EMIT2(0x1700, REG_W0, REG_W0); 1068 /* lr %w1,%dst */ 1069 EMIT2(0x1800, REG_W1, dst_reg); 1070 /* dlr %w0,%src */ 1071 EMIT4(0xb9970000, REG_W0, src_reg); 1072 break; 1073 case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */ 1074 /* lgfr %r1,%dst */ 1075 EMIT4(0xb9140000, REG_W1, dst_reg); 1076 /* dsgfr %r0,%src */ 1077 EMIT4(0xb91d0000, REG_W0, src_reg); 1078 break; 1079 } 1080 /* llgfr %dst,%rc */ 1081 EMIT4(0xb9160000, dst_reg, rc_reg); 1082 if (insn_is_zext(&insn[1])) 1083 insn_count = 2; 1084 break; 1085 } 1086 case BPF_ALU64 | BPF_DIV | BPF_X: 1087 case BPF_ALU64 | BPF_MOD | BPF_X: 1088 { 1089 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1090 1091 switch (off) { 1092 case 0: /* dst = dst {/,%} src */ 1093 /* lghi %w0,0 */ 1094 EMIT4_IMM(0xa7090000, REG_W0, 0); 1095 /* lgr %w1,%dst */ 1096 EMIT4(0xb9040000, REG_W1, dst_reg); 1097 /* dlgr %w0,%src */ 1098 EMIT4(0xb9870000, REG_W0, src_reg); 1099 break; 1100 case 1: /* dst = (s64) dst {/,%} (s64) src */ 1101 /* lgr %w1,%dst */ 1102 EMIT4(0xb9040000, REG_W1, dst_reg); 1103 /* dsgr %w0,%src */ 1104 EMIT4(0xb90d0000, REG_W0, src_reg); 1105 break; 1106 } 1107 /* lgr %dst,%rc */ 1108 EMIT4(0xb9040000, dst_reg, rc_reg); 1109 break; 1110 } 1111 case BPF_ALU | BPF_DIV | BPF_K: 1112 case BPF_ALU | BPF_MOD | BPF_K: 1113 { 1114 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1115 1116 if (imm == 1) { 1117 if (BPF_OP(insn->code) == BPF_MOD) 1118 /* lghi %dst,0 */ 1119 EMIT4_IMM(0xa7090000, dst_reg, 0); 1120 else 1121 EMIT_ZERO(dst_reg); 1122 break; 1123 } 1124 if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) { 1125 switch (off) { 1126 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1127 /* xr %w0,%w0 */ 1128 EMIT2(0x1700, REG_W0, REG_W0); 1129 /* lr %w1,%dst */ 1130 EMIT2(0x1800, REG_W1, dst_reg); 1131 /* dl %w0,<d(imm)>(%l) */ 1132 EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, 1133 REG_L, EMIT_CONST_U32(imm)); 1134 break; 1135 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1136 /* lgfr %r1,%dst */ 1137 EMIT4(0xb9140000, REG_W1, dst_reg); 1138 /* dsgf %r0,<d(imm)>(%l) */ 1139 EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0, 1140 REG_L, EMIT_CONST_U32(imm)); 1141 break; 1142 } 1143 } else { 1144 switch (off) { 1145 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1146 /* xr %w0,%w0 */ 1147 EMIT2(0x1700, REG_W0, REG_W0); 1148 /* lr %w1,%dst */ 1149 EMIT2(0x1800, REG_W1, dst_reg); 1150 /* lrl %dst,imm */ 1151 EMIT6_PCREL_RILB(0xc40d0000, dst_reg, 1152 _EMIT_CONST_U32(imm)); 1153 jit->seen |= SEEN_LITERAL; 1154 /* dlr %w0,%dst */ 1155 EMIT4(0xb9970000, REG_W0, dst_reg); 1156 break; 1157 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1158 /* lgfr %w1,%dst */ 1159 EMIT4(0xb9140000, REG_W1, dst_reg); 1160 /* lgfrl %dst,imm */ 1161 EMIT6_PCREL_RILB(0xc40c0000, dst_reg, 1162 _EMIT_CONST_U32(imm)); 1163 jit->seen |= SEEN_LITERAL; 1164 /* dsgr %w0,%dst */ 1165 EMIT4(0xb90d0000, REG_W0, dst_reg); 1166 break; 1167 } 1168 } 1169 /* llgfr %dst,%rc */ 1170 EMIT4(0xb9160000, dst_reg, rc_reg); 1171 if (insn_is_zext(&insn[1])) 1172 insn_count = 2; 1173 break; 1174 } 1175 case BPF_ALU64 | BPF_DIV | BPF_K: 1176 case BPF_ALU64 | BPF_MOD | BPF_K: 1177 { 1178 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1179 1180 if (imm == 1) { 1181 if (BPF_OP(insn->code) == BPF_MOD) 1182 /* lhgi %dst,0 */ 1183 EMIT4_IMM(0xa7090000, dst_reg, 0); 1184 break; 1185 } 1186 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1187 switch (off) { 1188 case 0: /* dst = dst {/,%} imm */ 1189 /* lghi %w0,0 */ 1190 EMIT4_IMM(0xa7090000, REG_W0, 0); 1191 /* lgr %w1,%dst */ 1192 EMIT4(0xb9040000, REG_W1, dst_reg); 1193 /* dlg %w0,<d(imm)>(%l) */ 1194 EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, 1195 REG_L, EMIT_CONST_U64(imm)); 1196 break; 1197 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1198 /* lgr %w1,%dst */ 1199 EMIT4(0xb9040000, REG_W1, dst_reg); 1200 /* dsg %w0,<d(imm)>(%l) */ 1201 EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0, 1202 REG_L, EMIT_CONST_U64(imm)); 1203 break; 1204 } 1205 } else { 1206 switch (off) { 1207 case 0: /* dst = dst {/,%} imm */ 1208 /* lghi %w0,0 */ 1209 EMIT4_IMM(0xa7090000, REG_W0, 0); 1210 /* lgr %w1,%dst */ 1211 EMIT4(0xb9040000, REG_W1, dst_reg); 1212 /* lgrl %dst,imm */ 1213 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1214 _EMIT_CONST_U64(imm)); 1215 jit->seen |= SEEN_LITERAL; 1216 /* dlgr %w0,%dst */ 1217 EMIT4(0xb9870000, REG_W0, dst_reg); 1218 break; 1219 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1220 /* lgr %w1,%dst */ 1221 EMIT4(0xb9040000, REG_W1, dst_reg); 1222 /* lgrl %dst,imm */ 1223 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1224 _EMIT_CONST_U64(imm)); 1225 jit->seen |= SEEN_LITERAL; 1226 /* dsgr %w0,%dst */ 1227 EMIT4(0xb90d0000, REG_W0, dst_reg); 1228 break; 1229 } 1230 } 1231 /* lgr %dst,%rc */ 1232 EMIT4(0xb9040000, dst_reg, rc_reg); 1233 break; 1234 } 1235 /* 1236 * BPF_AND 1237 */ 1238 case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */ 1239 /* nr %dst,%src */ 1240 EMIT2(0x1400, dst_reg, src_reg); 1241 EMIT_ZERO(dst_reg); 1242 break; 1243 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 1244 /* ngr %dst,%src */ 1245 EMIT4(0xb9800000, dst_reg, src_reg); 1246 break; 1247 case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */ 1248 /* nilf %dst,imm */ 1249 EMIT6_IMM(0xc00b0000, dst_reg, imm); 1250 EMIT_ZERO(dst_reg); 1251 break; 1252 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 1253 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1254 /* ng %dst,<d(imm)>(%l) */ 1255 EMIT6_DISP_LH(0xe3000000, 0x0080, 1256 dst_reg, REG_0, REG_L, 1257 EMIT_CONST_U64(imm)); 1258 } else { 1259 /* lgrl %w0,imm */ 1260 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1261 _EMIT_CONST_U64(imm)); 1262 jit->seen |= SEEN_LITERAL; 1263 /* ngr %dst,%w0 */ 1264 EMIT4(0xb9800000, dst_reg, REG_W0); 1265 } 1266 break; 1267 /* 1268 * BPF_OR 1269 */ 1270 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 1271 /* or %dst,%src */ 1272 EMIT2(0x1600, dst_reg, src_reg); 1273 EMIT_ZERO(dst_reg); 1274 break; 1275 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 1276 /* ogr %dst,%src */ 1277 EMIT4(0xb9810000, dst_reg, src_reg); 1278 break; 1279 case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */ 1280 /* oilf %dst,imm */ 1281 EMIT6_IMM(0xc00d0000, dst_reg, imm); 1282 EMIT_ZERO(dst_reg); 1283 break; 1284 case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */ 1285 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1286 /* og %dst,<d(imm)>(%l) */ 1287 EMIT6_DISP_LH(0xe3000000, 0x0081, 1288 dst_reg, REG_0, REG_L, 1289 EMIT_CONST_U64(imm)); 1290 } else { 1291 /* lgrl %w0,imm */ 1292 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1293 _EMIT_CONST_U64(imm)); 1294 jit->seen |= SEEN_LITERAL; 1295 /* ogr %dst,%w0 */ 1296 EMIT4(0xb9810000, dst_reg, REG_W0); 1297 } 1298 break; 1299 /* 1300 * BPF_XOR 1301 */ 1302 case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */ 1303 /* xr %dst,%src */ 1304 EMIT2(0x1700, dst_reg, src_reg); 1305 EMIT_ZERO(dst_reg); 1306 break; 1307 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */ 1308 /* xgr %dst,%src */ 1309 EMIT4(0xb9820000, dst_reg, src_reg); 1310 break; 1311 case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */ 1312 if (imm != 0) { 1313 /* xilf %dst,imm */ 1314 EMIT6_IMM(0xc0070000, dst_reg, imm); 1315 } 1316 EMIT_ZERO(dst_reg); 1317 break; 1318 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */ 1319 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1320 /* xg %dst,<d(imm)>(%l) */ 1321 EMIT6_DISP_LH(0xe3000000, 0x0082, 1322 dst_reg, REG_0, REG_L, 1323 EMIT_CONST_U64(imm)); 1324 } else { 1325 /* lgrl %w0,imm */ 1326 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1327 _EMIT_CONST_U64(imm)); 1328 jit->seen |= SEEN_LITERAL; 1329 /* xgr %dst,%w0 */ 1330 EMIT4(0xb9820000, dst_reg, REG_W0); 1331 } 1332 break; 1333 /* 1334 * BPF_LSH 1335 */ 1336 case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */ 1337 /* sll %dst,0(%src) */ 1338 EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); 1339 EMIT_ZERO(dst_reg); 1340 break; 1341 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ 1342 /* sllg %dst,%dst,0(%src) */ 1343 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0); 1344 break; 1345 case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */ 1346 if (imm != 0) { 1347 /* sll %dst,imm(%r0) */ 1348 EMIT4_DISP(0x89000000, dst_reg, REG_0, imm); 1349 } 1350 EMIT_ZERO(dst_reg); 1351 break; 1352 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */ 1353 if (imm == 0) 1354 break; 1355 /* sllg %dst,%dst,imm(%r0) */ 1356 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm); 1357 break; 1358 /* 1359 * BPF_RSH 1360 */ 1361 case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */ 1362 /* srl %dst,0(%src) */ 1363 EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); 1364 EMIT_ZERO(dst_reg); 1365 break; 1366 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ 1367 /* srlg %dst,%dst,0(%src) */ 1368 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0); 1369 break; 1370 case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */ 1371 if (imm != 0) { 1372 /* srl %dst,imm(%r0) */ 1373 EMIT4_DISP(0x88000000, dst_reg, REG_0, imm); 1374 } 1375 EMIT_ZERO(dst_reg); 1376 break; 1377 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */ 1378 if (imm == 0) 1379 break; 1380 /* srlg %dst,%dst,imm(%r0) */ 1381 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm); 1382 break; 1383 /* 1384 * BPF_ARSH 1385 */ 1386 case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */ 1387 /* sra %dst,%dst,0(%src) */ 1388 EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); 1389 EMIT_ZERO(dst_reg); 1390 break; 1391 case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ 1392 /* srag %dst,%dst,0(%src) */ 1393 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); 1394 break; 1395 case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */ 1396 if (imm != 0) { 1397 /* sra %dst,imm(%r0) */ 1398 EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm); 1399 } 1400 EMIT_ZERO(dst_reg); 1401 break; 1402 case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */ 1403 if (imm == 0) 1404 break; 1405 /* srag %dst,%dst,imm(%r0) */ 1406 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm); 1407 break; 1408 /* 1409 * BPF_NEG 1410 */ 1411 case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */ 1412 /* lcr %dst,%dst */ 1413 EMIT2(0x1300, dst_reg, dst_reg); 1414 EMIT_ZERO(dst_reg); 1415 break; 1416 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 1417 /* lcgr %dst,%dst */ 1418 EMIT4(0xb9030000, dst_reg, dst_reg); 1419 break; 1420 /* 1421 * BPF_FROM_BE/LE 1422 */ 1423 case BPF_ALU | BPF_END | BPF_FROM_BE: 1424 /* s390 is big endian, therefore only clear high order bytes */ 1425 switch (imm) { 1426 case 16: /* dst = (u16) cpu_to_be16(dst) */ 1427 /* llghr %dst,%dst */ 1428 EMIT4(0xb9850000, dst_reg, dst_reg); 1429 if (insn_is_zext(&insn[1])) 1430 insn_count = 2; 1431 break; 1432 case 32: /* dst = (u32) cpu_to_be32(dst) */ 1433 if (!fp->aux->verifier_zext) 1434 /* llgfr %dst,%dst */ 1435 EMIT4(0xb9160000, dst_reg, dst_reg); 1436 break; 1437 case 64: /* dst = (u64) cpu_to_be64(dst) */ 1438 break; 1439 } 1440 break; 1441 case BPF_ALU | BPF_END | BPF_FROM_LE: 1442 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1443 switch (imm) { 1444 case 16: /* dst = (u16) cpu_to_le16(dst) */ 1445 /* lrvr %dst,%dst */ 1446 EMIT4(0xb91f0000, dst_reg, dst_reg); 1447 /* srl %dst,16(%r0) */ 1448 EMIT4_DISP(0x88000000, dst_reg, REG_0, 16); 1449 /* llghr %dst,%dst */ 1450 EMIT4(0xb9850000, dst_reg, dst_reg); 1451 if (insn_is_zext(&insn[1])) 1452 insn_count = 2; 1453 break; 1454 case 32: /* dst = (u32) cpu_to_le32(dst) */ 1455 /* lrvr %dst,%dst */ 1456 EMIT4(0xb91f0000, dst_reg, dst_reg); 1457 if (!fp->aux->verifier_zext) 1458 /* llgfr %dst,%dst */ 1459 EMIT4(0xb9160000, dst_reg, dst_reg); 1460 break; 1461 case 64: /* dst = (u64) cpu_to_le64(dst) */ 1462 /* lrvgr %dst,%dst */ 1463 EMIT4(0xb90f0000, dst_reg, dst_reg); 1464 break; 1465 } 1466 break; 1467 /* 1468 * BPF_NOSPEC (speculation barrier) 1469 */ 1470 case BPF_ST | BPF_NOSPEC: 1471 break; 1472 /* 1473 * BPF_ST(X) 1474 */ 1475 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ 1476 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1477 bpf_jit_probe_store_pre(jit, insn, &probe); 1478 /* stcy %src,off(%dst,%arena) */ 1479 EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, 1480 probe.arena_reg, off); 1481 err = bpf_jit_probe_post(jit, fp, &probe); 1482 if (err < 0) 1483 return err; 1484 jit->seen |= SEEN_MEM; 1485 break; 1486 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 1487 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1488 bpf_jit_probe_store_pre(jit, insn, &probe); 1489 /* sthy %src,off(%dst,%arena) */ 1490 EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, 1491 probe.arena_reg, off); 1492 err = bpf_jit_probe_post(jit, fp, &probe); 1493 if (err < 0) 1494 return err; 1495 jit->seen |= SEEN_MEM; 1496 break; 1497 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 1498 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1499 bpf_jit_probe_store_pre(jit, insn, &probe); 1500 /* sty %src,off(%dst,%arena) */ 1501 EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, 1502 probe.arena_reg, off); 1503 err = bpf_jit_probe_post(jit, fp, &probe); 1504 if (err < 0) 1505 return err; 1506 jit->seen |= SEEN_MEM; 1507 break; 1508 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 1509 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1510 bpf_jit_probe_store_pre(jit, insn, &probe); 1511 /* stg %src,off(%dst,%arena) */ 1512 EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, 1513 probe.arena_reg, off); 1514 err = bpf_jit_probe_post(jit, fp, &probe); 1515 if (err < 0) 1516 return err; 1517 jit->seen |= SEEN_MEM; 1518 break; 1519 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 1520 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1521 /* lhi %w0,imm */ 1522 EMIT4_IMM(0xa7080000, REG_W0, (u8) imm); 1523 bpf_jit_probe_store_pre(jit, insn, &probe); 1524 /* stcy %w0,off(%dst,%arena) */ 1525 EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, 1526 probe.arena_reg, off); 1527 err = bpf_jit_probe_post(jit, fp, &probe); 1528 if (err < 0) 1529 return err; 1530 jit->seen |= SEEN_MEM; 1531 break; 1532 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 1533 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1534 /* lhi %w0,imm */ 1535 EMIT4_IMM(0xa7080000, REG_W0, (u16) imm); 1536 bpf_jit_probe_store_pre(jit, insn, &probe); 1537 /* sthy %w0,off(%dst,%arena) */ 1538 EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, 1539 probe.arena_reg, off); 1540 err = bpf_jit_probe_post(jit, fp, &probe); 1541 if (err < 0) 1542 return err; 1543 jit->seen |= SEEN_MEM; 1544 break; 1545 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 1546 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1547 /* llilf %w0,imm */ 1548 EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm); 1549 bpf_jit_probe_store_pre(jit, insn, &probe); 1550 /* sty %w0,off(%dst,%arena) */ 1551 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, 1552 probe.arena_reg, off); 1553 err = bpf_jit_probe_post(jit, fp, &probe); 1554 if (err < 0) 1555 return err; 1556 jit->seen |= SEEN_MEM; 1557 break; 1558 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 1559 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1560 /* lgfi %w0,imm */ 1561 EMIT6_IMM(0xc0010000, REG_W0, imm); 1562 bpf_jit_probe_store_pre(jit, insn, &probe); 1563 /* stg %w0,off(%dst,%arena) */ 1564 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, 1565 probe.arena_reg, off); 1566 err = bpf_jit_probe_post(jit, fp, &probe); 1567 if (err < 0) 1568 return err; 1569 jit->seen |= SEEN_MEM; 1570 break; 1571 /* 1572 * BPF_ATOMIC 1573 */ 1574 case BPF_STX | BPF_ATOMIC | BPF_DW: 1575 case BPF_STX | BPF_ATOMIC | BPF_W: 1576 case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: 1577 case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: 1578 { 1579 bool is32 = BPF_SIZE(insn->code) == BPF_W; 1580 1581 /* 1582 * Unlike loads and stores, atomics have only a base register, 1583 * but no index register. For the non-arena case, simply use 1584 * %dst as a base. For the arena case, use the work register 1585 * %r1: first, load the arena base into it, and then add %dst 1586 * to it. 1587 */ 1588 probe.arena_reg = dst_reg; 1589 1590 switch (insn->imm) { 1591 #define EMIT_ATOMIC(op32, op64) do { \ 1592 bpf_jit_probe_atomic_pre(jit, insn, &probe); \ 1593 /* {op32|op64} {%w0|%src},%src,off(%arena) */ \ 1594 EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ 1595 (insn->imm & BPF_FETCH) ? src_reg : REG_W0, \ 1596 src_reg, probe.arena_reg, off); \ 1597 err = bpf_jit_probe_post(jit, fp, &probe); \ 1598 if (err < 0) \ 1599 return err; \ 1600 if (insn->imm & BPF_FETCH) { \ 1601 /* bcr 14,0 - see atomic_fetch_{add,and,or,xor}() */ \ 1602 _EMIT2(0x07e0); \ 1603 if (is32) \ 1604 EMIT_ZERO(src_reg); \ 1605 } \ 1606 } while (0) 1607 case BPF_ADD: 1608 case BPF_ADD | BPF_FETCH: 1609 /* {laal|laalg} */ 1610 EMIT_ATOMIC(0x00fa, 0x00ea); 1611 break; 1612 case BPF_AND: 1613 case BPF_AND | BPF_FETCH: 1614 /* {lan|lang} */ 1615 EMIT_ATOMIC(0x00f4, 0x00e4); 1616 break; 1617 case BPF_OR: 1618 case BPF_OR | BPF_FETCH: 1619 /* {lao|laog} */ 1620 EMIT_ATOMIC(0x00f6, 0x00e6); 1621 break; 1622 case BPF_XOR: 1623 case BPF_XOR | BPF_FETCH: 1624 /* {lax|laxg} */ 1625 EMIT_ATOMIC(0x00f7, 0x00e7); 1626 break; 1627 #undef EMIT_ATOMIC 1628 case BPF_XCHG: { 1629 struct bpf_jit_probe load_probe = probe; 1630 int loop_start; 1631 1632 bpf_jit_probe_atomic_pre(jit, insn, &load_probe); 1633 /* {ly|lg} %w0,off(%arena) */ 1634 EMIT6_DISP_LH(0xe3000000, 1635 is32 ? 0x0058 : 0x0004, REG_W0, REG_0, 1636 load_probe.arena_reg, off); 1637 bpf_jit_probe_emit_nop(jit, &load_probe); 1638 /* Reuse {ly|lg}'s arena_reg for {csy|csg}. */ 1639 if (load_probe.prg != -1) { 1640 probe.prg = jit->prg; 1641 probe.arena_reg = load_probe.arena_reg; 1642 } 1643 loop_start = jit->prg; 1644 /* 0: {csy|csg} %w0,%src,off(%arena) */ 1645 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1646 REG_W0, src_reg, probe.arena_reg, off); 1647 bpf_jit_probe_emit_nop(jit, &probe); 1648 /* brc 4,0b */ 1649 EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); 1650 /* {llgfr|lgr} %src,%w0 */ 1651 EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); 1652 /* Both probes should land here on exception. */ 1653 err = bpf_jit_probe_post(jit, fp, &load_probe); 1654 if (err < 0) 1655 return err; 1656 err = bpf_jit_probe_post(jit, fp, &probe); 1657 if (err < 0) 1658 return err; 1659 if (is32 && insn_is_zext(&insn[1])) 1660 insn_count = 2; 1661 break; 1662 } 1663 case BPF_CMPXCHG: 1664 bpf_jit_probe_atomic_pre(jit, insn, &probe); 1665 /* 0: {csy|csg} %b0,%src,off(%arena) */ 1666 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1667 BPF_REG_0, src_reg, 1668 probe.arena_reg, off); 1669 err = bpf_jit_probe_post(jit, fp, &probe); 1670 if (err < 0) 1671 return err; 1672 break; 1673 default: 1674 pr_err("Unknown atomic operation %02x\n", insn->imm); 1675 return -1; 1676 } 1677 1678 jit->seen |= SEEN_MEM; 1679 break; 1680 } 1681 /* 1682 * BPF_LDX 1683 */ 1684 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 1685 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1686 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1687 bpf_jit_probe_load_pre(jit, insn, &probe); 1688 /* llgc %dst,off(%src,%arena) */ 1689 EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, 1690 probe.arena_reg, off); 1691 err = bpf_jit_probe_post(jit, fp, &probe); 1692 if (err < 0) 1693 return err; 1694 jit->seen |= SEEN_MEM; 1695 if (insn_is_zext(&insn[1])) 1696 insn_count = 2; 1697 break; 1698 case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ 1699 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1700 bpf_jit_probe_load_pre(jit, insn, &probe); 1701 /* lgb %dst,off(%src) */ 1702 EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off); 1703 err = bpf_jit_probe_post(jit, fp, &probe); 1704 if (err < 0) 1705 return err; 1706 jit->seen |= SEEN_MEM; 1707 break; 1708 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 1709 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1710 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1711 bpf_jit_probe_load_pre(jit, insn, &probe); 1712 /* llgh %dst,off(%src,%arena) */ 1713 EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, 1714 probe.arena_reg, off); 1715 err = bpf_jit_probe_post(jit, fp, &probe); 1716 if (err < 0) 1717 return err; 1718 jit->seen |= SEEN_MEM; 1719 if (insn_is_zext(&insn[1])) 1720 insn_count = 2; 1721 break; 1722 case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ 1723 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1724 bpf_jit_probe_load_pre(jit, insn, &probe); 1725 /* lgh %dst,off(%src) */ 1726 EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off); 1727 err = bpf_jit_probe_post(jit, fp, &probe); 1728 if (err < 0) 1729 return err; 1730 jit->seen |= SEEN_MEM; 1731 break; 1732 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 1733 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1734 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1735 bpf_jit_probe_load_pre(jit, insn, &probe); 1736 /* llgf %dst,off(%src) */ 1737 jit->seen |= SEEN_MEM; 1738 EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, 1739 probe.arena_reg, off); 1740 err = bpf_jit_probe_post(jit, fp, &probe); 1741 if (err < 0) 1742 return err; 1743 if (insn_is_zext(&insn[1])) 1744 insn_count = 2; 1745 break; 1746 case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ 1747 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1748 bpf_jit_probe_load_pre(jit, insn, &probe); 1749 /* lgf %dst,off(%src) */ 1750 jit->seen |= SEEN_MEM; 1751 EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off); 1752 err = bpf_jit_probe_post(jit, fp, &probe); 1753 if (err < 0) 1754 return err; 1755 break; 1756 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 1757 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1758 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1759 bpf_jit_probe_load_pre(jit, insn, &probe); 1760 /* lg %dst,off(%src,%arena) */ 1761 jit->seen |= SEEN_MEM; 1762 EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, 1763 probe.arena_reg, off); 1764 err = bpf_jit_probe_post(jit, fp, &probe); 1765 if (err < 0) 1766 return err; 1767 break; 1768 /* 1769 * BPF_JMP / CALL 1770 */ 1771 case BPF_JMP | BPF_CALL: 1772 { 1773 const struct btf_func_model *m; 1774 bool func_addr_fixed; 1775 int j, ret; 1776 u64 func; 1777 1778 ret = bpf_jit_get_func_addr(fp, insn, extra_pass, 1779 &func, &func_addr_fixed); 1780 if (ret < 0) 1781 return -1; 1782 1783 REG_SET_SEEN(BPF_REG_5); 1784 jit->seen |= SEEN_FUNC; 1785 /* 1786 * Copy the tail call counter to where the callee expects it. 1787 * 1788 * Note 1: The callee can increment the tail call counter, but 1789 * we do not load it back, since the x86 JIT does not do this 1790 * either. 1791 * 1792 * Note 2: We assume that the verifier does not let us call the 1793 * main program, which clears the tail call counter on entry. 1794 */ 1795 /* mvc tail_call_cnt(4,%r15),frame_off+tail_call_cnt(%r15) */ 1796 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), 1797 0xf000 | (jit->frame_off + 1798 offsetof(struct prog_frame, tail_call_cnt))); 1799 1800 /* Sign-extend the kfunc arguments. */ 1801 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1802 m = bpf_jit_find_kfunc_model(fp, insn); 1803 if (!m) 1804 return -1; 1805 1806 for (j = 0; j < m->nr_args; j++) { 1807 if (sign_extend(jit, BPF_REG_1 + j, 1808 m->arg_size[j], 1809 m->arg_flags[j])) 1810 return -1; 1811 } 1812 } 1813 1814 /* lgrl %w1,func */ 1815 EMIT6_PCREL_RILB(0xc4080000, REG_W1, _EMIT_CONST_U64(func)); 1816 /* %r1() */ 1817 call_r1(jit); 1818 /* lgr %b0,%r2: load return value into %b0 */ 1819 EMIT4(0xb9040000, BPF_REG_0, REG_2); 1820 break; 1821 } 1822 case BPF_JMP | BPF_TAIL_CALL: { 1823 int patch_1_clrj, patch_2_clij, patch_3_brc; 1824 1825 /* 1826 * Implicit input: 1827 * B1: pointer to ctx 1828 * B2: pointer to bpf_array 1829 * B3: index in bpf_array 1830 * 1831 * if (index >= array->map.max_entries) 1832 * goto out; 1833 */ 1834 1835 /* llgf %w1,map.max_entries(%b2) */ 1836 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2, 1837 offsetof(struct bpf_array, map.max_entries)); 1838 /* if ((u32)%b3 >= (u32)%w1) goto out; */ 1839 /* clrj %b3,%w1,0xa,out */ 1840 patch_1_clrj = jit->prg; 1841 EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa, 1842 jit->prg); 1843 1844 /* 1845 * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) 1846 * goto out; 1847 */ 1848 1849 off = jit->frame_off + 1850 offsetof(struct prog_frame, tail_call_cnt); 1851 /* lhi %w0,1 */ 1852 EMIT4_IMM(0xa7080000, REG_W0, 1); 1853 /* laal %w1,%w0,off(%r15) */ 1854 EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off); 1855 /* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */ 1856 patch_2_clij = jit->prg; 1857 EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1, 1858 2, jit->prg); 1859 1860 /* 1861 * prog = array->ptrs[index]; 1862 * if (prog == NULL) 1863 * goto out; 1864 */ 1865 1866 /* llgfr %r1,%b3: %r1 = (u32) index */ 1867 EMIT4(0xb9160000, REG_1, BPF_REG_3); 1868 /* sllg %r1,%r1,3: %r1 *= 8 */ 1869 EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3); 1870 /* ltg %r1,prog(%b2,%r1) */ 1871 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2, 1872 REG_1, offsetof(struct bpf_array, ptrs)); 1873 /* brc 0x8,out */ 1874 patch_3_brc = jit->prg; 1875 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); 1876 1877 /* 1878 * Restore registers before calling function 1879 */ 1880 save_restore_regs(jit, REGS_RESTORE, 0); 1881 1882 /* 1883 * goto *(prog->bpf_func + tail_call_start); 1884 */ 1885 1886 /* lg %r1,bpf_func(%r1) */ 1887 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0, 1888 offsetof(struct bpf_prog, bpf_func)); 1889 if (nospec_uses_trampoline()) { 1890 jit->seen |= SEEN_FUNC; 1891 /* aghi %r1,tail_call_start */ 1892 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start); 1893 /* brcl 0xf,__s390_indirect_jump_r1 */ 1894 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 1895 __s390_indirect_jump_r1); 1896 } else { 1897 /* bc 0xf,tail_call_start(%r1) */ 1898 _EMIT4(0x47f01000 + jit->tail_call_start); 1899 } 1900 /* out: */ 1901 if (jit->prg_buf) { 1902 *(u16 *)(jit->prg_buf + patch_1_clrj + 2) = 1903 (jit->prg - patch_1_clrj) >> 1; 1904 *(u16 *)(jit->prg_buf + patch_2_clij + 2) = 1905 (jit->prg - patch_2_clij) >> 1; 1906 *(u16 *)(jit->prg_buf + patch_3_brc + 2) = 1907 (jit->prg - patch_3_brc) >> 1; 1908 } 1909 break; 1910 } 1911 case BPF_JMP | BPF_EXIT: /* return b0 */ 1912 last = (i == fp->len - 1) ? 1 : 0; 1913 if (last) 1914 break; 1915 if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip)) 1916 /* brc 0xf, <exit> */ 1917 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip); 1918 else 1919 /* brcl 0xf, <exit> */ 1920 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip); 1921 break; 1922 /* 1923 * Branch relative (number of skipped instructions) to offset on 1924 * condition. 1925 * 1926 * Condition code to mask mapping: 1927 * 1928 * CC | Description | Mask 1929 * ------------------------------ 1930 * 0 | Operands equal | 8 1931 * 1 | First operand low | 4 1932 * 2 | First operand high | 2 1933 * 3 | Unused | 1 1934 * 1935 * For s390x relative branches: ip = ip + off_bytes 1936 * For BPF relative branches: insn = insn + off_insns + 1 1937 * 1938 * For example for s390x with offset 0 we jump to the branch 1939 * instruction itself (loop) and for BPF with offset 0 we 1940 * branch to the instruction behind the branch. 1941 */ 1942 case BPF_JMP32 | BPF_JA: /* if (true) */ 1943 branch_oc_off = imm; 1944 fallthrough; 1945 case BPF_JMP | BPF_JA: /* if (true) */ 1946 mask = 0xf000; /* j */ 1947 goto branch_oc; 1948 case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */ 1949 case BPF_JMP32 | BPF_JSGT | BPF_K: /* ((s32) dst > (s32) imm) */ 1950 mask = 0x2000; /* jh */ 1951 goto branch_ks; 1952 case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */ 1953 case BPF_JMP32 | BPF_JSLT | BPF_K: /* ((s32) dst < (s32) imm) */ 1954 mask = 0x4000; /* jl */ 1955 goto branch_ks; 1956 case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */ 1957 case BPF_JMP32 | BPF_JSGE | BPF_K: /* ((s32) dst >= (s32) imm) */ 1958 mask = 0xa000; /* jhe */ 1959 goto branch_ks; 1960 case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */ 1961 case BPF_JMP32 | BPF_JSLE | BPF_K: /* ((s32) dst <= (s32) imm) */ 1962 mask = 0xc000; /* jle */ 1963 goto branch_ks; 1964 case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */ 1965 case BPF_JMP32 | BPF_JGT | BPF_K: /* ((u32) dst_reg > (u32) imm) */ 1966 mask = 0x2000; /* jh */ 1967 goto branch_ku; 1968 case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */ 1969 case BPF_JMP32 | BPF_JLT | BPF_K: /* ((u32) dst_reg < (u32) imm) */ 1970 mask = 0x4000; /* jl */ 1971 goto branch_ku; 1972 case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */ 1973 case BPF_JMP32 | BPF_JGE | BPF_K: /* ((u32) dst_reg >= (u32) imm) */ 1974 mask = 0xa000; /* jhe */ 1975 goto branch_ku; 1976 case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */ 1977 case BPF_JMP32 | BPF_JLE | BPF_K: /* ((u32) dst_reg <= (u32) imm) */ 1978 mask = 0xc000; /* jle */ 1979 goto branch_ku; 1980 case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */ 1981 case BPF_JMP32 | BPF_JNE | BPF_K: /* ((u32) dst_reg != (u32) imm) */ 1982 mask = 0x7000; /* jne */ 1983 goto branch_ku; 1984 case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */ 1985 case BPF_JMP32 | BPF_JEQ | BPF_K: /* ((u32) dst_reg == (u32) imm) */ 1986 mask = 0x8000; /* je */ 1987 goto branch_ku; 1988 case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */ 1989 case BPF_JMP32 | BPF_JSET | BPF_K: /* ((u32) dst_reg & (u32) imm) */ 1990 mask = 0x7000; /* jnz */ 1991 if (BPF_CLASS(insn->code) == BPF_JMP32) { 1992 /* llilf %w1,imm (load zero extend imm) */ 1993 EMIT6_IMM(0xc00f0000, REG_W1, imm); 1994 /* nr %w1,%dst */ 1995 EMIT2(0x1400, REG_W1, dst_reg); 1996 } else { 1997 /* lgfi %w1,imm (load sign extend imm) */ 1998 EMIT6_IMM(0xc0010000, REG_W1, imm); 1999 /* ngr %w1,%dst */ 2000 EMIT4(0xb9800000, REG_W1, dst_reg); 2001 } 2002 goto branch_oc; 2003 2004 case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */ 2005 case BPF_JMP32 | BPF_JSGT | BPF_X: /* ((s32) dst > (s32) src) */ 2006 mask = 0x2000; /* jh */ 2007 goto branch_xs; 2008 case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */ 2009 case BPF_JMP32 | BPF_JSLT | BPF_X: /* ((s32) dst < (s32) src) */ 2010 mask = 0x4000; /* jl */ 2011 goto branch_xs; 2012 case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */ 2013 case BPF_JMP32 | BPF_JSGE | BPF_X: /* ((s32) dst >= (s32) src) */ 2014 mask = 0xa000; /* jhe */ 2015 goto branch_xs; 2016 case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */ 2017 case BPF_JMP32 | BPF_JSLE | BPF_X: /* ((s32) dst <= (s32) src) */ 2018 mask = 0xc000; /* jle */ 2019 goto branch_xs; 2020 case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */ 2021 case BPF_JMP32 | BPF_JGT | BPF_X: /* ((u32) dst > (u32) src) */ 2022 mask = 0x2000; /* jh */ 2023 goto branch_xu; 2024 case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */ 2025 case BPF_JMP32 | BPF_JLT | BPF_X: /* ((u32) dst < (u32) src) */ 2026 mask = 0x4000; /* jl */ 2027 goto branch_xu; 2028 case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */ 2029 case BPF_JMP32 | BPF_JGE | BPF_X: /* ((u32) dst >= (u32) src) */ 2030 mask = 0xa000; /* jhe */ 2031 goto branch_xu; 2032 case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */ 2033 case BPF_JMP32 | BPF_JLE | BPF_X: /* ((u32) dst <= (u32) src) */ 2034 mask = 0xc000; /* jle */ 2035 goto branch_xu; 2036 case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */ 2037 case BPF_JMP32 | BPF_JNE | BPF_X: /* ((u32) dst != (u32) src) */ 2038 mask = 0x7000; /* jne */ 2039 goto branch_xu; 2040 case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */ 2041 case BPF_JMP32 | BPF_JEQ | BPF_X: /* ((u32) dst == (u32) src) */ 2042 mask = 0x8000; /* je */ 2043 goto branch_xu; 2044 case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */ 2045 case BPF_JMP32 | BPF_JSET | BPF_X: /* ((u32) dst & (u32) src) */ 2046 { 2047 bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2048 2049 mask = 0x7000; /* jnz */ 2050 /* nrk or ngrk %w1,%dst,%src */ 2051 EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000), 2052 REG_W1, dst_reg, src_reg); 2053 goto branch_oc; 2054 branch_ks: 2055 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2056 /* cfi or cgfi %dst,imm */ 2057 EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000, 2058 dst_reg, imm); 2059 if (!is_first_pass(jit) && 2060 can_use_rel(jit, addrs[i + off + 1])) { 2061 /* brc mask,off */ 2062 EMIT4_PCREL_RIC(0xa7040000, 2063 mask >> 12, addrs[i + off + 1]); 2064 } else { 2065 /* brcl mask,off */ 2066 EMIT6_PCREL_RILC(0xc0040000, 2067 mask >> 12, addrs[i + off + 1]); 2068 } 2069 break; 2070 branch_ku: 2071 /* lgfi %w1,imm (load sign extend imm) */ 2072 src_reg = REG_1; 2073 EMIT6_IMM(0xc0010000, src_reg, imm); 2074 goto branch_xu; 2075 branch_xs: 2076 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2077 if (!is_first_pass(jit) && 2078 can_use_rel(jit, addrs[i + off + 1])) { 2079 /* crj or cgrj %dst,%src,mask,off */ 2080 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064), 2081 dst_reg, src_reg, i, off, mask); 2082 } else { 2083 /* cr or cgr %dst,%src */ 2084 if (is_jmp32) 2085 EMIT2(0x1900, dst_reg, src_reg); 2086 else 2087 EMIT4(0xb9200000, dst_reg, src_reg); 2088 /* brcl mask,off */ 2089 EMIT6_PCREL_RILC(0xc0040000, 2090 mask >> 12, addrs[i + off + 1]); 2091 } 2092 break; 2093 branch_xu: 2094 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2095 if (!is_first_pass(jit) && 2096 can_use_rel(jit, addrs[i + off + 1])) { 2097 /* clrj or clgrj %dst,%src,mask,off */ 2098 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065), 2099 dst_reg, src_reg, i, off, mask); 2100 } else { 2101 /* clr or clgr %dst,%src */ 2102 if (is_jmp32) 2103 EMIT2(0x1500, dst_reg, src_reg); 2104 else 2105 EMIT4(0xb9210000, dst_reg, src_reg); 2106 /* brcl mask,off */ 2107 EMIT6_PCREL_RILC(0xc0040000, 2108 mask >> 12, addrs[i + off + 1]); 2109 } 2110 break; 2111 branch_oc: 2112 if (!is_first_pass(jit) && 2113 can_use_rel(jit, addrs[i + branch_oc_off + 1])) { 2114 /* brc mask,off */ 2115 EMIT4_PCREL_RIC(0xa7040000, 2116 mask >> 12, 2117 addrs[i + branch_oc_off + 1]); 2118 } else { 2119 /* brcl mask,off */ 2120 EMIT6_PCREL_RILC(0xc0040000, 2121 mask >> 12, 2122 addrs[i + branch_oc_off + 1]); 2123 } 2124 break; 2125 } 2126 default: /* too complex, give up */ 2127 pr_err("Unknown opcode %02x\n", insn->code); 2128 return -1; 2129 } 2130 2131 return insn_count; 2132 } 2133 2134 /* 2135 * Return whether new i-th instruction address does not violate any invariant 2136 */ 2137 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i) 2138 { 2139 /* On the first pass anything goes */ 2140 if (is_first_pass(jit)) 2141 return true; 2142 2143 /* The codegen pass must not change anything */ 2144 if (is_codegen_pass(jit)) 2145 return jit->addrs[i] == jit->prg; 2146 2147 /* Passes in between must not increase code size */ 2148 return jit->addrs[i] >= jit->prg; 2149 } 2150 2151 /* 2152 * Update the address of i-th instruction 2153 */ 2154 static int bpf_set_addr(struct bpf_jit *jit, int i) 2155 { 2156 int delta; 2157 2158 if (is_codegen_pass(jit)) { 2159 delta = jit->prg - jit->addrs[i]; 2160 if (delta < 0) 2161 bpf_skip(jit, -delta); 2162 } 2163 if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i))) 2164 return -1; 2165 jit->addrs[i] = jit->prg; 2166 return 0; 2167 } 2168 2169 /* 2170 * Compile eBPF program into s390x code 2171 */ 2172 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, 2173 bool extra_pass) 2174 { 2175 int i, insn_count, lit32_size, lit64_size; 2176 u64 kern_arena; 2177 2178 jit->lit32 = jit->lit32_start; 2179 jit->lit64 = jit->lit64_start; 2180 jit->prg = 0; 2181 jit->excnt = 0; 2182 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) 2183 jit->frame_off = sizeof(struct prog_frame) - 2184 offsetofend(struct prog_frame, unused) + 2185 round_up(fp->aux->stack_depth, 8); 2186 else 2187 jit->frame_off = 0; 2188 2189 kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); 2190 if (kern_arena) 2191 jit->kern_arena = _EMIT_CONST_U64(kern_arena); 2192 jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); 2193 2194 bpf_jit_prologue(jit, fp); 2195 if (bpf_set_addr(jit, 0) < 0) 2196 return -1; 2197 for (i = 0; i < fp->len; i += insn_count) { 2198 insn_count = bpf_jit_insn(jit, fp, i, extra_pass); 2199 if (insn_count < 0) 2200 return -1; 2201 /* Next instruction address */ 2202 if (bpf_set_addr(jit, i + insn_count) < 0) 2203 return -1; 2204 } 2205 bpf_jit_epilogue(jit); 2206 2207 lit32_size = jit->lit32 - jit->lit32_start; 2208 lit64_size = jit->lit64 - jit->lit64_start; 2209 jit->lit32_start = jit->prg; 2210 if (lit32_size) 2211 jit->lit32_start = ALIGN(jit->lit32_start, 4); 2212 jit->lit64_start = jit->lit32_start + lit32_size; 2213 if (lit64_size) 2214 jit->lit64_start = ALIGN(jit->lit64_start, 8); 2215 jit->size = jit->lit64_start + lit64_size; 2216 jit->size_prg = jit->prg; 2217 2218 if (WARN_ON_ONCE(fp->aux->extable && 2219 jit->excnt != fp->aux->num_exentries)) 2220 /* Verifier bug - too many entries. */ 2221 return -1; 2222 2223 return 0; 2224 } 2225 2226 bool bpf_jit_needs_zext(void) 2227 { 2228 return true; 2229 } 2230 2231 struct s390_jit_data { 2232 struct bpf_binary_header *header; 2233 struct bpf_jit ctx; 2234 int pass; 2235 }; 2236 2237 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, 2238 struct bpf_prog *fp) 2239 { 2240 struct bpf_binary_header *header; 2241 struct bpf_insn *insn; 2242 u32 extable_size; 2243 u32 code_size; 2244 int i; 2245 2246 for (i = 0; i < fp->len; i++) { 2247 insn = &fp->insnsi[i]; 2248 2249 if (BPF_CLASS(insn->code) == BPF_STX && 2250 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC && 2251 (BPF_SIZE(insn->code) == BPF_DW || 2252 BPF_SIZE(insn->code) == BPF_W) && 2253 insn->imm == BPF_XCHG) 2254 /* 2255 * bpf_jit_insn() emits a load and a compare-and-swap, 2256 * both of which need to be probed. 2257 */ 2258 fp->aux->num_exentries += 1; 2259 } 2260 /* We need two entries per insn. */ 2261 fp->aux->num_exentries *= 2; 2262 2263 code_size = roundup(jit->size, 2264 __alignof__(struct exception_table_entry)); 2265 extable_size = fp->aux->num_exentries * 2266 sizeof(struct exception_table_entry); 2267 header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf, 2268 8, jit_fill_hole); 2269 if (!header) 2270 return NULL; 2271 fp->aux->extable = (struct exception_table_entry *) 2272 (jit->prg_buf + code_size); 2273 return header; 2274 } 2275 2276 /* 2277 * Compile eBPF program "fp" 2278 */ 2279 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 2280 { 2281 struct bpf_prog *tmp, *orig_fp = fp; 2282 struct bpf_binary_header *header; 2283 struct s390_jit_data *jit_data; 2284 bool tmp_blinded = false; 2285 bool extra_pass = false; 2286 struct bpf_jit jit; 2287 int pass; 2288 2289 if (!fp->jit_requested) 2290 return orig_fp; 2291 2292 tmp = bpf_jit_blind_constants(fp); 2293 /* 2294 * If blinding was requested and we failed during blinding, 2295 * we must fall back to the interpreter. 2296 */ 2297 if (IS_ERR(tmp)) 2298 return orig_fp; 2299 if (tmp != fp) { 2300 tmp_blinded = true; 2301 fp = tmp; 2302 } 2303 2304 jit_data = fp->aux->jit_data; 2305 if (!jit_data) { 2306 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 2307 if (!jit_data) { 2308 fp = orig_fp; 2309 goto out; 2310 } 2311 fp->aux->jit_data = jit_data; 2312 } 2313 if (jit_data->ctx.addrs) { 2314 jit = jit_data->ctx; 2315 header = jit_data->header; 2316 extra_pass = true; 2317 pass = jit_data->pass + 1; 2318 goto skip_init_ctx; 2319 } 2320 2321 memset(&jit, 0, sizeof(jit)); 2322 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); 2323 if (jit.addrs == NULL) { 2324 fp = orig_fp; 2325 goto free_addrs; 2326 } 2327 /* 2328 * Three initial passes: 2329 * - 1/2: Determine clobbered registers 2330 * - 3: Calculate program size and addrs array 2331 */ 2332 for (pass = 1; pass <= 3; pass++) { 2333 if (bpf_jit_prog(&jit, fp, extra_pass)) { 2334 fp = orig_fp; 2335 goto free_addrs; 2336 } 2337 } 2338 /* 2339 * Final pass: Allocate and generate program 2340 */ 2341 header = bpf_jit_alloc(&jit, fp); 2342 if (!header) { 2343 fp = orig_fp; 2344 goto free_addrs; 2345 } 2346 skip_init_ctx: 2347 if (bpf_jit_prog(&jit, fp, extra_pass)) { 2348 bpf_jit_binary_free(header); 2349 fp = orig_fp; 2350 goto free_addrs; 2351 } 2352 if (bpf_jit_enable > 1) { 2353 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); 2354 print_fn_code(jit.prg_buf, jit.size_prg); 2355 } 2356 if (!fp->is_func || extra_pass) { 2357 if (bpf_jit_binary_lock_ro(header)) { 2358 bpf_jit_binary_free(header); 2359 fp = orig_fp; 2360 goto free_addrs; 2361 } 2362 } else { 2363 jit_data->header = header; 2364 jit_data->ctx = jit; 2365 jit_data->pass = pass; 2366 } 2367 fp->bpf_func = (void *) jit.prg_buf; 2368 fp->jited = 1; 2369 fp->jited_len = jit.size; 2370 2371 if (!fp->is_func || extra_pass) { 2372 bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); 2373 free_addrs: 2374 kvfree(jit.addrs); 2375 kfree(jit_data); 2376 fp->aux->jit_data = NULL; 2377 } 2378 out: 2379 if (tmp_blinded) 2380 bpf_jit_prog_release_other(fp, fp == orig_fp ? 2381 tmp : orig_fp); 2382 return fp; 2383 } 2384 2385 bool bpf_jit_supports_kfunc_call(void) 2386 { 2387 return true; 2388 } 2389 2390 bool bpf_jit_supports_far_kfunc_call(void) 2391 { 2392 return true; 2393 } 2394 2395 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 2396 void *old_addr, void *new_addr) 2397 { 2398 struct bpf_plt expected_plt, current_plt, new_plt, *plt; 2399 struct { 2400 u16 opc; 2401 s32 disp; 2402 } __packed insn; 2403 char *ret; 2404 int err; 2405 2406 /* Verify the branch to be patched. */ 2407 err = copy_from_kernel_nofault(&insn, ip, sizeof(insn)); 2408 if (err < 0) 2409 return err; 2410 if (insn.opc != (0xc004 | (old_addr ? 0xf0 : 0))) 2411 return -EINVAL; 2412 2413 if (t == BPF_MOD_JUMP && 2414 insn.disp == ((char *)new_addr - (char *)ip) >> 1) { 2415 /* 2416 * The branch already points to the destination, 2417 * there is no PLT. 2418 */ 2419 } else { 2420 /* Verify the PLT. */ 2421 plt = ip + (insn.disp << 1); 2422 err = copy_from_kernel_nofault(¤t_plt, plt, 2423 sizeof(current_plt)); 2424 if (err < 0) 2425 return err; 2426 ret = (char *)ip + 6; 2427 bpf_jit_plt(&expected_plt, ret, old_addr); 2428 if (memcmp(¤t_plt, &expected_plt, sizeof(current_plt))) 2429 return -EINVAL; 2430 /* Adjust the call address. */ 2431 bpf_jit_plt(&new_plt, ret, new_addr); 2432 s390_kernel_write(&plt->target, &new_plt.target, 2433 sizeof(void *)); 2434 } 2435 2436 /* Adjust the mask of the branch. */ 2437 insn.opc = 0xc004 | (new_addr ? 0xf0 : 0); 2438 s390_kernel_write((char *)ip + 1, (char *)&insn.opc + 1, 1); 2439 2440 /* Make the new code visible to the other CPUs. */ 2441 text_poke_sync_lock(); 2442 2443 return 0; 2444 } 2445 2446 struct bpf_tramp_jit { 2447 struct bpf_jit common; 2448 int orig_stack_args_off;/* Offset of arguments placed on stack by the 2449 * func_addr's original caller 2450 */ 2451 int stack_size; /* Trampoline stack size */ 2452 int backchain_off; /* Offset of backchain */ 2453 int stack_args_off; /* Offset of stack arguments for calling 2454 * func_addr, has to be at the top 2455 */ 2456 int reg_args_off; /* Offset of register arguments for calling 2457 * func_addr 2458 */ 2459 int ip_off; /* For bpf_get_func_ip(), has to be at 2460 * (ctx - 16) 2461 */ 2462 int arg_cnt_off; /* For bpf_get_func_arg_cnt(), has to be at 2463 * (ctx - 8) 2464 */ 2465 int bpf_args_off; /* Offset of BPF_PROG context, which consists 2466 * of BPF arguments followed by return value 2467 */ 2468 int retval_off; /* Offset of return value (see above) */ 2469 int r7_r8_off; /* Offset of saved %r7 and %r8, which are used 2470 * for __bpf_prog_enter() return value and 2471 * func_addr respectively 2472 */ 2473 int run_ctx_off; /* Offset of struct bpf_tramp_run_ctx */ 2474 int tccnt_off; /* Offset of saved tailcall counter */ 2475 int r14_off; /* Offset of saved %r14, has to be at the 2476 * bottom */ 2477 int do_fexit; /* do_fexit: label */ 2478 }; 2479 2480 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) 2481 { 2482 /* llihf %dst_reg,val_hi */ 2483 EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32)); 2484 /* oilf %rdst_reg,val_lo */ 2485 EMIT6_IMM(0xc00d0000, dst_reg, val); 2486 } 2487 2488 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, 2489 const struct btf_func_model *m, 2490 struct bpf_tramp_link *tlink, bool save_ret) 2491 { 2492 struct bpf_jit *jit = &tjit->common; 2493 int cookie_off = tjit->run_ctx_off + 2494 offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 2495 struct bpf_prog *p = tlink->link.prog; 2496 int patch; 2497 2498 /* 2499 * run_ctx.cookie = tlink->cookie; 2500 */ 2501 2502 /* %r0 = tlink->cookie */ 2503 load_imm64(jit, REG_W0, tlink->cookie); 2504 /* stg %r0,cookie_off(%r15) */ 2505 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off); 2506 2507 /* 2508 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) 2509 * goto skip; 2510 */ 2511 2512 /* %r1 = __bpf_prog_enter */ 2513 load_imm64(jit, REG_1, (u64)bpf_trampoline_enter(p)); 2514 /* %r2 = p */ 2515 load_imm64(jit, REG_2, (u64)p); 2516 /* la %r3,run_ctx_off(%r15) */ 2517 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); 2518 /* %r1() */ 2519 call_r1(jit); 2520 /* ltgr %r7,%r2 */ 2521 EMIT4(0xb9020000, REG_7, REG_2); 2522 /* brcl 8,skip */ 2523 patch = jit->prg; 2524 EMIT6_PCREL_RILC(0xc0040000, 8, 0); 2525 2526 /* 2527 * retval = bpf_func(args, p->insnsi); 2528 */ 2529 2530 /* %r1 = p->bpf_func */ 2531 load_imm64(jit, REG_1, (u64)p->bpf_func); 2532 /* la %r2,bpf_args_off(%r15) */ 2533 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); 2534 /* %r3 = p->insnsi */ 2535 if (!p->jited) 2536 load_imm64(jit, REG_3, (u64)p->insnsi); 2537 /* %r1() */ 2538 call_r1(jit); 2539 /* stg %r2,retval_off(%r15) */ 2540 if (save_ret) { 2541 if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags)) 2542 return -1; 2543 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2544 tjit->retval_off); 2545 } 2546 2547 /* skip: */ 2548 if (jit->prg_buf) 2549 *(u32 *)&jit->prg_buf[patch + 2] = (jit->prg - patch) >> 1; 2550 2551 /* 2552 * __bpf_prog_exit(p, start, &run_ctx); 2553 */ 2554 2555 /* %r1 = __bpf_prog_exit */ 2556 load_imm64(jit, REG_1, (u64)bpf_trampoline_exit(p)); 2557 /* %r2 = p */ 2558 load_imm64(jit, REG_2, (u64)p); 2559 /* lgr %r3,%r7 */ 2560 EMIT4(0xb9040000, REG_3, REG_7); 2561 /* la %r4,run_ctx_off(%r15) */ 2562 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); 2563 /* %r1() */ 2564 call_r1(jit); 2565 2566 return 0; 2567 } 2568 2569 static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size) 2570 { 2571 int stack_offset = tjit->stack_size; 2572 2573 tjit->stack_size += size; 2574 return stack_offset; 2575 } 2576 2577 /* ABI uses %r2 - %r6 for parameter passing. */ 2578 #define MAX_NR_REG_ARGS 5 2579 2580 /* The "L" field of the "mvc" instruction is 8 bits. */ 2581 #define MAX_MVC_SIZE 256 2582 #define MAX_NR_STACK_ARGS (MAX_MVC_SIZE / sizeof(u64)) 2583 2584 /* -mfentry generates a 6-byte nop on s390x. */ 2585 #define S390X_PATCH_SIZE 6 2586 2587 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 2588 struct bpf_tramp_jit *tjit, 2589 const struct btf_func_model *m, 2590 u32 flags, 2591 struct bpf_tramp_links *tlinks, 2592 void *func_addr) 2593 { 2594 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 2595 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 2596 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 2597 int nr_bpf_args, nr_reg_args, nr_stack_args; 2598 struct bpf_jit *jit = &tjit->common; 2599 int arg, bpf_arg_off; 2600 int i, j; 2601 2602 /* Support as many stack arguments as "mvc" instruction can handle. */ 2603 nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS); 2604 nr_stack_args = m->nr_args - nr_reg_args; 2605 if (nr_stack_args > MAX_NR_STACK_ARGS) 2606 return -ENOTSUPP; 2607 2608 /* Return to %r14 in the struct_ops case. */ 2609 if (flags & BPF_TRAMP_F_INDIRECT) 2610 flags |= BPF_TRAMP_F_SKIP_FRAME; 2611 2612 /* 2613 * Compute how many arguments we need to pass to BPF programs. 2614 * BPF ABI mirrors that of x86_64: arguments that are 16 bytes or 2615 * smaller are packed into 1 or 2 registers; larger arguments are 2616 * passed via pointers. 2617 * In s390x ABI, arguments that are 8 bytes or smaller are packed into 2618 * a register; larger arguments are passed via pointers. 2619 * We need to deal with this difference. 2620 */ 2621 nr_bpf_args = 0; 2622 for (i = 0; i < m->nr_args; i++) { 2623 if (m->arg_size[i] <= 8) 2624 nr_bpf_args += 1; 2625 else if (m->arg_size[i] <= 16) 2626 nr_bpf_args += 2; 2627 else 2628 return -ENOTSUPP; 2629 } 2630 2631 /* 2632 * Calculate the stack layout. 2633 */ 2634 2635 /* 2636 * Allocate STACK_FRAME_OVERHEAD bytes for the callees. As the s390x 2637 * ABI requires, put our backchain at the end of the allocated memory. 2638 */ 2639 tjit->stack_size = STACK_FRAME_OVERHEAD; 2640 tjit->backchain_off = tjit->stack_size - sizeof(u64); 2641 tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64)); 2642 tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64)); 2643 tjit->ip_off = alloc_stack(tjit, sizeof(u64)); 2644 tjit->arg_cnt_off = alloc_stack(tjit, sizeof(u64)); 2645 tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64)); 2646 tjit->retval_off = alloc_stack(tjit, sizeof(u64)); 2647 tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64)); 2648 tjit->run_ctx_off = alloc_stack(tjit, 2649 sizeof(struct bpf_tramp_run_ctx)); 2650 tjit->tccnt_off = alloc_stack(tjit, sizeof(u64)); 2651 tjit->r14_off = alloc_stack(tjit, sizeof(u64) * 2); 2652 /* 2653 * In accordance with the s390x ABI, the caller has allocated 2654 * STACK_FRAME_OVERHEAD bytes for us. 8 of them contain the caller's 2655 * backchain, and the rest we can use. 2656 */ 2657 tjit->stack_size -= STACK_FRAME_OVERHEAD - sizeof(u64); 2658 tjit->orig_stack_args_off = tjit->stack_size + STACK_FRAME_OVERHEAD; 2659 2660 /* lgr %r1,%r15 */ 2661 EMIT4(0xb9040000, REG_1, REG_15); 2662 /* aghi %r15,-stack_size */ 2663 EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size); 2664 /* stg %r1,backchain_off(%r15) */ 2665 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15, 2666 tjit->backchain_off); 2667 /* mvc tccnt_off(4,%r15),stack_size+tail_call_cnt(%r15) */ 2668 _EMIT6(0xd203f000 | tjit->tccnt_off, 2669 0xf000 | (tjit->stack_size + 2670 offsetof(struct prog_frame, tail_call_cnt))); 2671 /* stmg %r2,%rN,fwd_reg_args_off(%r15) */ 2672 if (nr_reg_args) 2673 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2, 2674 REG_2 + (nr_reg_args - 1), REG_15, 2675 tjit->reg_args_off); 2676 for (i = 0, j = 0; i < m->nr_args; i++) { 2677 if (i < MAX_NR_REG_ARGS) 2678 arg = REG_2 + i; 2679 else 2680 arg = tjit->orig_stack_args_off + 2681 (i - MAX_NR_REG_ARGS) * sizeof(u64); 2682 bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64); 2683 if (m->arg_size[i] <= 8) { 2684 if (i < MAX_NR_REG_ARGS) 2685 /* stg %arg,bpf_arg_off(%r15) */ 2686 EMIT6_DISP_LH(0xe3000000, 0x0024, arg, 2687 REG_0, REG_15, bpf_arg_off); 2688 else 2689 /* mvc bpf_arg_off(8,%r15),arg(%r15) */ 2690 _EMIT6(0xd207f000 | bpf_arg_off, 2691 0xf000 | arg); 2692 j += 1; 2693 } else { 2694 if (i < MAX_NR_REG_ARGS) { 2695 /* mvc bpf_arg_off(16,%r15),0(%arg) */ 2696 _EMIT6(0xd20ff000 | bpf_arg_off, 2697 reg2hex[arg] << 12); 2698 } else { 2699 /* lg %r1,arg(%r15) */ 2700 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_0, 2701 REG_15, arg); 2702 /* mvc bpf_arg_off(16,%r15),0(%r1) */ 2703 _EMIT6(0xd20ff000 | bpf_arg_off, 0x1000); 2704 } 2705 j += 2; 2706 } 2707 } 2708 /* stmg %r7,%r8,r7_r8_off(%r15) */ 2709 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_7, REG_8, REG_15, 2710 tjit->r7_r8_off); 2711 /* stg %r14,r14_off(%r15) */ 2712 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off); 2713 2714 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2715 /* 2716 * The ftrace trampoline puts the return address (which is the 2717 * address of the original function + S390X_PATCH_SIZE) into 2718 * %r0; see ftrace_shared_hotpatch_trampoline_br and 2719 * ftrace_init_nop() for details. 2720 */ 2721 2722 /* lgr %r8,%r0 */ 2723 EMIT4(0xb9040000, REG_8, REG_0); 2724 } else { 2725 /* %r8 = func_addr + S390X_PATCH_SIZE */ 2726 load_imm64(jit, REG_8, (u64)func_addr + S390X_PATCH_SIZE); 2727 } 2728 2729 /* 2730 * ip = func_addr; 2731 * arg_cnt = m->nr_args; 2732 */ 2733 2734 if (flags & BPF_TRAMP_F_IP_ARG) { 2735 /* %r0 = func_addr */ 2736 load_imm64(jit, REG_0, (u64)func_addr); 2737 /* stg %r0,ip_off(%r15) */ 2738 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, 2739 tjit->ip_off); 2740 } 2741 /* lghi %r0,nr_bpf_args */ 2742 EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args); 2743 /* stg %r0,arg_cnt_off(%r15) */ 2744 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, 2745 tjit->arg_cnt_off); 2746 2747 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2748 /* 2749 * __bpf_tramp_enter(im); 2750 */ 2751 2752 /* %r1 = __bpf_tramp_enter */ 2753 load_imm64(jit, REG_1, (u64)__bpf_tramp_enter); 2754 /* %r2 = im */ 2755 load_imm64(jit, REG_2, (u64)im); 2756 /* %r1() */ 2757 call_r1(jit); 2758 } 2759 2760 for (i = 0; i < fentry->nr_links; i++) 2761 if (invoke_bpf_prog(tjit, m, fentry->links[i], 2762 flags & BPF_TRAMP_F_RET_FENTRY_RET)) 2763 return -EINVAL; 2764 2765 if (fmod_ret->nr_links) { 2766 /* 2767 * retval = 0; 2768 */ 2769 2770 /* xc retval_off(8,%r15),retval_off(%r15) */ 2771 _EMIT6(0xd707f000 | tjit->retval_off, 2772 0xf000 | tjit->retval_off); 2773 2774 for (i = 0; i < fmod_ret->nr_links; i++) { 2775 if (invoke_bpf_prog(tjit, m, fmod_ret->links[i], true)) 2776 return -EINVAL; 2777 2778 /* 2779 * if (retval) 2780 * goto do_fexit; 2781 */ 2782 2783 /* ltg %r0,retval_off(%r15) */ 2784 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_0, REG_0, REG_15, 2785 tjit->retval_off); 2786 /* brcl 7,do_fexit */ 2787 EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit); 2788 } 2789 } 2790 2791 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2792 /* 2793 * retval = func_addr(args); 2794 */ 2795 2796 /* lmg %r2,%rN,reg_args_off(%r15) */ 2797 if (nr_reg_args) 2798 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2799 REG_2 + (nr_reg_args - 1), REG_15, 2800 tjit->reg_args_off); 2801 /* mvc stack_args_off(N,%r15),orig_stack_args_off(%r15) */ 2802 if (nr_stack_args) 2803 _EMIT6(0xd200f000 | 2804 (nr_stack_args * sizeof(u64) - 1) << 16 | 2805 tjit->stack_args_off, 2806 0xf000 | tjit->orig_stack_args_off); 2807 /* mvc tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2808 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), 2809 0xf000 | tjit->tccnt_off); 2810 /* lgr %r1,%r8 */ 2811 EMIT4(0xb9040000, REG_1, REG_8); 2812 /* %r1() */ 2813 call_r1(jit); 2814 /* stg %r2,retval_off(%r15) */ 2815 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2816 tjit->retval_off); 2817 2818 im->ip_after_call = jit->prg_buf + jit->prg; 2819 2820 /* 2821 * The following nop will be patched by bpf_tramp_image_put(). 2822 */ 2823 2824 /* brcl 0,im->ip_epilogue */ 2825 EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); 2826 } 2827 2828 /* do_fexit: */ 2829 tjit->do_fexit = jit->prg; 2830 for (i = 0; i < fexit->nr_links; i++) 2831 if (invoke_bpf_prog(tjit, m, fexit->links[i], false)) 2832 return -EINVAL; 2833 2834 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2835 im->ip_epilogue = jit->prg_buf + jit->prg; 2836 2837 /* 2838 * __bpf_tramp_exit(im); 2839 */ 2840 2841 /* %r1 = __bpf_tramp_exit */ 2842 load_imm64(jit, REG_1, (u64)__bpf_tramp_exit); 2843 /* %r2 = im */ 2844 load_imm64(jit, REG_2, (u64)im); 2845 /* %r1() */ 2846 call_r1(jit); 2847 } 2848 2849 /* lmg %r2,%rN,reg_args_off(%r15) */ 2850 if ((flags & BPF_TRAMP_F_RESTORE_REGS) && nr_reg_args) 2851 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2852 REG_2 + (nr_reg_args - 1), REG_15, 2853 tjit->reg_args_off); 2854 /* lgr %r1,%r8 */ 2855 if (!(flags & BPF_TRAMP_F_SKIP_FRAME)) 2856 EMIT4(0xb9040000, REG_1, REG_8); 2857 /* lmg %r7,%r8,r7_r8_off(%r15) */ 2858 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15, 2859 tjit->r7_r8_off); 2860 /* lg %r14,r14_off(%r15) */ 2861 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off); 2862 /* lg %r2,retval_off(%r15) */ 2863 if (flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET)) 2864 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15, 2865 tjit->retval_off); 2866 /* mvc stack_size+tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2867 _EMIT6(0xd203f000 | (tjit->stack_size + 2868 offsetof(struct prog_frame, tail_call_cnt)), 2869 0xf000 | tjit->tccnt_off); 2870 /* aghi %r15,stack_size */ 2871 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); 2872 if (flags & BPF_TRAMP_F_SKIP_FRAME) 2873 EMIT_JUMP_REG(14); 2874 else 2875 EMIT_JUMP_REG(1); 2876 2877 return 0; 2878 } 2879 2880 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 2881 struct bpf_tramp_links *tlinks, void *orig_call) 2882 { 2883 struct bpf_tramp_image im; 2884 struct bpf_tramp_jit tjit; 2885 int ret; 2886 2887 memset(&tjit, 0, sizeof(tjit)); 2888 2889 ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, 2890 tlinks, orig_call); 2891 2892 return ret < 0 ? ret : tjit.common.prg; 2893 } 2894 2895 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 2896 void *image_end, const struct btf_func_model *m, 2897 u32 flags, struct bpf_tramp_links *tlinks, 2898 void *func_addr) 2899 { 2900 struct bpf_tramp_jit tjit; 2901 int ret; 2902 2903 /* Compute offsets, check whether the code fits. */ 2904 memset(&tjit, 0, sizeof(tjit)); 2905 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 2906 tlinks, func_addr); 2907 2908 if (ret < 0) 2909 return ret; 2910 if (tjit.common.prg > (char *)image_end - (char *)image) 2911 /* 2912 * Use the same error code as for exceeding 2913 * BPF_MAX_TRAMP_LINKS. 2914 */ 2915 return -E2BIG; 2916 2917 tjit.common.prg = 0; 2918 tjit.common.prg_buf = image; 2919 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 2920 tlinks, func_addr); 2921 2922 return ret < 0 ? ret : tjit.common.prg; 2923 } 2924 2925 bool bpf_jit_supports_subprog_tailcalls(void) 2926 { 2927 return true; 2928 } 2929 2930 bool bpf_jit_supports_arena(void) 2931 { 2932 return true; 2933 } 2934 2935 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 2936 { 2937 if (!in_arena) 2938 return true; 2939 switch (insn->code) { 2940 case BPF_STX | BPF_ATOMIC | BPF_B: 2941 case BPF_STX | BPF_ATOMIC | BPF_H: 2942 case BPF_STX | BPF_ATOMIC | BPF_W: 2943 case BPF_STX | BPF_ATOMIC | BPF_DW: 2944 if (bpf_atomic_is_load_store(insn)) 2945 return false; 2946 } 2947 return true; 2948 } 2949 2950 bool bpf_jit_supports_exceptions(void) 2951 { 2952 /* 2953 * Exceptions require unwinding support, which is always available, 2954 * because the kernel is always built with backchain. 2955 */ 2956 return true; 2957 } 2958 2959 void arch_bpf_stack_walk(bool (*consume_fn)(void *, u64, u64, u64), 2960 void *cookie) 2961 { 2962 unsigned long addr, prev_addr = 0; 2963 struct unwind_state state; 2964 2965 unwind_for_each_frame(&state, NULL, NULL, 0) { 2966 addr = unwind_get_return_address(&state); 2967 if (!addr) 2968 break; 2969 /* 2970 * addr is a return address and state.sp is the value of %r15 2971 * at this address. exception_cb needs %r15 at entry to the 2972 * function containing addr, so take the next state.sp. 2973 * 2974 * There is no bp, and the exception_cb prog does not need one 2975 * to perform a quasi-longjmp. The common code requires a 2976 * non-zero bp, so pass sp there as well. 2977 */ 2978 if (prev_addr && !consume_fn(cookie, prev_addr, state.sp, 2979 state.sp)) 2980 break; 2981 prev_addr = addr; 2982 } 2983 } 2984