1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * BPF Jit compiler for s390. 4 * 5 * Minimum build requirements: 6 * 7 * - HAVE_MARCH_Z196_FEATURES: laal, laalg 8 * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj 9 * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf 10 * - 64BIT 11 * 12 * Copyright IBM Corp. 2012,2015 13 * 14 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 15 * Michael Holzheu <holzheu@linux.vnet.ibm.com> 16 */ 17 18 #define pr_fmt(fmt) "bpf_jit: " fmt 19 20 #include <linux/netdevice.h> 21 #include <linux/filter.h> 22 #include <linux/init.h> 23 #include <linux/bpf.h> 24 #include <linux/cfi.h> 25 #include <linux/mm.h> 26 #include <linux/kernel.h> 27 #include <asm/cacheflush.h> 28 #include <asm/extable.h> 29 #include <asm/dis.h> 30 #include <asm/facility.h> 31 #include <asm/lowcore.h> 32 #include <asm/nospec-branch.h> 33 #include <asm/set_memory.h> 34 #include <asm/text-patching.h> 35 #include <asm/unwind.h> 36 37 struct bpf_jit { 38 u32 seen; /* Flags to remember seen eBPF instructions */ 39 u16 seen_regs; /* Mask to remember which registers are used */ 40 u32 *addrs; /* Array with relative instruction addresses */ 41 u8 *prg_buf; /* Start of program */ 42 int size; /* Size of program and literal pool */ 43 int size_prg; /* Size of program */ 44 int prg; /* Current position in program */ 45 int lit32_start; /* Start of 32-bit literal pool */ 46 int lit32; /* Current position in 32-bit literal pool */ 47 int lit64_start; /* Start of 64-bit literal pool */ 48 int lit64; /* Current position in 64-bit literal pool */ 49 int base_ip; /* Base address for literal pool */ 50 int exit_ip; /* Address of exit */ 51 int tail_call_start; /* Tail call start offset */ 52 int excnt; /* Number of exception table entries */ 53 int prologue_plt_ret; /* Return address for prologue hotpatch PLT */ 54 int prologue_plt; /* Start of prologue hotpatch PLT */ 55 int kern_arena; /* Pool offset of kernel arena address */ 56 u64 user_arena; /* User arena address */ 57 u32 frame_off; /* Offset of struct bpf_prog from %r15 */ 58 }; 59 60 #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */ 61 #define SEEN_LITERAL BIT(1) /* code uses literals */ 62 #define SEEN_FUNC BIT(2) /* calls C functions */ 63 #define SEEN_STACK (SEEN_FUNC | SEEN_MEM) 64 65 #define NVREGS 0xffc0 /* %r6-%r15 */ 66 67 /* 68 * s390 registers 69 */ 70 #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ 71 #define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ 72 #define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */ 73 #define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */ 74 #define REG_0 REG_W0 /* Register 0 */ 75 #define REG_1 REG_W1 /* Register 1 */ 76 #define REG_2 BPF_REG_1 /* Register 2 */ 77 #define REG_3 BPF_REG_2 /* Register 3 */ 78 #define REG_4 BPF_REG_3 /* Register 4 */ 79 #define REG_7 BPF_REG_6 /* Register 7 */ 80 #define REG_8 BPF_REG_7 /* Register 8 */ 81 #define REG_14 BPF_REG_0 /* Register 14 */ 82 83 /* 84 * Mapping of BPF registers to s390 registers 85 */ 86 static const int reg2hex[] = { 87 /* Return code */ 88 [BPF_REG_0] = 14, 89 /* Function parameters */ 90 [BPF_REG_1] = 2, 91 [BPF_REG_2] = 3, 92 [BPF_REG_3] = 4, 93 [BPF_REG_4] = 5, 94 [BPF_REG_5] = 6, 95 /* Call saved registers */ 96 [BPF_REG_6] = 7, 97 [BPF_REG_7] = 8, 98 [BPF_REG_8] = 9, 99 [BPF_REG_9] = 10, 100 /* BPF stack pointer */ 101 [BPF_REG_FP] = 13, 102 /* Register for blinding */ 103 [BPF_REG_AX] = 12, 104 /* Work registers for s390x backend */ 105 [REG_W0] = 0, 106 [REG_W1] = 1, 107 [REG_L] = 11, 108 [REG_15] = 15, 109 }; 110 111 static inline u32 reg(u32 dst_reg, u32 src_reg) 112 { 113 return reg2hex[dst_reg] << 4 | reg2hex[src_reg]; 114 } 115 116 static inline u32 reg_high(u32 reg) 117 { 118 return reg2hex[reg] << 4; 119 } 120 121 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 122 { 123 u32 r1 = reg2hex[b1]; 124 125 if (r1 >= 6 && r1 <= 15) 126 jit->seen_regs |= (1 << r1); 127 } 128 129 static s32 off_to_pcrel(struct bpf_jit *jit, u32 off) 130 { 131 return off - jit->prg; 132 } 133 134 static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr) 135 { 136 if (jit->prg_buf) 137 return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg); 138 return 0; 139 } 140 141 #define REG_SET_SEEN(b1) \ 142 ({ \ 143 reg_set_seen(jit, b1); \ 144 }) 145 146 /* 147 * EMIT macros for code generation 148 */ 149 150 #define _EMIT2(op) \ 151 ({ \ 152 if (jit->prg_buf) \ 153 *(u16 *) (jit->prg_buf + jit->prg) = (op); \ 154 jit->prg += 2; \ 155 }) 156 157 #define EMIT2(op, b1, b2) \ 158 ({ \ 159 _EMIT2((op) | reg(b1, b2)); \ 160 REG_SET_SEEN(b1); \ 161 REG_SET_SEEN(b2); \ 162 }) 163 164 #define _EMIT4(op) \ 165 ({ \ 166 if (jit->prg_buf) \ 167 *(u32 *) (jit->prg_buf + jit->prg) = (op); \ 168 jit->prg += 4; \ 169 }) 170 171 #define EMIT4(op, b1, b2) \ 172 ({ \ 173 _EMIT4((op) | reg(b1, b2)); \ 174 REG_SET_SEEN(b1); \ 175 REG_SET_SEEN(b2); \ 176 }) 177 178 #define EMIT4_RRF(op, b1, b2, b3) \ 179 ({ \ 180 _EMIT4((op) | reg_high(b3) << 8 | reg(b1, b2)); \ 181 REG_SET_SEEN(b1); \ 182 REG_SET_SEEN(b2); \ 183 REG_SET_SEEN(b3); \ 184 }) 185 186 #define _EMIT4_DISP(op, disp) \ 187 ({ \ 188 unsigned int __disp = (disp) & 0xfff; \ 189 _EMIT4((op) | __disp); \ 190 }) 191 192 #define EMIT4_DISP(op, b1, b2, disp) \ 193 ({ \ 194 _EMIT4_DISP((op) | reg_high(b1) << 16 | \ 195 reg_high(b2) << 8, (disp)); \ 196 REG_SET_SEEN(b1); \ 197 REG_SET_SEEN(b2); \ 198 }) 199 200 #define EMIT4_IMM(op, b1, imm) \ 201 ({ \ 202 unsigned int __imm = (imm) & 0xffff; \ 203 _EMIT4((op) | reg_high(b1) << 16 | __imm); \ 204 REG_SET_SEEN(b1); \ 205 }) 206 207 #define EMIT4_PCREL(op, pcrel) \ 208 ({ \ 209 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 210 _EMIT4((op) | __pcrel); \ 211 }) 212 213 #define EMIT4_PCREL_RIC(op, mask, target) \ 214 ({ \ 215 int __rel = off_to_pcrel(jit, target) / 2; \ 216 _EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \ 217 }) 218 219 #define _EMIT6(op1, op2) \ 220 ({ \ 221 if (jit->prg_buf) { \ 222 *(u32 *) (jit->prg_buf + jit->prg) = (op1); \ 223 *(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \ 224 } \ 225 jit->prg += 6; \ 226 }) 227 228 #define _EMIT6_DISP(op1, op2, disp) \ 229 ({ \ 230 unsigned int __disp = (disp) & 0xfff; \ 231 _EMIT6((op1) | __disp, op2); \ 232 }) 233 234 #define _EMIT6_DISP_LH(op1, op2, disp) \ 235 ({ \ 236 u32 _disp = (u32) (disp); \ 237 unsigned int __disp_h = _disp & 0xff000; \ 238 unsigned int __disp_l = _disp & 0x00fff; \ 239 _EMIT6((op1) | __disp_l, (op2) | __disp_h >> 4); \ 240 }) 241 242 #define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \ 243 ({ \ 244 _EMIT6_DISP_LH((op1) | reg(b1, b2) << 16 | \ 245 reg_high(b3) << 8, op2, disp); \ 246 REG_SET_SEEN(b1); \ 247 REG_SET_SEEN(b2); \ 248 REG_SET_SEEN(b3); \ 249 }) 250 251 #define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \ 252 ({ \ 253 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 254 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \ 255 (op2) | (mask) << 12); \ 256 REG_SET_SEEN(b1); \ 257 REG_SET_SEEN(b2); \ 258 }) 259 260 #define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \ 261 ({ \ 262 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 263 _EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \ 264 (rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \ 265 REG_SET_SEEN(b1); \ 266 BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \ 267 }) 268 269 #define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \ 270 ({ \ 271 int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\ 272 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\ 273 REG_SET_SEEN(b1); \ 274 REG_SET_SEEN(b2); \ 275 }) 276 277 static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel) 278 { 279 u32 pc32dbl = (s32)(pcrel / 2); 280 281 _EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff); 282 } 283 284 static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel) 285 { 286 emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel); 287 REG_SET_SEEN(b); 288 } 289 290 #define EMIT6_PCREL_RILB(op, b, target) \ 291 emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target)) 292 293 #define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \ 294 emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr)) 295 296 static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel) 297 { 298 emit6_pcrel_ril(jit, op | mask << 20, pcrel); 299 } 300 301 #define EMIT6_PCREL_RILC(op, mask, target) \ 302 emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target)) 303 304 #define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \ 305 emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr)) 306 307 #define _EMIT6_IMM(op, imm) \ 308 ({ \ 309 unsigned int __imm = (imm); \ 310 _EMIT6((op) | (__imm >> 16), __imm & 0xffff); \ 311 }) 312 313 #define EMIT6_IMM(op, b1, imm) \ 314 ({ \ 315 _EMIT6_IMM((op) | reg_high(b1) << 16, imm); \ 316 REG_SET_SEEN(b1); \ 317 }) 318 319 #define _EMIT_CONST_U32(val) \ 320 ({ \ 321 unsigned int ret; \ 322 ret = jit->lit32; \ 323 if (jit->prg_buf) \ 324 *(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\ 325 jit->lit32 += 4; \ 326 ret; \ 327 }) 328 329 #define EMIT_CONST_U32(val) \ 330 ({ \ 331 jit->seen |= SEEN_LITERAL; \ 332 _EMIT_CONST_U32(val) - jit->base_ip; \ 333 }) 334 335 #define _EMIT_CONST_U64(val) \ 336 ({ \ 337 unsigned int ret; \ 338 ret = jit->lit64; \ 339 if (jit->prg_buf) \ 340 *(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\ 341 jit->lit64 += 8; \ 342 ret; \ 343 }) 344 345 #define EMIT_CONST_U64(val) \ 346 ({ \ 347 jit->seen |= SEEN_LITERAL; \ 348 _EMIT_CONST_U64(val) - jit->base_ip; \ 349 }) 350 351 #define EMIT_ZERO(b1) \ 352 ({ \ 353 if (!fp->aux->verifier_zext) { \ 354 /* llgfr %dst,%dst (zero extend to 64 bit) */ \ 355 EMIT4(0xb9160000, b1, b1); \ 356 REG_SET_SEEN(b1); \ 357 } \ 358 }) 359 360 static inline void emit_u32_data(const u32 data, struct bpf_jit *jit) 361 { 362 if (jit->prg_buf) 363 *(u32 *)(jit->prg_buf + jit->prg) = data; 364 jit->prg += 4; 365 } 366 367 static inline void emit_kcfi(u32 hash, struct bpf_jit *jit) 368 { 369 if (IS_ENABLED(CONFIG_CFI)) 370 emit_u32_data(hash, jit); 371 } 372 373 /* 374 * Return whether this is the first pass. The first pass is special, since we 375 * don't know any sizes yet, and thus must be conservative. 376 */ 377 static bool is_first_pass(struct bpf_jit *jit) 378 { 379 return jit->size == 0; 380 } 381 382 /* 383 * Return whether this is the code generation pass. The code generation pass is 384 * special, since we should change as little as possible. 385 */ 386 static bool is_codegen_pass(struct bpf_jit *jit) 387 { 388 return jit->prg_buf; 389 } 390 391 /* 392 * Return whether "rel" can be encoded as a short PC-relative offset 393 */ 394 static bool is_valid_rel(int rel) 395 { 396 return rel >= -65536 && rel <= 65534; 397 } 398 399 /* 400 * Return whether "off" can be reached using a short PC-relative offset 401 */ 402 static bool can_use_rel(struct bpf_jit *jit, int off) 403 { 404 return is_valid_rel(off - jit->prg); 405 } 406 407 /* 408 * Return whether given displacement can be encoded using 409 * Long-Displacement Facility 410 */ 411 static bool is_valid_ldisp(int disp) 412 { 413 return disp >= -524288 && disp <= 524287; 414 } 415 416 /* 417 * Return whether the next 32-bit literal pool entry can be referenced using 418 * Long-Displacement Facility 419 */ 420 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit) 421 { 422 return is_valid_ldisp(jit->lit32 - jit->base_ip); 423 } 424 425 /* 426 * Return whether the next 64-bit literal pool entry can be referenced using 427 * Long-Displacement Facility 428 */ 429 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit) 430 { 431 return is_valid_ldisp(jit->lit64 - jit->base_ip); 432 } 433 434 /* 435 * Fill whole space with illegal instructions 436 */ 437 static void jit_fill_hole(void *area, unsigned int size) 438 { 439 memset(area, 0, size); 440 } 441 442 /* 443 * Caller-allocated part of the frame. 444 * Thanks to packed stack, its otherwise unused initial part can be used for 445 * the BPF stack and for the next frame. 446 */ 447 struct prog_frame { 448 u64 unused[8]; 449 /* BPF stack starts here and grows towards 0 */ 450 u32 tail_call_cnt; 451 u32 pad; 452 u64 r6[10]; /* r6 - r15 */ 453 u64 backchain; 454 } __packed; 455 456 /* 457 * Save registers from "rs" (register start) to "re" (register end) on stack 458 */ 459 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) 460 { 461 u32 off = offsetof(struct prog_frame, r6) + (rs - 6) * 8; 462 463 if (rs == re) 464 /* stg %rs,off(%r15) */ 465 _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); 466 else 467 /* stmg %rs,%re,off(%r15) */ 468 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); 469 } 470 471 /* 472 * Restore registers from "rs" (register start) to "re" (register end) on stack 473 */ 474 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re) 475 { 476 u32 off = jit->frame_off + offsetof(struct prog_frame, r6) + (rs - 6) * 8; 477 478 if (rs == re) 479 /* lg %rs,off(%r15) */ 480 _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); 481 else 482 /* lmg %rs,%re,off(%r15) */ 483 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); 484 } 485 486 /* 487 * Return first seen register (from start) 488 */ 489 static int get_start(u16 seen_regs, int start) 490 { 491 int i; 492 493 for (i = start; i <= 15; i++) { 494 if (seen_regs & (1 << i)) 495 return i; 496 } 497 return 0; 498 } 499 500 /* 501 * Return last seen register (from start) (gap >= 2) 502 */ 503 static int get_end(u16 seen_regs, int start) 504 { 505 int i; 506 507 for (i = start; i < 15; i++) { 508 if (!(seen_regs & (3 << i))) 509 return i - 1; 510 } 511 return (seen_regs & (1 << 15)) ? 15 : 14; 512 } 513 514 #define REGS_SAVE 1 515 #define REGS_RESTORE 0 516 /* 517 * Save and restore clobbered registers (6-15) on stack. 518 * We save/restore registers in chunks with gap >= 2 registers. 519 */ 520 static void save_restore_regs(struct bpf_jit *jit, int op, u16 extra_regs) 521 { 522 u16 seen_regs = jit->seen_regs | extra_regs; 523 const int last = 15, save_restore_size = 6; 524 int re = 6, rs; 525 526 if (is_first_pass(jit)) { 527 /* 528 * We don't know yet which registers are used. Reserve space 529 * conservatively. 530 */ 531 jit->prg += (last - re + 1) * save_restore_size; 532 return; 533 } 534 535 do { 536 rs = get_start(seen_regs, re); 537 if (!rs) 538 break; 539 re = get_end(seen_regs, rs + 1); 540 if (op == REGS_SAVE) 541 save_regs(jit, rs, re); 542 else 543 restore_regs(jit, rs, re); 544 re++; 545 } while (re <= last); 546 } 547 548 static void bpf_skip(struct bpf_jit *jit, int size) 549 { 550 if (size >= 6 && !is_valid_rel(size)) { 551 /* brcl 0xf,size */ 552 EMIT6_PCREL_RILC(0xc0040000, 0xf, size); 553 size -= 6; 554 } else if (size >= 4 && is_valid_rel(size)) { 555 /* brc 0xf,size */ 556 EMIT4_PCREL(0xa7f40000, size); 557 size -= 4; 558 } 559 while (size >= 2) { 560 /* bcr 0,%0 */ 561 _EMIT2(0x0700); 562 size -= 2; 563 } 564 } 565 566 /* 567 * PLT for hotpatchable calls. The calling convention is the same as for the 568 * ftrace hotpatch trampolines: %r0 is return address, %r1 is clobbered. 569 */ 570 struct bpf_plt { 571 char code[16]; 572 void *ret; 573 void *target; 574 } __packed; 575 extern const struct bpf_plt bpf_plt; 576 asm( 577 ".pushsection .rodata\n" 578 " .balign 8\n" 579 "bpf_plt:\n" 580 " lgrl %r0,bpf_plt_ret\n" 581 " lgrl %r1,bpf_plt_target\n" 582 " br %r1\n" 583 " .balign 8\n" 584 "bpf_plt_ret: .quad 0\n" 585 "bpf_plt_target: .quad 0\n" 586 " .popsection\n" 587 ); 588 589 static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target) 590 { 591 memcpy(plt, &bpf_plt, sizeof(*plt)); 592 plt->ret = ret; 593 /* 594 * (target == NULL) implies that the branch to this PLT entry was 595 * patched and became a no-op. However, some CPU could have jumped 596 * to this PLT entry before patching and may be still executing it. 597 * 598 * Since the intention in this case is to make the PLT entry a no-op, 599 * make the target point to the return label instead of NULL. 600 */ 601 plt->target = target ?: ret; 602 } 603 604 /* 605 * Emit function prologue 606 * 607 * Save registers and create stack frame if necessary. 608 * Stack frame layout is described by struct prog_frame. 609 */ 610 static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) 611 { 612 BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD); 613 614 emit_kcfi(bpf_is_subprog(fp) ? cfi_bpf_subprog_hash : cfi_bpf_hash, jit); 615 616 /* No-op for hotpatching */ 617 /* brcl 0,prologue_plt */ 618 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); 619 jit->prologue_plt_ret = jit->prg; 620 621 if (!bpf_is_subprog(fp)) { 622 /* Initialize the tail call counter in the main program. */ 623 /* xc tail_call_cnt(4,%r15),tail_call_cnt(%r15) */ 624 _EMIT6(0xd703f000 | offsetof(struct prog_frame, tail_call_cnt), 625 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); 626 } else { 627 /* 628 * Skip the tail call counter initialization in subprograms. 629 * Insert nops in order to have tail_call_start at a 630 * predictable offset. 631 */ 632 bpf_skip(jit, 6); 633 } 634 /* Tail calls have to skip above initialization */ 635 jit->tail_call_start = jit->prg - cfi_get_offset(); 636 if (fp->aux->exception_cb) { 637 /* 638 * Switch stack, the new address is in the 2nd parameter. 639 * 640 * Arrange the restoration of %r6-%r15 in the epilogue. 641 * Do not restore them now, the prog does not need them. 642 */ 643 /* lgr %r15,%r3 */ 644 EMIT4(0xb9040000, REG_15, REG_3); 645 jit->seen_regs |= NVREGS; 646 } else { 647 /* Save registers */ 648 save_restore_regs(jit, REGS_SAVE, 649 fp->aux->exception_boundary ? NVREGS : 0); 650 } 651 /* Setup literal pool */ 652 if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) { 653 if (!is_first_pass(jit) && 654 is_valid_ldisp(jit->size - (jit->prg + 2))) { 655 /* basr %l,0 */ 656 EMIT2(0x0d00, REG_L, REG_0); 657 jit->base_ip = jit->prg; 658 } else { 659 /* larl %l,lit32_start */ 660 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start); 661 jit->base_ip = jit->lit32_start; 662 } 663 } 664 /* Setup stack and backchain */ 665 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) { 666 /* lgr %w1,%r15 (backchain) */ 667 EMIT4(0xb9040000, REG_W1, REG_15); 668 /* la %bfp,unused_end(%r15) (BPF frame pointer) */ 669 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, 670 offsetofend(struct prog_frame, unused)); 671 /* aghi %r15,-frame_off */ 672 EMIT4_IMM(0xa70b0000, REG_15, -jit->frame_off); 673 /* stg %w1,backchain(%r15) */ 674 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, 675 REG_15, 676 offsetof(struct prog_frame, backchain)); 677 } 678 } 679 680 /* 681 * Jump using a register either directly or via an expoline thunk 682 */ 683 #define EMIT_JUMP_REG(reg) do { \ 684 if (nospec_uses_trampoline()) \ 685 /* brcl 0xf,__s390_indirect_jump_rN */ \ 686 EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \ 687 __s390_indirect_jump_r ## reg); \ 688 else \ 689 /* br %rN */ \ 690 _EMIT2(0x07f0 | reg); \ 691 } while (0) 692 693 /* 694 * Function epilogue 695 */ 696 static void bpf_jit_epilogue(struct bpf_jit *jit) 697 { 698 jit->exit_ip = jit->prg; 699 /* Load exit code: lgr %r2,%b0 */ 700 EMIT4(0xb9040000, REG_2, BPF_REG_0); 701 /* Restore registers */ 702 save_restore_regs(jit, REGS_RESTORE, 0); 703 EMIT_JUMP_REG(14); 704 705 jit->prg = ALIGN(jit->prg, 8); 706 jit->prologue_plt = jit->prg; 707 if (jit->prg_buf) 708 bpf_jit_plt((struct bpf_plt *)(jit->prg_buf + jit->prg), 709 jit->prg_buf + jit->prologue_plt_ret, NULL); 710 jit->prg += sizeof(struct bpf_plt); 711 } 712 713 bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) 714 { 715 regs->psw.addr = extable_fixup(x); 716 if (x->data != -1) 717 regs->gprs[x->data] = 0; 718 return true; 719 } 720 721 /* 722 * A single BPF probe instruction 723 */ 724 struct bpf_jit_probe { 725 int prg; /* JITed instruction offset */ 726 int reg; /* Register to clear on exception */ 727 int arena_reg; /* Register to use for arena addressing */ 728 }; 729 730 static void bpf_jit_probe_init(struct bpf_jit_probe *probe) 731 { 732 probe->prg = -1; 733 probe->reg = -1; 734 probe->arena_reg = REG_0; 735 } 736 737 static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, 738 struct bpf_jit_probe *probe) 739 { 740 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 741 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 742 BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 743 BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 744 return; 745 746 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 747 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { 748 /* lgrl %r1,kern_arena */ 749 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 750 probe->arena_reg = REG_W1; 751 } 752 probe->prg = jit->prg; 753 probe->reg = reg2hex[insn->dst_reg]; 754 } 755 756 static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, 757 struct bpf_jit_probe *probe) 758 { 759 if (BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 760 BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 761 return; 762 763 /* lgrl %r1,kern_arena */ 764 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 765 probe->arena_reg = REG_W1; 766 probe->prg = jit->prg; 767 } 768 769 static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, 770 struct bpf_insn *insn, 771 struct bpf_jit_probe *probe) 772 { 773 int load_reg; 774 775 if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 776 return; 777 778 /* lgrl %r1,kern_arena */ 779 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 780 /* agr %r1,%dst */ 781 EMIT4(0xb9080000, REG_W1, insn->dst_reg); 782 probe->arena_reg = REG_W1; 783 probe->prg = jit->prg; 784 /* 785 * A read-modify-write carrying BPF_FETCH reads the old value into 786 * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on 787 * fault, the remaining atomics only write memory. 788 */ 789 load_reg = bpf_atomic_load_reg(insn); 790 if (load_reg >= 0) 791 probe->reg = reg2hex[load_reg]; 792 } 793 794 static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, 795 struct bpf_jit_probe *probe) 796 { 797 struct exception_table_entry *ex; 798 s64 delta; 799 800 if (probe->prg == -1) 801 /* The probe is not armed. */ 802 return 0; 803 if (!fp->aux->extable) 804 /* Do nothing during early JIT passes. */ 805 return 0; 806 if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) 807 /* Verifier bug - not enough entries. */ 808 return -1; 809 ex = &fp->aux->extable[jit->excnt]; 810 delta = jit->prg_buf + probe->prg - (u8 *)&ex->insn; 811 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 812 /* JIT bug - code and extable must be close. */ 813 return -1; 814 ex->insn = delta; 815 /* 816 * Land on the current instruction. Note that the extable 817 * infrastructure ignores the fixup field; it is handled by 818 * ex_handler_bpf(). 819 */ 820 delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; 821 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 822 /* JIT bug - landing pad and extable must be close. */ 823 return -1; 824 ex->fixup = delta; 825 ex->type = EX_TYPE_BPF; 826 ex->data = probe->reg; 827 jit->excnt++; 828 return 0; 829 } 830 831 static void emit_shift_count32(struct bpf_jit *jit, u32 src_reg) 832 { 833 /* risbg %w1,%src,59,128+63,0 */ 834 _EMIT6(0xec000000 | reg(REG_W1, src_reg) << 16 | 59 << 8 | (0x80 | 63), 0x0055); 835 REG_SET_SEEN(REG_W1); 836 REG_SET_SEEN(src_reg); 837 } 838 839 static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) 840 { 841 struct bpf_jit_probe probe; 842 843 bpf_jit_probe_init(&probe); 844 bpf_jit_probe_load_pre(jit, insn, &probe); 845 846 switch (BPF_SIZE(insn->code)) { 847 case BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 848 /* llgc %dst,off(%src,%arena) */ 849 EMIT6_DISP_LH(0xe3000000, 0x0090, insn->dst_reg, insn->src_reg, 850 probe.arena_reg, insn->off); 851 break; 852 case BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 853 /* llgh %dst,off(%src,%arena) */ 854 EMIT6_DISP_LH(0xe3000000, 0x0091, insn->dst_reg, insn->src_reg, 855 probe.arena_reg, insn->off); 856 break; 857 case BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 858 /* llgf %dst,off(%src,%arena) */ 859 EMIT6_DISP_LH(0xe3000000, 0x0016, insn->dst_reg, insn->src_reg, 860 probe.arena_reg, insn->off); 861 break; 862 case BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 863 /* lg %dst,off(%src,%arena) */ 864 EMIT6_DISP_LH(0xe3000000, 0x0004, insn->dst_reg, insn->src_reg, 865 probe.arena_reg, insn->off); 866 break; 867 } 868 869 return bpf_jit_probe_post(jit, fp, &probe); 870 } 871 872 static int emit_stx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) 873 { 874 struct bpf_jit_probe probe; 875 876 bpf_jit_probe_init(&probe); 877 bpf_jit_probe_store_pre(jit, insn, &probe); 878 879 switch (BPF_SIZE(insn->code)) { 880 case BPF_B: /* *(u8 *)(dst + off) = src_reg */ 881 /* stcy %src,off(%dst,%arena) */ 882 EMIT6_DISP_LH(0xe3000000, 0x0072, insn->src_reg, insn->dst_reg, 883 probe.arena_reg, insn->off); 884 break; 885 case BPF_H: /* (u16 *)(dst + off) = src */ 886 /* sthy %src,off(%dst,%arena) */ 887 EMIT6_DISP_LH(0xe3000000, 0x0070, insn->src_reg, insn->dst_reg, 888 probe.arena_reg, insn->off); 889 break; 890 case BPF_W: /* *(u32 *)(dst + off) = src */ 891 /* sty %src,off(%dst,%arena) */ 892 EMIT6_DISP_LH(0xe3000000, 0x0050, insn->src_reg, insn->dst_reg, 893 probe.arena_reg, insn->off); 894 break; 895 case BPF_DW: /* (u64 *)(dst + off) = src */ 896 /* stg %src,off(%dst,%arena) */ 897 EMIT6_DISP_LH(0xe3000000, 0x0024, insn->src_reg, insn->dst_reg, 898 probe.arena_reg, insn->off); 899 break; 900 } 901 902 return bpf_jit_probe_post(jit, fp, &probe); 903 } 904 905 /* 906 * Sign- or zero-extend the register if necessary 907 */ 908 static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) 909 { 910 switch (size) { 911 case 1: 912 if (flags & BTF_FMODEL_SIGNED_ARG) 913 /* lgbr %r,%r */ 914 EMIT4(0xb9060000, r, r); 915 else 916 /* llgcr %r,%r */ 917 EMIT4(0xb9840000, r, r); 918 return 0; 919 case 2: 920 if (flags & BTF_FMODEL_SIGNED_ARG) 921 /* lghr %r,%r */ 922 EMIT4(0xb9070000, r, r); 923 else 924 /* llghr %r,%r */ 925 EMIT4(0xb9850000, r, r); 926 return 0; 927 case 4: 928 if (flags & BTF_FMODEL_SIGNED_ARG) 929 /* lgfr %r,%r */ 930 EMIT4(0xb9140000, r, r); 931 else 932 /* llgfr %r,%r */ 933 EMIT4(0xb9160000, r, r); 934 return 0; 935 case 8: 936 return 0; 937 default: 938 return -1; 939 } 940 } 941 942 /* 943 * Compile one eBPF instruction into s390x code 944 * 945 * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of 946 * stack space for the large switch statement. 947 */ 948 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 949 int i, bool extra_pass) 950 { 951 struct bpf_insn *insn = &fp->insnsi[i]; 952 s32 branch_oc_off = insn->off; 953 u32 dst_reg = insn->dst_reg; 954 u32 src_reg = insn->src_reg; 955 struct bpf_jit_probe probe; 956 int last, insn_count = 1; 957 u32 *addrs = jit->addrs; 958 s32 imm = insn->imm; 959 s16 off = insn->off; 960 unsigned int mask; 961 int err; 962 963 bpf_jit_probe_init(&probe); 964 965 switch (insn->code) { 966 /* 967 * BPF_MOV 968 */ 969 case BPF_ALU | BPF_MOV | BPF_X: 970 switch (insn->off) { 971 case 0: /* DST = (u32) SRC */ 972 /* llgfr %dst,%src */ 973 EMIT4(0xb9160000, dst_reg, src_reg); 974 if (insn_is_zext(&insn[1])) 975 insn_count = 2; 976 break; 977 case 8: /* DST = (u32)(s8) SRC */ 978 /* lbr %dst,%src */ 979 EMIT4(0xb9260000, dst_reg, src_reg); 980 /* llgfr %dst,%dst */ 981 EMIT4(0xb9160000, dst_reg, dst_reg); 982 break; 983 case 16: /* DST = (u32)(s16) SRC */ 984 /* lhr %dst,%src */ 985 EMIT4(0xb9270000, dst_reg, src_reg); 986 /* llgfr %dst,%dst */ 987 EMIT4(0xb9160000, dst_reg, dst_reg); 988 break; 989 } 990 break; 991 case BPF_ALU64 | BPF_MOV | BPF_X: 992 if (insn_is_cast_user(insn)) { 993 int patch_brc; 994 995 /* ltgr %dst,%src */ 996 EMIT4(0xb9020000, dst_reg, src_reg); 997 /* brc 8,0f */ 998 patch_brc = jit->prg; 999 EMIT4_PCREL_RIC(0xa7040000, 8, 0); 1000 /* iihf %dst,user_arena>>32 */ 1001 EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32); 1002 /* 0: */ 1003 if (jit->prg_buf) 1004 *(u16 *)(jit->prg_buf + patch_brc + 2) = 1005 (jit->prg - patch_brc) >> 1; 1006 break; 1007 } 1008 switch (insn->off) { 1009 case 0: /* DST = SRC */ 1010 /* lgr %dst,%src */ 1011 EMIT4(0xb9040000, dst_reg, src_reg); 1012 break; 1013 case 8: /* DST = (s8) SRC */ 1014 /* lgbr %dst,%src */ 1015 EMIT4(0xb9060000, dst_reg, src_reg); 1016 break; 1017 case 16: /* DST = (s16) SRC */ 1018 /* lghr %dst,%src */ 1019 EMIT4(0xb9070000, dst_reg, src_reg); 1020 break; 1021 case 32: /* DST = (s32) SRC */ 1022 /* lgfr %dst,%src */ 1023 EMIT4(0xb9140000, dst_reg, src_reg); 1024 break; 1025 } 1026 break; 1027 case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */ 1028 /* llilf %dst,imm */ 1029 EMIT6_IMM(0xc00f0000, dst_reg, imm); 1030 if (insn_is_zext(&insn[1])) 1031 insn_count = 2; 1032 break; 1033 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */ 1034 /* lgfi %dst,imm */ 1035 EMIT6_IMM(0xc0010000, dst_reg, imm); 1036 break; 1037 /* 1038 * BPF_LD 64 1039 */ 1040 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 1041 { 1042 /* 16 byte instruction that uses two 'struct bpf_insn' */ 1043 u64 imm64; 1044 1045 imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; 1046 /* lgrl %dst,imm */ 1047 EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64)); 1048 insn_count = 2; 1049 break; 1050 } 1051 /* 1052 * BPF_ADD 1053 */ 1054 case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */ 1055 /* ar %dst,%src */ 1056 EMIT2(0x1a00, dst_reg, src_reg); 1057 EMIT_ZERO(dst_reg); 1058 break; 1059 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */ 1060 /* agr %dst,%src */ 1061 EMIT4(0xb9080000, dst_reg, src_reg); 1062 break; 1063 case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */ 1064 if (imm != 0) { 1065 /* alfi %dst,imm */ 1066 EMIT6_IMM(0xc20b0000, dst_reg, imm); 1067 } 1068 EMIT_ZERO(dst_reg); 1069 break; 1070 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */ 1071 if (!imm) 1072 break; 1073 /* agfi %dst,imm */ 1074 EMIT6_IMM(0xc2080000, dst_reg, imm); 1075 break; 1076 /* 1077 * BPF_SUB 1078 */ 1079 case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */ 1080 /* sr %dst,%src */ 1081 EMIT2(0x1b00, dst_reg, src_reg); 1082 EMIT_ZERO(dst_reg); 1083 break; 1084 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */ 1085 /* sgr %dst,%src */ 1086 EMIT4(0xb9090000, dst_reg, src_reg); 1087 break; 1088 case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */ 1089 if (imm != 0) { 1090 /* alfi %dst,-imm */ 1091 EMIT6_IMM(0xc20b0000, dst_reg, -imm); 1092 } 1093 EMIT_ZERO(dst_reg); 1094 break; 1095 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */ 1096 if (!imm) 1097 break; 1098 if (imm == -0x80000000) { 1099 /* algfi %dst,0x80000000 */ 1100 EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000); 1101 } else { 1102 /* agfi %dst,-imm */ 1103 EMIT6_IMM(0xc2080000, dst_reg, -imm); 1104 } 1105 break; 1106 /* 1107 * BPF_MUL 1108 */ 1109 case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */ 1110 /* msr %dst,%src */ 1111 EMIT4(0xb2520000, dst_reg, src_reg); 1112 EMIT_ZERO(dst_reg); 1113 break; 1114 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */ 1115 /* msgr %dst,%src */ 1116 EMIT4(0xb90c0000, dst_reg, src_reg); 1117 break; 1118 case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */ 1119 if (imm != 1) { 1120 /* msfi %r5,imm */ 1121 EMIT6_IMM(0xc2010000, dst_reg, imm); 1122 } 1123 EMIT_ZERO(dst_reg); 1124 break; 1125 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */ 1126 if (imm == 1) 1127 break; 1128 /* msgfi %dst,imm */ 1129 EMIT6_IMM(0xc2000000, dst_reg, imm); 1130 break; 1131 /* 1132 * BPF_DIV / BPF_MOD 1133 */ 1134 case BPF_ALU | BPF_DIV | BPF_X: 1135 case BPF_ALU | BPF_MOD | BPF_X: 1136 { 1137 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1138 1139 switch (off) { 1140 case 0: /* dst = (u32) dst {/,%} (u32) src */ 1141 /* xr %w0,%w0 */ 1142 EMIT2(0x1700, REG_W0, REG_W0); 1143 /* lr %w1,%dst */ 1144 EMIT2(0x1800, REG_W1, dst_reg); 1145 /* dlr %w0,%src */ 1146 EMIT4(0xb9970000, REG_W0, src_reg); 1147 break; 1148 case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */ 1149 /* lgfr %r1,%dst */ 1150 EMIT4(0xb9140000, REG_W1, dst_reg); 1151 /* dsgfr %r0,%src */ 1152 EMIT4(0xb91d0000, REG_W0, src_reg); 1153 break; 1154 } 1155 /* llgfr %dst,%rc */ 1156 EMIT4(0xb9160000, dst_reg, rc_reg); 1157 if (insn_is_zext(&insn[1])) 1158 insn_count = 2; 1159 break; 1160 } 1161 case BPF_ALU64 | BPF_DIV | BPF_X: 1162 case BPF_ALU64 | BPF_MOD | BPF_X: 1163 { 1164 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1165 1166 switch (off) { 1167 case 0: /* dst = dst {/,%} src */ 1168 /* lghi %w0,0 */ 1169 EMIT4_IMM(0xa7090000, REG_W0, 0); 1170 /* lgr %w1,%dst */ 1171 EMIT4(0xb9040000, REG_W1, dst_reg); 1172 /* dlgr %w0,%src */ 1173 EMIT4(0xb9870000, REG_W0, src_reg); 1174 break; 1175 case 1: /* dst = (s64) dst {/,%} (s64) src */ 1176 /* lgr %w1,%dst */ 1177 EMIT4(0xb9040000, REG_W1, dst_reg); 1178 /* dsgr %w0,%src */ 1179 EMIT4(0xb90d0000, REG_W0, src_reg); 1180 break; 1181 } 1182 /* lgr %dst,%rc */ 1183 EMIT4(0xb9040000, dst_reg, rc_reg); 1184 break; 1185 } 1186 case BPF_ALU | BPF_DIV | BPF_K: 1187 case BPF_ALU | BPF_MOD | BPF_K: 1188 { 1189 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1190 1191 if (imm == 1) { 1192 if (BPF_OP(insn->code) == BPF_MOD) 1193 /* lghi %dst,0 */ 1194 EMIT4_IMM(0xa7090000, dst_reg, 0); 1195 else 1196 EMIT_ZERO(dst_reg); 1197 break; 1198 } 1199 if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) { 1200 switch (off) { 1201 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1202 /* xr %w0,%w0 */ 1203 EMIT2(0x1700, REG_W0, REG_W0); 1204 /* lr %w1,%dst */ 1205 EMIT2(0x1800, REG_W1, dst_reg); 1206 /* dl %w0,<d(imm)>(%l) */ 1207 EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, 1208 REG_L, EMIT_CONST_U32(imm)); 1209 break; 1210 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1211 /* lgfr %r1,%dst */ 1212 EMIT4(0xb9140000, REG_W1, dst_reg); 1213 /* dsgf %r0,<d(imm)>(%l) */ 1214 EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0, 1215 REG_L, EMIT_CONST_U32(imm)); 1216 break; 1217 } 1218 } else { 1219 switch (off) { 1220 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1221 /* xr %w0,%w0 */ 1222 EMIT2(0x1700, REG_W0, REG_W0); 1223 /* lr %w1,%dst */ 1224 EMIT2(0x1800, REG_W1, dst_reg); 1225 /* lrl %dst,imm */ 1226 EMIT6_PCREL_RILB(0xc40d0000, dst_reg, 1227 _EMIT_CONST_U32(imm)); 1228 jit->seen |= SEEN_LITERAL; 1229 /* dlr %w0,%dst */ 1230 EMIT4(0xb9970000, REG_W0, dst_reg); 1231 break; 1232 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1233 /* lgfr %w1,%dst */ 1234 EMIT4(0xb9140000, REG_W1, dst_reg); 1235 /* lgfrl %dst,imm */ 1236 EMIT6_PCREL_RILB(0xc40c0000, dst_reg, 1237 _EMIT_CONST_U32(imm)); 1238 jit->seen |= SEEN_LITERAL; 1239 /* dsgr %w0,%dst */ 1240 EMIT4(0xb90d0000, REG_W0, dst_reg); 1241 break; 1242 } 1243 } 1244 /* llgfr %dst,%rc */ 1245 EMIT4(0xb9160000, dst_reg, rc_reg); 1246 if (insn_is_zext(&insn[1])) 1247 insn_count = 2; 1248 break; 1249 } 1250 case BPF_ALU64 | BPF_DIV | BPF_K: 1251 case BPF_ALU64 | BPF_MOD | BPF_K: 1252 { 1253 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1254 1255 if (imm == 1) { 1256 if (BPF_OP(insn->code) == BPF_MOD) 1257 /* lhgi %dst,0 */ 1258 EMIT4_IMM(0xa7090000, dst_reg, 0); 1259 break; 1260 } 1261 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1262 switch (off) { 1263 case 0: /* dst = dst {/,%} imm */ 1264 /* lghi %w0,0 */ 1265 EMIT4_IMM(0xa7090000, REG_W0, 0); 1266 /* lgr %w1,%dst */ 1267 EMIT4(0xb9040000, REG_W1, dst_reg); 1268 /* dlg %w0,<d(imm)>(%l) */ 1269 EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, 1270 REG_L, EMIT_CONST_U64(imm)); 1271 break; 1272 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1273 /* lgr %w1,%dst */ 1274 EMIT4(0xb9040000, REG_W1, dst_reg); 1275 /* dsg %w0,<d(imm)>(%l) */ 1276 EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0, 1277 REG_L, EMIT_CONST_U64(imm)); 1278 break; 1279 } 1280 } else { 1281 switch (off) { 1282 case 0: /* dst = dst {/,%} imm */ 1283 /* lghi %w0,0 */ 1284 EMIT4_IMM(0xa7090000, REG_W0, 0); 1285 /* lgr %w1,%dst */ 1286 EMIT4(0xb9040000, REG_W1, dst_reg); 1287 /* lgrl %dst,imm */ 1288 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1289 _EMIT_CONST_U64(imm)); 1290 jit->seen |= SEEN_LITERAL; 1291 /* dlgr %w0,%dst */ 1292 EMIT4(0xb9870000, REG_W0, dst_reg); 1293 break; 1294 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1295 /* lgr %w1,%dst */ 1296 EMIT4(0xb9040000, REG_W1, dst_reg); 1297 /* lgrl %dst,imm */ 1298 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1299 _EMIT_CONST_U64(imm)); 1300 jit->seen |= SEEN_LITERAL; 1301 /* dsgr %w0,%dst */ 1302 EMIT4(0xb90d0000, REG_W0, dst_reg); 1303 break; 1304 } 1305 } 1306 /* lgr %dst,%rc */ 1307 EMIT4(0xb9040000, dst_reg, rc_reg); 1308 break; 1309 } 1310 /* 1311 * BPF_AND 1312 */ 1313 case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */ 1314 /* nr %dst,%src */ 1315 EMIT2(0x1400, dst_reg, src_reg); 1316 EMIT_ZERO(dst_reg); 1317 break; 1318 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 1319 /* ngr %dst,%src */ 1320 EMIT4(0xb9800000, dst_reg, src_reg); 1321 break; 1322 case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */ 1323 /* nilf %dst,imm */ 1324 EMIT6_IMM(0xc00b0000, dst_reg, imm); 1325 EMIT_ZERO(dst_reg); 1326 break; 1327 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 1328 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1329 /* ng %dst,<d(imm)>(%l) */ 1330 EMIT6_DISP_LH(0xe3000000, 0x0080, 1331 dst_reg, REG_0, REG_L, 1332 EMIT_CONST_U64(imm)); 1333 } else { 1334 /* lgrl %w0,imm */ 1335 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1336 _EMIT_CONST_U64(imm)); 1337 jit->seen |= SEEN_LITERAL; 1338 /* ngr %dst,%w0 */ 1339 EMIT4(0xb9800000, dst_reg, REG_W0); 1340 } 1341 break; 1342 /* 1343 * BPF_OR 1344 */ 1345 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 1346 /* or %dst,%src */ 1347 EMIT2(0x1600, dst_reg, src_reg); 1348 EMIT_ZERO(dst_reg); 1349 break; 1350 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 1351 /* ogr %dst,%src */ 1352 EMIT4(0xb9810000, dst_reg, src_reg); 1353 break; 1354 case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */ 1355 /* oilf %dst,imm */ 1356 EMIT6_IMM(0xc00d0000, dst_reg, imm); 1357 EMIT_ZERO(dst_reg); 1358 break; 1359 case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */ 1360 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1361 /* og %dst,<d(imm)>(%l) */ 1362 EMIT6_DISP_LH(0xe3000000, 0x0081, 1363 dst_reg, REG_0, REG_L, 1364 EMIT_CONST_U64(imm)); 1365 } else { 1366 /* lgrl %w0,imm */ 1367 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1368 _EMIT_CONST_U64(imm)); 1369 jit->seen |= SEEN_LITERAL; 1370 /* ogr %dst,%w0 */ 1371 EMIT4(0xb9810000, dst_reg, REG_W0); 1372 } 1373 break; 1374 /* 1375 * BPF_XOR 1376 */ 1377 case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */ 1378 /* xr %dst,%src */ 1379 EMIT2(0x1700, dst_reg, src_reg); 1380 EMIT_ZERO(dst_reg); 1381 break; 1382 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */ 1383 /* xgr %dst,%src */ 1384 EMIT4(0xb9820000, dst_reg, src_reg); 1385 break; 1386 case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */ 1387 if (imm != 0) { 1388 /* xilf %dst,imm */ 1389 EMIT6_IMM(0xc0070000, dst_reg, imm); 1390 } 1391 EMIT_ZERO(dst_reg); 1392 break; 1393 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */ 1394 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1395 /* xg %dst,<d(imm)>(%l) */ 1396 EMIT6_DISP_LH(0xe3000000, 0x0082, 1397 dst_reg, REG_0, REG_L, 1398 EMIT_CONST_U64(imm)); 1399 } else { 1400 /* lgrl %w0,imm */ 1401 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1402 _EMIT_CONST_U64(imm)); 1403 jit->seen |= SEEN_LITERAL; 1404 /* xgr %dst,%w0 */ 1405 EMIT4(0xb9820000, dst_reg, REG_W0); 1406 } 1407 break; 1408 /* 1409 * BPF_LSH 1410 */ 1411 case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << ((u32) src & 31) */ 1412 emit_shift_count32(jit, src_reg); 1413 /* sll %dst,0(%w1) */ 1414 EMIT4_DISP(0x89000000, dst_reg, REG_W1, 0); 1415 EMIT_ZERO(dst_reg); 1416 break; 1417 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ 1418 /* sllg %dst,%dst,0(%src) */ 1419 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0); 1420 break; 1421 case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */ 1422 if (imm != 0) { 1423 /* sll %dst,imm(%r0) */ 1424 EMIT4_DISP(0x89000000, dst_reg, REG_0, imm); 1425 } 1426 EMIT_ZERO(dst_reg); 1427 break; 1428 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */ 1429 if (imm == 0) 1430 break; 1431 /* sllg %dst,%dst,imm(%r0) */ 1432 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm); 1433 break; 1434 /* 1435 * BPF_RSH 1436 */ 1437 case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> ((u32) src & 31) */ 1438 emit_shift_count32(jit, src_reg); 1439 /* srl %dst,0(%w1) */ 1440 EMIT4_DISP(0x88000000, dst_reg, REG_W1, 0); 1441 EMIT_ZERO(dst_reg); 1442 break; 1443 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ 1444 /* srlg %dst,%dst,0(%src) */ 1445 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0); 1446 break; 1447 case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */ 1448 if (imm != 0) { 1449 /* srl %dst,imm(%r0) */ 1450 EMIT4_DISP(0x88000000, dst_reg, REG_0, imm); 1451 } 1452 EMIT_ZERO(dst_reg); 1453 break; 1454 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */ 1455 if (imm == 0) 1456 break; 1457 /* srlg %dst,%dst,imm(%r0) */ 1458 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm); 1459 break; 1460 /* 1461 * BPF_ARSH 1462 */ 1463 case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= (u32) src & 31 */ 1464 emit_shift_count32(jit, src_reg); 1465 /* sra %dst,0(%w1) */ 1466 EMIT4_DISP(0x8a000000, dst_reg, REG_W1, 0); 1467 EMIT_ZERO(dst_reg); 1468 break; 1469 case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ 1470 /* srag %dst,%dst,0(%src) */ 1471 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); 1472 break; 1473 case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */ 1474 if (imm != 0) { 1475 /* sra %dst,imm(%r0) */ 1476 EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm); 1477 } 1478 EMIT_ZERO(dst_reg); 1479 break; 1480 case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */ 1481 if (imm == 0) 1482 break; 1483 /* srag %dst,%dst,imm(%r0) */ 1484 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm); 1485 break; 1486 /* 1487 * BPF_NEG 1488 */ 1489 case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */ 1490 /* lcr %dst,%dst */ 1491 EMIT2(0x1300, dst_reg, dst_reg); 1492 EMIT_ZERO(dst_reg); 1493 break; 1494 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 1495 /* lcgr %dst,%dst */ 1496 EMIT4(0xb9030000, dst_reg, dst_reg); 1497 break; 1498 /* 1499 * BPF_FROM_BE/LE 1500 */ 1501 case BPF_ALU | BPF_END | BPF_FROM_BE: 1502 /* s390 is big endian, therefore only clear high order bytes */ 1503 switch (imm) { 1504 case 16: /* dst = (u16) cpu_to_be16(dst) */ 1505 /* llghr %dst,%dst */ 1506 EMIT4(0xb9850000, dst_reg, dst_reg); 1507 if (insn_is_zext(&insn[1])) 1508 insn_count = 2; 1509 break; 1510 case 32: /* dst = (u32) cpu_to_be32(dst) */ 1511 if (!fp->aux->verifier_zext) 1512 /* llgfr %dst,%dst */ 1513 EMIT4(0xb9160000, dst_reg, dst_reg); 1514 break; 1515 case 64: /* dst = (u64) cpu_to_be64(dst) */ 1516 break; 1517 } 1518 break; 1519 case BPF_ALU | BPF_END | BPF_FROM_LE: 1520 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1521 switch (imm) { 1522 case 16: /* dst = (u16) cpu_to_le16(dst) */ 1523 /* lrvr %dst,%dst */ 1524 EMIT4(0xb91f0000, dst_reg, dst_reg); 1525 /* srl %dst,16(%r0) */ 1526 EMIT4_DISP(0x88000000, dst_reg, REG_0, 16); 1527 /* llghr %dst,%dst */ 1528 EMIT4(0xb9850000, dst_reg, dst_reg); 1529 if (insn_is_zext(&insn[1])) 1530 insn_count = 2; 1531 break; 1532 case 32: /* dst = (u32) cpu_to_le32(dst) */ 1533 /* lrvr %dst,%dst */ 1534 EMIT4(0xb91f0000, dst_reg, dst_reg); 1535 if (!fp->aux->verifier_zext) 1536 /* llgfr %dst,%dst */ 1537 EMIT4(0xb9160000, dst_reg, dst_reg); 1538 break; 1539 case 64: /* dst = (u64) cpu_to_le64(dst) */ 1540 /* lrvgr %dst,%dst */ 1541 EMIT4(0xb90f0000, dst_reg, dst_reg); 1542 break; 1543 } 1544 break; 1545 /* 1546 * BPF_NOSPEC (speculation barrier) 1547 */ 1548 case BPF_ST | BPF_NOSPEC: 1549 break; 1550 /* 1551 * BPF_ST(X) 1552 */ 1553 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ 1554 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1555 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 1556 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1557 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 1558 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1559 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 1560 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1561 err = emit_stx(jit, fp, insn); 1562 if (err < 0) 1563 return err; 1564 jit->seen |= SEEN_MEM; 1565 break; 1566 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 1567 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1568 /* lhi %w0,imm */ 1569 EMIT4_IMM(0xa7080000, REG_W0, (u8) imm); 1570 bpf_jit_probe_store_pre(jit, insn, &probe); 1571 /* stcy %w0,off(%dst,%arena) */ 1572 EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, 1573 probe.arena_reg, off); 1574 err = bpf_jit_probe_post(jit, fp, &probe); 1575 if (err < 0) 1576 return err; 1577 jit->seen |= SEEN_MEM; 1578 break; 1579 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 1580 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1581 /* lhi %w0,imm */ 1582 EMIT4_IMM(0xa7080000, REG_W0, (u16) imm); 1583 bpf_jit_probe_store_pre(jit, insn, &probe); 1584 /* sthy %w0,off(%dst,%arena) */ 1585 EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, 1586 probe.arena_reg, off); 1587 err = bpf_jit_probe_post(jit, fp, &probe); 1588 if (err < 0) 1589 return err; 1590 jit->seen |= SEEN_MEM; 1591 break; 1592 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 1593 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1594 /* llilf %w0,imm */ 1595 EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm); 1596 bpf_jit_probe_store_pre(jit, insn, &probe); 1597 /* sty %w0,off(%dst,%arena) */ 1598 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, 1599 probe.arena_reg, off); 1600 err = bpf_jit_probe_post(jit, fp, &probe); 1601 if (err < 0) 1602 return err; 1603 jit->seen |= SEEN_MEM; 1604 break; 1605 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 1606 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1607 /* lgfi %w0,imm */ 1608 EMIT6_IMM(0xc0010000, REG_W0, imm); 1609 bpf_jit_probe_store_pre(jit, insn, &probe); 1610 /* stg %w0,off(%dst,%arena) */ 1611 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, 1612 probe.arena_reg, off); 1613 err = bpf_jit_probe_post(jit, fp, &probe); 1614 if (err < 0) 1615 return err; 1616 jit->seen |= SEEN_MEM; 1617 break; 1618 /* 1619 * BPF_ATOMIC 1620 */ 1621 case BPF_STX | BPF_ATOMIC | BPF_B: 1622 case BPF_STX | BPF_ATOMIC | BPF_H: 1623 case BPF_STX | BPF_ATOMIC | BPF_DW: 1624 case BPF_STX | BPF_ATOMIC | BPF_W: 1625 case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: 1626 case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: 1627 case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: 1628 case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: 1629 { 1630 bool is32 = BPF_SIZE(insn->code) == BPF_W; 1631 1632 /* 1633 * Unlike loads and stores, s390 atomics have only a base 1634 * register, but no index register. For the non-arena case, 1635 * simply use %dst as a base. For the arena case, use the 1636 * work register %r1: first, load the arena base into it, 1637 * and then add %dst to it. 1638 */ 1639 probe.arena_reg = dst_reg; 1640 1641 switch (insn->imm) { 1642 #define EMIT_ATOMIC(op32, op64) do { \ 1643 bpf_jit_probe_atomic_pre(jit, insn, &probe); \ 1644 /* {op32|op64} {%w0|%src},%src,off(%arena) */ \ 1645 EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ 1646 (insn->imm & BPF_FETCH) ? src_reg : REG_W0, \ 1647 src_reg, probe.arena_reg, off); \ 1648 err = bpf_jit_probe_post(jit, fp, &probe); \ 1649 if (err < 0) \ 1650 return err; \ 1651 if (insn->imm & BPF_FETCH) { \ 1652 /* bcr 14,0 - see atomic_fetch_{add,and,or,xor}() */ \ 1653 _EMIT2(0x07e0); \ 1654 if (is32) \ 1655 EMIT_ZERO(src_reg); \ 1656 } \ 1657 } while (0) 1658 case BPF_ADD: 1659 case BPF_ADD | BPF_FETCH: 1660 /* {laal|laalg} */ 1661 EMIT_ATOMIC(0x00fa, 0x00ea); 1662 break; 1663 case BPF_AND: 1664 case BPF_AND | BPF_FETCH: 1665 /* {lan|lang} */ 1666 EMIT_ATOMIC(0x00f4, 0x00e4); 1667 break; 1668 case BPF_OR: 1669 case BPF_OR | BPF_FETCH: 1670 /* {lao|laog} */ 1671 EMIT_ATOMIC(0x00f6, 0x00e6); 1672 break; 1673 case BPF_XOR: 1674 case BPF_XOR | BPF_FETCH: 1675 /* {lax|laxg} */ 1676 EMIT_ATOMIC(0x00f7, 0x00e7); 1677 break; 1678 #undef EMIT_ATOMIC 1679 case BPF_XCHG: { 1680 struct bpf_jit_probe load_probe = probe; 1681 int loop_start; 1682 1683 bpf_jit_probe_atomic_pre(jit, insn, &load_probe); 1684 /* {ly|lg} %w0,off(%arena) */ 1685 EMIT6_DISP_LH(0xe3000000, 1686 is32 ? 0x0058 : 0x0004, REG_W0, REG_0, 1687 load_probe.arena_reg, off); 1688 /* Reuse {ly|lg}'s arena_reg for {csy|csg}. */ 1689 if (load_probe.prg != -1) { 1690 probe.prg = jit->prg; 1691 probe.arena_reg = load_probe.arena_reg; 1692 probe.reg = load_probe.reg; 1693 } 1694 loop_start = jit->prg; 1695 /* 0: {csy|csg} %w0,%src,off(%arena) */ 1696 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1697 REG_W0, src_reg, probe.arena_reg, off); 1698 /* brc 4,0b */ 1699 EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); 1700 /* {llgfr|lgr} %src,%w0 */ 1701 EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); 1702 /* Both probes should land here on exception. */ 1703 err = bpf_jit_probe_post(jit, fp, &load_probe); 1704 if (err < 0) 1705 return err; 1706 err = bpf_jit_probe_post(jit, fp, &probe); 1707 if (err < 0) 1708 return err; 1709 if (is32 && insn_is_zext(&insn[1])) 1710 insn_count = 2; 1711 break; 1712 } 1713 case BPF_CMPXCHG: 1714 bpf_jit_probe_atomic_pre(jit, insn, &probe); 1715 /* 0: {csy|csg} %b0,%src,off(%arena) */ 1716 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1717 BPF_REG_0, src_reg, 1718 probe.arena_reg, off); 1719 err = bpf_jit_probe_post(jit, fp, &probe); 1720 if (err < 0) 1721 return err; 1722 break; 1723 case BPF_LOAD_ACQ: 1724 /* s390 has strong ordering, just use load */ 1725 err = emit_ldx(jit, fp, insn); 1726 if (err < 0) 1727 return err; 1728 break; 1729 case BPF_STORE_REL: 1730 /* s390 has strong ordering, just use store */ 1731 err = emit_stx(jit, fp, insn); 1732 if (err < 0) 1733 return err; 1734 break; 1735 default: 1736 pr_err("Unknown atomic operation %02x\n", insn->imm); 1737 return -1; 1738 } 1739 1740 jit->seen |= SEEN_MEM; 1741 break; 1742 } 1743 /* 1744 * BPF_LDX 1745 */ 1746 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 1747 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1748 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1749 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 1750 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1751 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1752 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 1753 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1754 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1755 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 1756 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1757 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1758 err = emit_ldx(jit, fp, insn); 1759 if (err < 0) 1760 return err; 1761 jit->seen |= SEEN_MEM; 1762 if (BPF_SIZE(insn->code) != BPF_DW && insn_is_zext(&insn[1])) 1763 insn_count = 2; 1764 break; 1765 case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ 1766 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1767 bpf_jit_probe_load_pre(jit, insn, &probe); 1768 /* lgb %dst,off(%src) */ 1769 EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off); 1770 err = bpf_jit_probe_post(jit, fp, &probe); 1771 if (err < 0) 1772 return err; 1773 jit->seen |= SEEN_MEM; 1774 break; 1775 case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ 1776 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1777 bpf_jit_probe_load_pre(jit, insn, &probe); 1778 /* lgh %dst,off(%src) */ 1779 EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off); 1780 err = bpf_jit_probe_post(jit, fp, &probe); 1781 if (err < 0) 1782 return err; 1783 jit->seen |= SEEN_MEM; 1784 break; 1785 case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ 1786 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1787 bpf_jit_probe_load_pre(jit, insn, &probe); 1788 /* lgf %dst,off(%src) */ 1789 jit->seen |= SEEN_MEM; 1790 EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off); 1791 err = bpf_jit_probe_post(jit, fp, &probe); 1792 if (err < 0) 1793 return err; 1794 break; 1795 /* 1796 * BPF_JMP / CALL 1797 */ 1798 case BPF_JMP | BPF_CALL: 1799 { 1800 const struct btf_func_model *m; 1801 bool func_addr_fixed; 1802 int j, ret; 1803 u64 func; 1804 1805 /* Implement helper call to bpf_get_smp_processor_id() inline */ 1806 if (insn->src_reg == 0 && 1807 insn->imm == BPF_FUNC_get_smp_processor_id) { 1808 const u32 *cpu_nr = &get_lowcore()->cpu_nr; 1809 1810 /* llgf %b0, cpu_nr */ 1811 EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0, 1812 (unsigned long)cpu_nr); 1813 break; 1814 } 1815 1816 /* Implement helper call to bpf_get_current_task/_btf() inline */ 1817 if (insn->src_reg == 0 && 1818 (insn->imm == BPF_FUNC_get_current_task || 1819 insn->imm == BPF_FUNC_get_current_task_btf)) { 1820 const u64 *current_task = 1821 &get_lowcore()->current_task; 1822 1823 /* lg %b0, current_task */ 1824 EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_0, REG_0, REG_0, 1825 (unsigned long)current_task); 1826 break; 1827 } 1828 1829 ret = bpf_jit_get_func_addr(fp, insn, extra_pass, 1830 &func, &func_addr_fixed); 1831 if (ret < 0) 1832 return -1; 1833 1834 REG_SET_SEEN(BPF_REG_5); 1835 jit->seen |= SEEN_FUNC; 1836 1837 /* 1838 * Copy the tail call counter to where the callee expects it. 1839 */ 1840 1841 if (insn->src_reg == BPF_PSEUDO_CALL) 1842 /* 1843 * mvc tail_call_cnt(4,%r15), 1844 * frame_off+tail_call_cnt(%r15) 1845 */ 1846 _EMIT6(0xd203f000 | offsetof(struct prog_frame, 1847 tail_call_cnt), 1848 0xf000 | (jit->frame_off + 1849 offsetof(struct prog_frame, 1850 tail_call_cnt))); 1851 1852 /* Sign-extend the kfunc arguments. */ 1853 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1854 m = bpf_jit_find_kfunc_model(fp, insn); 1855 if (!m) 1856 return -1; 1857 1858 for (j = 0; j < m->nr_args; j++) { 1859 if (sign_zero_extend(jit, BPF_REG_1 + j, 1860 m->arg_size[j], 1861 m->arg_flags[j])) 1862 return -1; 1863 } 1864 } 1865 1866 if ((void *)func == arch_bpf_timed_may_goto) { 1867 /* 1868 * arch_bpf_timed_may_goto() has a special ABI: the 1869 * parameters are in BPF_REG_AX and BPF_REG_10; the 1870 * return value is in BPF_REG_AX; and all GPRs except 1871 * REG_W0, REG_W1, and BPF_REG_AX are callee-saved. 1872 */ 1873 1874 /* brasl %r0,func */ 1875 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_0, (void *)func); 1876 } else { 1877 /* brasl %r14,func */ 1878 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, (void *)func); 1879 /* lgr %b0,%r2: load return value into %b0 */ 1880 EMIT4(0xb9040000, BPF_REG_0, REG_2); 1881 } 1882 1883 /* 1884 * Copy the potentially updated tail call counter back. 1885 */ 1886 1887 if (insn->src_reg == BPF_PSEUDO_CALL) 1888 /* 1889 * mvc frame_off+tail_call_cnt(%r15), 1890 * tail_call_cnt(4,%r15) 1891 */ 1892 _EMIT6(0xd203f000 | (jit->frame_off + 1893 offsetof(struct prog_frame, 1894 tail_call_cnt)), 1895 0xf000 | offsetof(struct prog_frame, 1896 tail_call_cnt)); 1897 1898 break; 1899 } 1900 case BPF_JMP | BPF_TAIL_CALL: { 1901 int patch_1_clrj, patch_2_clij, patch_3_brc; 1902 1903 /* 1904 * Implicit input: 1905 * B1: pointer to ctx 1906 * B2: pointer to bpf_array 1907 * B3: index in bpf_array 1908 * 1909 * if (index >= array->map.max_entries) 1910 * goto out; 1911 */ 1912 1913 /* llgf %w1,map.max_entries(%b2) */ 1914 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2, 1915 offsetof(struct bpf_array, map.max_entries)); 1916 /* if ((u32)%b3 >= (u32)%w1) goto out; */ 1917 /* clrj %b3,%w1,0xa,out */ 1918 patch_1_clrj = jit->prg; 1919 EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa, 1920 jit->prg); 1921 1922 /* 1923 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) 1924 * goto out; 1925 * 1926 * tail_call_cnt is read into %w0, which needs to be preserved 1927 * until it's incremented and flushed. 1928 */ 1929 1930 off = jit->frame_off + 1931 offsetof(struct prog_frame, tail_call_cnt); 1932 /* ly %w0,off(%r15) */ 1933 EMIT6_DISP_LH(0xe3000000, 0x0058, REG_W0, REG_0, REG_15, off); 1934 /* clij %w0,MAX_TAIL_CALL_CNT,0xa,out */ 1935 patch_2_clij = jit->prg; 1936 EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W0, MAX_TAIL_CALL_CNT, 1937 0xa, jit->prg); 1938 1939 /* 1940 * prog = array->ptrs[index]; 1941 * if (prog == NULL) 1942 * goto out; 1943 */ 1944 1945 /* llgfr %r1,%b3: %r1 = (u32) index */ 1946 EMIT4(0xb9160000, REG_1, BPF_REG_3); 1947 /* sllg %r1,%r1,3: %r1 *= 8 */ 1948 EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3); 1949 /* ltg %r1,prog(%b2,%r1) */ 1950 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2, 1951 REG_1, offsetof(struct bpf_array, ptrs)); 1952 /* brc 0x8,out */ 1953 patch_3_brc = jit->prg; 1954 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); 1955 1956 /* tail_call_cnt++; */ 1957 /* ahi %w0,1 */ 1958 EMIT4_IMM(0xa70a0000, REG_W0, 1); 1959 /* sty %w0,off(%r15) */ 1960 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, REG_0, REG_15, off); 1961 1962 /* 1963 * Restore registers before calling function 1964 */ 1965 save_restore_regs(jit, REGS_RESTORE, 0); 1966 1967 /* 1968 * goto *(prog->bpf_func + tail_call_start); 1969 */ 1970 1971 /* lg %r1,bpf_func(%r1) */ 1972 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0, 1973 offsetof(struct bpf_prog, bpf_func)); 1974 if (nospec_uses_trampoline()) { 1975 jit->seen |= SEEN_FUNC; 1976 /* aghi %r1,tail_call_start */ 1977 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start); 1978 /* brcl 0xf,__s390_indirect_jump_r1 */ 1979 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 1980 __s390_indirect_jump_r1); 1981 } else { 1982 /* bc 0xf,tail_call_start(%r1) */ 1983 _EMIT4(0x47f01000 + jit->tail_call_start); 1984 } 1985 /* out: */ 1986 if (jit->prg_buf) { 1987 *(u16 *)(jit->prg_buf + patch_1_clrj + 2) = 1988 (jit->prg - patch_1_clrj) >> 1; 1989 *(u16 *)(jit->prg_buf + patch_2_clij + 2) = 1990 (jit->prg - patch_2_clij) >> 1; 1991 *(u16 *)(jit->prg_buf + patch_3_brc + 2) = 1992 (jit->prg - patch_3_brc) >> 1; 1993 } 1994 break; 1995 } 1996 case BPF_JMP | BPF_EXIT: /* return b0 */ 1997 last = (i == fp->len - 1) ? 1 : 0; 1998 if (last) 1999 break; 2000 if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip)) 2001 /* brc 0xf, <exit> */ 2002 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip); 2003 else 2004 /* brcl 0xf, <exit> */ 2005 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip); 2006 break; 2007 /* 2008 * Branch relative (number of skipped instructions) to offset on 2009 * condition. 2010 * 2011 * Condition code to mask mapping: 2012 * 2013 * CC | Description | Mask 2014 * ------------------------------ 2015 * 0 | Operands equal | 8 2016 * 1 | First operand low | 4 2017 * 2 | First operand high | 2 2018 * 3 | Unused | 1 2019 * 2020 * For s390x relative branches: ip = ip + off_bytes 2021 * For BPF relative branches: insn = insn + off_insns + 1 2022 * 2023 * For example for s390x with offset 0 we jump to the branch 2024 * instruction itself (loop) and for BPF with offset 0 we 2025 * branch to the instruction behind the branch. 2026 */ 2027 case BPF_JMP32 | BPF_JA: /* if (true) */ 2028 branch_oc_off = imm; 2029 fallthrough; 2030 case BPF_JMP | BPF_JA: /* if (true) */ 2031 mask = 0xf000; /* j */ 2032 goto branch_oc; 2033 case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */ 2034 case BPF_JMP32 | BPF_JSGT | BPF_K: /* ((s32) dst > (s32) imm) */ 2035 mask = 0x2000; /* jh */ 2036 goto branch_ks; 2037 case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */ 2038 case BPF_JMP32 | BPF_JSLT | BPF_K: /* ((s32) dst < (s32) imm) */ 2039 mask = 0x4000; /* jl */ 2040 goto branch_ks; 2041 case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */ 2042 case BPF_JMP32 | BPF_JSGE | BPF_K: /* ((s32) dst >= (s32) imm) */ 2043 mask = 0xa000; /* jhe */ 2044 goto branch_ks; 2045 case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */ 2046 case BPF_JMP32 | BPF_JSLE | BPF_K: /* ((s32) dst <= (s32) imm) */ 2047 mask = 0xc000; /* jle */ 2048 goto branch_ks; 2049 case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */ 2050 case BPF_JMP32 | BPF_JGT | BPF_K: /* ((u32) dst_reg > (u32) imm) */ 2051 mask = 0x2000; /* jh */ 2052 goto branch_ku; 2053 case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */ 2054 case BPF_JMP32 | BPF_JLT | BPF_K: /* ((u32) dst_reg < (u32) imm) */ 2055 mask = 0x4000; /* jl */ 2056 goto branch_ku; 2057 case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */ 2058 case BPF_JMP32 | BPF_JGE | BPF_K: /* ((u32) dst_reg >= (u32) imm) */ 2059 mask = 0xa000; /* jhe */ 2060 goto branch_ku; 2061 case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */ 2062 case BPF_JMP32 | BPF_JLE | BPF_K: /* ((u32) dst_reg <= (u32) imm) */ 2063 mask = 0xc000; /* jle */ 2064 goto branch_ku; 2065 case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */ 2066 case BPF_JMP32 | BPF_JNE | BPF_K: /* ((u32) dst_reg != (u32) imm) */ 2067 mask = 0x7000; /* jne */ 2068 goto branch_ku; 2069 case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */ 2070 case BPF_JMP32 | BPF_JEQ | BPF_K: /* ((u32) dst_reg == (u32) imm) */ 2071 mask = 0x8000; /* je */ 2072 goto branch_ku; 2073 case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */ 2074 case BPF_JMP32 | BPF_JSET | BPF_K: /* ((u32) dst_reg & (u32) imm) */ 2075 mask = 0x7000; /* jnz */ 2076 if (BPF_CLASS(insn->code) == BPF_JMP32) { 2077 /* llilf %w1,imm (load zero extend imm) */ 2078 EMIT6_IMM(0xc00f0000, REG_W1, imm); 2079 /* nr %w1,%dst */ 2080 EMIT2(0x1400, REG_W1, dst_reg); 2081 } else { 2082 /* lgfi %w1,imm (load sign extend imm) */ 2083 EMIT6_IMM(0xc0010000, REG_W1, imm); 2084 /* ngr %w1,%dst */ 2085 EMIT4(0xb9800000, REG_W1, dst_reg); 2086 } 2087 goto branch_oc; 2088 2089 case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */ 2090 case BPF_JMP32 | BPF_JSGT | BPF_X: /* ((s32) dst > (s32) src) */ 2091 mask = 0x2000; /* jh */ 2092 goto branch_xs; 2093 case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */ 2094 case BPF_JMP32 | BPF_JSLT | BPF_X: /* ((s32) dst < (s32) src) */ 2095 mask = 0x4000; /* jl */ 2096 goto branch_xs; 2097 case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */ 2098 case BPF_JMP32 | BPF_JSGE | BPF_X: /* ((s32) dst >= (s32) src) */ 2099 mask = 0xa000; /* jhe */ 2100 goto branch_xs; 2101 case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */ 2102 case BPF_JMP32 | BPF_JSLE | BPF_X: /* ((s32) dst <= (s32) src) */ 2103 mask = 0xc000; /* jle */ 2104 goto branch_xs; 2105 case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */ 2106 case BPF_JMP32 | BPF_JGT | BPF_X: /* ((u32) dst > (u32) src) */ 2107 mask = 0x2000; /* jh */ 2108 goto branch_xu; 2109 case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */ 2110 case BPF_JMP32 | BPF_JLT | BPF_X: /* ((u32) dst < (u32) src) */ 2111 mask = 0x4000; /* jl */ 2112 goto branch_xu; 2113 case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */ 2114 case BPF_JMP32 | BPF_JGE | BPF_X: /* ((u32) dst >= (u32) src) */ 2115 mask = 0xa000; /* jhe */ 2116 goto branch_xu; 2117 case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */ 2118 case BPF_JMP32 | BPF_JLE | BPF_X: /* ((u32) dst <= (u32) src) */ 2119 mask = 0xc000; /* jle */ 2120 goto branch_xu; 2121 case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */ 2122 case BPF_JMP32 | BPF_JNE | BPF_X: /* ((u32) dst != (u32) src) */ 2123 mask = 0x7000; /* jne */ 2124 goto branch_xu; 2125 case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */ 2126 case BPF_JMP32 | BPF_JEQ | BPF_X: /* ((u32) dst == (u32) src) */ 2127 mask = 0x8000; /* je */ 2128 goto branch_xu; 2129 case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */ 2130 case BPF_JMP32 | BPF_JSET | BPF_X: /* ((u32) dst & (u32) src) */ 2131 { 2132 bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2133 2134 mask = 0x7000; /* jnz */ 2135 /* nrk or ngrk %w1,%dst,%src */ 2136 EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000), 2137 REG_W1, dst_reg, src_reg); 2138 goto branch_oc; 2139 branch_ks: 2140 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2141 /* cfi or cgfi %dst,imm */ 2142 EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000, 2143 dst_reg, imm); 2144 if (!is_first_pass(jit) && 2145 can_use_rel(jit, addrs[i + off + 1])) { 2146 /* brc mask,off */ 2147 EMIT4_PCREL_RIC(0xa7040000, 2148 mask >> 12, addrs[i + off + 1]); 2149 } else { 2150 /* brcl mask,off */ 2151 EMIT6_PCREL_RILC(0xc0040000, 2152 mask >> 12, addrs[i + off + 1]); 2153 } 2154 break; 2155 branch_ku: 2156 /* lgfi %w1,imm (load sign extend imm) */ 2157 src_reg = REG_1; 2158 EMIT6_IMM(0xc0010000, src_reg, imm); 2159 goto branch_xu; 2160 branch_xs: 2161 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2162 if (!is_first_pass(jit) && 2163 can_use_rel(jit, addrs[i + off + 1])) { 2164 /* crj or cgrj %dst,%src,mask,off */ 2165 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064), 2166 dst_reg, src_reg, i, off, mask); 2167 } else { 2168 /* cr or cgr %dst,%src */ 2169 if (is_jmp32) 2170 EMIT2(0x1900, dst_reg, src_reg); 2171 else 2172 EMIT4(0xb9200000, dst_reg, src_reg); 2173 /* brcl mask,off */ 2174 EMIT6_PCREL_RILC(0xc0040000, 2175 mask >> 12, addrs[i + off + 1]); 2176 } 2177 break; 2178 branch_xu: 2179 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2180 if (!is_first_pass(jit) && 2181 can_use_rel(jit, addrs[i + off + 1])) { 2182 /* clrj or clgrj %dst,%src,mask,off */ 2183 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065), 2184 dst_reg, src_reg, i, off, mask); 2185 } else { 2186 /* clr or clgr %dst,%src */ 2187 if (is_jmp32) 2188 EMIT2(0x1500, dst_reg, src_reg); 2189 else 2190 EMIT4(0xb9210000, dst_reg, src_reg); 2191 /* brcl mask,off */ 2192 EMIT6_PCREL_RILC(0xc0040000, 2193 mask >> 12, addrs[i + off + 1]); 2194 } 2195 break; 2196 branch_oc: 2197 if (!is_first_pass(jit) && 2198 can_use_rel(jit, addrs[i + branch_oc_off + 1])) { 2199 /* brc mask,off */ 2200 EMIT4_PCREL_RIC(0xa7040000, 2201 mask >> 12, 2202 addrs[i + branch_oc_off + 1]); 2203 } else { 2204 /* brcl mask,off */ 2205 EMIT6_PCREL_RILC(0xc0040000, 2206 mask >> 12, 2207 addrs[i + branch_oc_off + 1]); 2208 } 2209 break; 2210 } 2211 default: /* too complex, give up */ 2212 pr_err("Unknown opcode %02x\n", insn->code); 2213 return -1; 2214 } 2215 2216 return insn_count; 2217 } 2218 2219 /* 2220 * Return whether new i-th instruction address does not violate any invariant 2221 */ 2222 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i) 2223 { 2224 /* On the first pass anything goes */ 2225 if (is_first_pass(jit)) 2226 return true; 2227 2228 /* The codegen pass must not change anything */ 2229 if (is_codegen_pass(jit)) 2230 return jit->addrs[i] == jit->prg; 2231 2232 /* Passes in between must not increase code size */ 2233 return jit->addrs[i] >= jit->prg; 2234 } 2235 2236 /* 2237 * Update the address of i-th instruction 2238 */ 2239 static int bpf_set_addr(struct bpf_jit *jit, int i) 2240 { 2241 int delta; 2242 2243 if (is_codegen_pass(jit)) { 2244 delta = jit->prg - jit->addrs[i]; 2245 if (delta < 0) 2246 bpf_skip(jit, -delta); 2247 } 2248 if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i))) 2249 return -1; 2250 jit->addrs[i] = jit->prg; 2251 return 0; 2252 } 2253 2254 /* 2255 * Compile eBPF program into s390x code 2256 */ 2257 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, 2258 bool extra_pass) 2259 { 2260 int i, insn_count, lit32_size, lit64_size; 2261 u64 kern_arena; 2262 2263 jit->lit32 = jit->lit32_start; 2264 jit->lit64 = jit->lit64_start; 2265 jit->prg = 0; 2266 jit->excnt = 0; 2267 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) 2268 jit->frame_off = sizeof(struct prog_frame) - 2269 offsetofend(struct prog_frame, unused) + 2270 round_up(fp->aux->stack_depth, 8); 2271 else 2272 jit->frame_off = 0; 2273 2274 if (bpf_prog_uses_arena_insns(fp)) { 2275 kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); 2276 jit->kern_arena = _EMIT_CONST_U64(kern_arena); 2277 jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); 2278 } 2279 2280 bpf_jit_prologue(jit, fp); 2281 if (bpf_set_addr(jit, 0) < 0) 2282 return -1; 2283 for (i = 0; i < fp->len; i += insn_count) { 2284 insn_count = bpf_jit_insn(jit, fp, i, extra_pass); 2285 if (insn_count < 0) 2286 return -1; 2287 /* Next instruction address */ 2288 if (bpf_set_addr(jit, i + insn_count) < 0) 2289 return -1; 2290 } 2291 bpf_jit_epilogue(jit); 2292 2293 lit32_size = jit->lit32 - jit->lit32_start; 2294 lit64_size = jit->lit64 - jit->lit64_start; 2295 jit->lit32_start = jit->prg; 2296 if (lit32_size) 2297 jit->lit32_start = ALIGN(jit->lit32_start, 4); 2298 jit->lit64_start = jit->lit32_start + lit32_size; 2299 if (lit64_size) 2300 jit->lit64_start = ALIGN(jit->lit64_start, 8); 2301 jit->size = jit->lit64_start + lit64_size; 2302 jit->size_prg = jit->prg; 2303 2304 if (WARN_ON_ONCE(fp->aux->extable && 2305 jit->excnt != fp->aux->num_exentries)) 2306 /* Verifier bug - too many entries. */ 2307 return -1; 2308 2309 return 0; 2310 } 2311 2312 bool bpf_jit_needs_zext(void) 2313 { 2314 return true; 2315 } 2316 2317 struct s390_jit_data { 2318 struct bpf_binary_header *header; 2319 struct bpf_jit ctx; 2320 int pass; 2321 }; 2322 2323 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, 2324 struct bpf_prog *fp) 2325 { 2326 struct bpf_binary_header *header; 2327 struct bpf_insn *insn; 2328 u32 extable_size; 2329 u32 code_size; 2330 int i; 2331 2332 for (i = 0; i < fp->len; i++) { 2333 insn = &fp->insnsi[i]; 2334 2335 if (BPF_CLASS(insn->code) == BPF_STX && 2336 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC && 2337 (BPF_SIZE(insn->code) == BPF_DW || 2338 BPF_SIZE(insn->code) == BPF_W) && 2339 insn->imm == BPF_XCHG) 2340 /* 2341 * bpf_jit_insn() emits a load and a compare-and-swap, 2342 * both of which need to be probed. 2343 */ 2344 fp->aux->num_exentries += 1; 2345 } 2346 code_size = roundup(jit->size, 2347 __alignof__(struct exception_table_entry)); 2348 extable_size = fp->aux->num_exentries * 2349 sizeof(struct exception_table_entry); 2350 header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf, 2351 8, jit_fill_hole); 2352 if (!header) 2353 return NULL; 2354 fp->aux->extable = (struct exception_table_entry *) 2355 (jit->prg_buf + code_size); 2356 return header; 2357 } 2358 2359 /* 2360 * Compile eBPF program "fp" 2361 */ 2362 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 2363 { 2364 struct bpf_binary_header *header; 2365 struct s390_jit_data *jit_data; 2366 bool extra_pass = false; 2367 struct bpf_jit jit; 2368 int pass; 2369 2370 if (!fp->jit_requested) 2371 return fp; 2372 2373 jit_data = fp->aux->jit_data; 2374 if (!jit_data) { 2375 jit_data = kzalloc_obj(*jit_data); 2376 if (!jit_data) 2377 return fp; 2378 fp->aux->jit_data = jit_data; 2379 } 2380 if (jit_data->ctx.addrs) { 2381 jit = jit_data->ctx; 2382 header = jit_data->header; 2383 extra_pass = true; 2384 pass = jit_data->pass + 1; 2385 goto skip_init_ctx; 2386 } 2387 2388 memset(&jit, 0, sizeof(jit)); 2389 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); 2390 if (jit.addrs == NULL) 2391 goto out_err; 2392 /* 2393 * Three initial passes: 2394 * - 1/2: Determine clobbered registers 2395 * - 3: Calculate program size and addrs array 2396 */ 2397 for (pass = 1; pass <= 3; pass++) { 2398 if (bpf_jit_prog(&jit, fp, extra_pass)) 2399 goto out_err; 2400 } 2401 /* 2402 * Final pass: Allocate and generate program 2403 */ 2404 header = bpf_jit_alloc(&jit, fp); 2405 if (!header) 2406 goto out_err; 2407 skip_init_ctx: 2408 if (bpf_jit_prog(&jit, fp, extra_pass)) { 2409 bpf_jit_binary_free(header); 2410 goto out_err; 2411 } 2412 if (bpf_jit_enable > 1) { 2413 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); 2414 print_fn_code(jit.prg_buf, jit.size_prg); 2415 } 2416 if (!fp->is_func || extra_pass) { 2417 if (bpf_jit_binary_lock_ro(header)) { 2418 bpf_jit_binary_free(header); 2419 goto out_err; 2420 } 2421 } else { 2422 jit_data->header = header; 2423 jit_data->ctx = jit; 2424 jit_data->pass = pass; 2425 } 2426 fp->bpf_func = (void *)jit.prg_buf + cfi_get_offset(); 2427 fp->jited = 1; 2428 fp->jited_len = jit.size - cfi_get_offset(); 2429 2430 if (!fp->is_func || extra_pass) { 2431 for (int i = 0; i < fp->len; i++) 2432 jit.addrs[i] -= cfi_get_offset(); 2433 bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); 2434 free_addrs: 2435 kvfree(jit.addrs); 2436 kfree(jit_data); 2437 fp->aux->jit_data = NULL; 2438 } 2439 2440 return fp; 2441 2442 out_err: 2443 if (extra_pass) { 2444 fp->bpf_func = NULL; 2445 fp->jited = 0; 2446 fp->jited_len = 0; 2447 } 2448 goto free_addrs; 2449 } 2450 2451 bool bpf_jit_supports_kfunc_call(void) 2452 { 2453 return true; 2454 } 2455 2456 bool bpf_jit_supports_far_kfunc_call(void) 2457 { 2458 return true; 2459 } 2460 2461 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 2462 enum bpf_text_poke_type new_t, void *old_addr, 2463 void *new_addr) 2464 { 2465 struct bpf_plt expected_plt, current_plt, new_plt, *plt; 2466 struct { 2467 u16 opc; 2468 s32 disp; 2469 } __packed insn; 2470 char *ret; 2471 int err; 2472 2473 /* Verify the branch to be patched. */ 2474 err = copy_from_kernel_nofault(&insn, ip, sizeof(insn)); 2475 if (err < 0) 2476 return err; 2477 if (insn.opc != (0xc004 | (old_addr ? 0xf0 : 0))) 2478 return -EINVAL; 2479 2480 if ((new_t == BPF_MOD_JUMP || old_t == BPF_MOD_JUMP) && 2481 insn.disp == ((char *)new_addr - (char *)ip) >> 1) { 2482 /* 2483 * The branch already points to the destination, 2484 * there is no PLT. 2485 */ 2486 } else { 2487 /* Verify the PLT. */ 2488 plt = ip + (insn.disp << 1); 2489 err = copy_from_kernel_nofault(¤t_plt, plt, 2490 sizeof(current_plt)); 2491 if (err < 0) 2492 return err; 2493 ret = (char *)ip + 6; 2494 bpf_jit_plt(&expected_plt, ret, old_addr); 2495 if (memcmp(¤t_plt, &expected_plt, sizeof(current_plt))) 2496 return -EINVAL; 2497 /* Adjust the call address. */ 2498 bpf_jit_plt(&new_plt, ret, new_addr); 2499 s390_kernel_write(&plt->target, &new_plt.target, 2500 sizeof(void *)); 2501 } 2502 2503 /* Adjust the mask of the branch. */ 2504 insn.opc = 0xc004 | (new_addr ? 0xf0 : 0); 2505 s390_kernel_write((char *)ip + 1, (char *)&insn.opc + 1, 1); 2506 2507 /* Make the new code visible to the other CPUs. */ 2508 text_poke_sync_lock(); 2509 2510 return 0; 2511 } 2512 2513 struct bpf_tramp_jit { 2514 struct bpf_jit common; 2515 int orig_stack_args_off;/* Offset of arguments placed on stack by the 2516 * func_addr's original caller 2517 */ 2518 int stack_size; /* Trampoline stack size */ 2519 int backchain_off; /* Offset of backchain */ 2520 int stack_args_off; /* Offset of stack arguments for calling 2521 * func_addr, has to be at the top 2522 */ 2523 int reg_args_off; /* Offset of register arguments for calling 2524 * func_addr 2525 */ 2526 int ip_off; /* For bpf_get_func_ip(), has to be at 2527 * (ctx - 16) 2528 */ 2529 int func_meta_off; /* For bpf_get_func_arg_cnt()/fsession, has 2530 * to be at (ctx - 8) 2531 */ 2532 int bpf_args_off; /* Offset of BPF_PROG context, which consists 2533 * of BPF arguments followed by return value 2534 */ 2535 int retval_off; /* Offset of return value (see above) */ 2536 int r7_r8_off; /* Offset of saved %r7 and %r8, which are used 2537 * for __bpf_prog_enter() return value and 2538 * func_addr respectively 2539 */ 2540 int run_ctx_off; /* Offset of struct bpf_tramp_run_ctx */ 2541 int tccnt_off; /* Offset of saved tailcall counter */ 2542 int r14_off; /* Offset of saved %r14, has to be at the 2543 * bottom */ 2544 int do_fexit; /* do_fexit: label */ 2545 int skip[BPF_MAX_TRAMP_LINKS]; /* skip: labels after each prog */ 2546 int nr_progs; 2547 }; 2548 2549 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) 2550 { 2551 /* llihf %dst_reg,val_hi */ 2552 EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32)); 2553 /* oilf %rdst_reg,val_lo */ 2554 EMIT6_IMM(0xc00d0000, dst_reg, val); 2555 } 2556 2557 static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm) 2558 { 2559 load_imm64(jit, tmp_reg, imm); 2560 /* stg %tmp_reg,stack_off(%r15) */ 2561 EMIT6_DISP_LH(0xe3000000, 0x0024, tmp_reg, REG_0, REG_15, stack_off); 2562 } 2563 2564 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, 2565 struct bpf_tramp_image *im, 2566 const struct btf_func_model *m, 2567 struct bpf_tramp_node *node, bool save_ret) 2568 { 2569 struct bpf_jit *jit = &tjit->common; 2570 int cookie_off = tjit->run_ctx_off + 2571 offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 2572 struct bpf_prog *p = node->link->prog; 2573 void *skip = jit->prg_buf + jit->prg; 2574 int idx = tjit->nr_progs++; 2575 int patch; 2576 2577 if (idx >= ARRAY_SIZE(tjit->skip)) 2578 return -E2BIG; 2579 2580 /* 2581 * nop, patched to skip this prog when it is detached 2582 */ 2583 2584 /* brcl 0,skip */ 2585 EMIT6_PCREL_RILC(0xc0040000, 0, tjit->skip[idx]); 2586 2587 /* 2588 * run_ctx.cookie = node->cookie; 2589 */ 2590 2591 emit_store_stack_imm64(jit, REG_W0, cookie_off, node->cookie); 2592 2593 /* 2594 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) 2595 * goto skip; 2596 */ 2597 2598 /* %r2 = p */ 2599 load_imm64(jit, REG_2, (u64)p); 2600 /* la %r3,run_ctx_off(%r15) */ 2601 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); 2602 /* brasl %r14,__bpf_prog_enter */ 2603 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_enter(p)); 2604 /* ltgr %r7,%r2 */ 2605 EMIT4(0xb9020000, REG_7, REG_2); 2606 /* brcl 8,skip */ 2607 patch = jit->prg; 2608 EMIT6_PCREL_RILC(0xc0040000, 8, 0); 2609 2610 /* 2611 * retval = bpf_func(args, p->insnsi); 2612 */ 2613 2614 /* la %r2,bpf_args_off(%r15) */ 2615 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); 2616 /* %r3 = p->insnsi */ 2617 if (!p->jited) 2618 load_imm64(jit, REG_3, (u64)p->insnsi); 2619 /* brasl %r14,p->bpf_func */ 2620 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func); 2621 /* stg %r2,retval_off(%r15) */ 2622 if (save_ret) { 2623 if (sign_zero_extend(jit, REG_2, m->ret_size, m->ret_flags)) 2624 return -1; 2625 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2626 tjit->retval_off); 2627 } 2628 2629 /* skip: */ 2630 if (jit->prg_buf) 2631 *(u32 *)&jit->prg_buf[patch + 2] = (jit->prg - patch) >> 1; 2632 2633 /* 2634 * __bpf_prog_exit(p, start, &run_ctx); 2635 */ 2636 2637 /* %r2 = p */ 2638 load_imm64(jit, REG_2, (u64)p); 2639 /* lgr %r3,%r7 */ 2640 EMIT4(0xb9040000, REG_3, REG_7); 2641 /* la %r4,run_ctx_off(%r15) */ 2642 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); 2643 /* brasl %r14,__bpf_prog_exit */ 2644 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p)); 2645 2646 /* skip: */ 2647 tjit->skip[idx] = jit->prg; 2648 bpf_tramp_image_add_skip(im, p, skip, jit->prg_buf + jit->prg); 2649 2650 return 0; 2651 } 2652 2653 static int invoke_bpf(struct bpf_tramp_jit *tjit, 2654 struct bpf_tramp_image *im, 2655 const struct btf_func_model *m, 2656 struct bpf_tramp_nodes *tn, bool save_ret, 2657 u64 func_meta, int cookie_off) 2658 { 2659 int i, cur_cookie = (tjit->bpf_args_off - cookie_off) / sizeof(u64); 2660 struct bpf_jit *jit = &tjit->common; 2661 2662 for (i = 0; i < tn->nr_nodes; i++) { 2663 if (bpf_prog_calls_session_cookie(tn->nodes[i])) { 2664 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); 2665 2666 emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); 2667 cur_cookie--; 2668 } 2669 if (invoke_bpf_prog(tjit, im, m, tn->nodes[i], save_ret)) 2670 return -EINVAL; 2671 } 2672 2673 return 0; 2674 } 2675 2676 static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size) 2677 { 2678 int stack_offset = tjit->stack_size; 2679 2680 tjit->stack_size += size; 2681 return stack_offset; 2682 } 2683 2684 /* ABI uses %r2 - %r6 for parameter passing. */ 2685 #define MAX_NR_REG_ARGS 5 2686 2687 /* The "L" field of the "mvc" instruction is 8 bits. */ 2688 #define MAX_MVC_SIZE 256 2689 #define MAX_NR_STACK_ARGS (MAX_MVC_SIZE / sizeof(u64)) 2690 2691 /* -mfentry generates a 6-byte nop on s390x. */ 2692 #define S390X_PATCH_SIZE 6 2693 2694 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 2695 struct bpf_tramp_jit *tjit, 2696 const struct btf_func_model *m, 2697 u32 flags, 2698 struct bpf_tramp_nodes *tnodes, 2699 void *func_addr) 2700 { 2701 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 2702 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 2703 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 2704 int nr_bpf_args, nr_reg_args, nr_stack_args; 2705 int cookie_cnt, cookie_off, fsession_cnt; 2706 struct bpf_jit *jit = &tjit->common; 2707 int arg, bpf_arg_off; 2708 u64 func_meta; 2709 int i, j; 2710 2711 /* The skip labels are taken from the previous pass. */ 2712 tjit->nr_progs = 0; 2713 if (im) 2714 im->nr_skips = 0; 2715 2716 /* Support as many stack arguments as "mvc" instruction can handle. */ 2717 nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS); 2718 nr_stack_args = m->nr_args - nr_reg_args; 2719 if (nr_stack_args > MAX_NR_STACK_ARGS) 2720 return -ENOTSUPP; 2721 2722 /* Return to %r14 in the struct_ops case. */ 2723 if (flags & BPF_TRAMP_F_INDIRECT) { 2724 flags |= BPF_TRAMP_F_SKIP_FRAME; 2725 emit_kcfi(cfi_get_func_hash(func_addr), jit); 2726 } 2727 2728 /* 2729 * Compute how many arguments we need to pass to BPF programs. 2730 * BPF ABI mirrors that of x86_64: arguments that are 16 bytes or 2731 * smaller are packed into 1 or 2 registers; larger arguments are 2732 * passed via pointers. 2733 * In s390x ABI, arguments that are 8 bytes or smaller are packed into 2734 * a register; larger arguments are passed via pointers. 2735 * We need to deal with this difference. 2736 */ 2737 nr_bpf_args = 0; 2738 for (i = 0; i < m->nr_args; i++) { 2739 if (m->arg_size[i] <= 8) 2740 nr_bpf_args += 1; 2741 else if (m->arg_size[i] <= 16) 2742 nr_bpf_args += 2; 2743 else 2744 return -ENOTSUPP; 2745 } 2746 2747 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 2748 fsession_cnt = bpf_fsession_cnt(tnodes); 2749 2750 /* 2751 * Calculate the stack layout. 2752 */ 2753 2754 /* 2755 * Allocate STACK_FRAME_OVERHEAD bytes for the callees. As the s390x 2756 * ABI requires, put our backchain at the end of the allocated memory. 2757 */ 2758 tjit->stack_size = STACK_FRAME_OVERHEAD; 2759 tjit->backchain_off = tjit->stack_size - sizeof(u64); 2760 tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64)); 2761 tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64)); 2762 cookie_off = alloc_stack(tjit, cookie_cnt * sizeof(u64)); 2763 tjit->ip_off = alloc_stack(tjit, sizeof(u64)); 2764 tjit->func_meta_off = alloc_stack(tjit, sizeof(u64)); 2765 tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64)); 2766 tjit->retval_off = alloc_stack(tjit, sizeof(u64)); 2767 tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64)); 2768 tjit->run_ctx_off = alloc_stack(tjit, 2769 sizeof(struct bpf_tramp_run_ctx)); 2770 tjit->tccnt_off = alloc_stack(tjit, sizeof(u64)); 2771 tjit->r14_off = alloc_stack(tjit, sizeof(u64) * 2); 2772 /* 2773 * In accordance with the s390x ABI, the caller has allocated 2774 * STACK_FRAME_OVERHEAD bytes for us. 8 of them contain the caller's 2775 * backchain, and the rest we can use. 2776 */ 2777 tjit->stack_size -= STACK_FRAME_OVERHEAD - sizeof(u64); 2778 tjit->orig_stack_args_off = tjit->stack_size + STACK_FRAME_OVERHEAD; 2779 2780 /* lgr %r1,%r15 */ 2781 EMIT4(0xb9040000, REG_1, REG_15); 2782 /* aghi %r15,-stack_size */ 2783 EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size); 2784 /* stg %r1,backchain_off(%r15) */ 2785 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15, 2786 tjit->backchain_off); 2787 /* mvc tccnt_off(4,%r15),stack_size+tail_call_cnt(%r15) */ 2788 _EMIT6(0xd203f000 | tjit->tccnt_off, 2789 0xf000 | (tjit->stack_size + 2790 offsetof(struct prog_frame, tail_call_cnt))); 2791 /* stmg %r2,%rN,fwd_reg_args_off(%r15) */ 2792 if (nr_reg_args) 2793 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2, 2794 REG_2 + (nr_reg_args - 1), REG_15, 2795 tjit->reg_args_off); 2796 for (i = 0, j = 0; i < m->nr_args; i++) { 2797 if (i < MAX_NR_REG_ARGS) 2798 arg = REG_2 + i; 2799 else 2800 arg = tjit->orig_stack_args_off + 2801 (i - MAX_NR_REG_ARGS) * sizeof(u64); 2802 bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64); 2803 if (m->arg_size[i] <= 8) { 2804 if (i < MAX_NR_REG_ARGS) 2805 /* stg %arg,bpf_arg_off(%r15) */ 2806 EMIT6_DISP_LH(0xe3000000, 0x0024, arg, 2807 REG_0, REG_15, bpf_arg_off); 2808 else 2809 /* mvc bpf_arg_off(8,%r15),arg(%r15) */ 2810 _EMIT6(0xd207f000 | bpf_arg_off, 2811 0xf000 | arg); 2812 j += 1; 2813 } else { 2814 if (i < MAX_NR_REG_ARGS) { 2815 /* mvc bpf_arg_off(16,%r15),0(%arg) */ 2816 _EMIT6(0xd20ff000 | bpf_arg_off, 2817 reg2hex[arg] << 12); 2818 } else { 2819 /* lg %r1,arg(%r15) */ 2820 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_0, 2821 REG_15, arg); 2822 /* mvc bpf_arg_off(16,%r15),0(%r1) */ 2823 _EMIT6(0xd20ff000 | bpf_arg_off, 0x1000); 2824 } 2825 j += 2; 2826 } 2827 } 2828 /* stmg %r7,%r8,r7_r8_off(%r15) */ 2829 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_7, REG_8, REG_15, 2830 tjit->r7_r8_off); 2831 /* stg %r14,r14_off(%r15) */ 2832 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off); 2833 2834 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2835 /* 2836 * The ftrace trampoline puts the return address (which is the 2837 * address of the original function + S390X_PATCH_SIZE) into 2838 * %r0; see ftrace_shared_hotpatch_trampoline_br and 2839 * ftrace_init_nop() for details. 2840 */ 2841 2842 /* lgr %r8,%r0 */ 2843 EMIT4(0xb9040000, REG_8, REG_0); 2844 } 2845 2846 /* 2847 * ip = func_addr; 2848 * arg_cnt = m->nr_args; 2849 */ 2850 2851 if (flags & BPF_TRAMP_F_IP_ARG) 2852 emit_store_stack_imm64(jit, REG_0, tjit->ip_off, (u64)func_addr); 2853 func_meta = nr_bpf_args; 2854 /* lghi %r0,func_meta */ 2855 EMIT4_IMM(0xa7090000, REG_0, func_meta); 2856 /* stg %r0,func_meta_off(%r15) */ 2857 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, 2858 tjit->func_meta_off); 2859 2860 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2861 /* 2862 * __bpf_tramp_enter(im); 2863 */ 2864 2865 /* %r2 = im */ 2866 load_imm64(jit, REG_2, (u64)im); 2867 /* brasl %r14,__bpf_tramp_enter */ 2868 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter); 2869 } 2870 2871 if (fsession_cnt) { 2872 /* Clear all the session cookies' value. */ 2873 for (i = 0; i < cookie_cnt; i++) 2874 emit_store_stack_imm64(jit, REG_0, cookie_off + 8 * i, 0); 2875 /* Clear the return value to make sure fentry always gets 0. */ 2876 emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); 2877 } 2878 2879 if (invoke_bpf(tjit, im, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, 2880 func_meta, cookie_off)) 2881 return -EINVAL; 2882 2883 if (fmod_ret->nr_nodes) { 2884 /* 2885 * retval = 0; 2886 */ 2887 2888 /* xc retval_off(8,%r15),retval_off(%r15) */ 2889 _EMIT6(0xd707f000 | tjit->retval_off, 2890 0xf000 | tjit->retval_off); 2891 2892 for (i = 0; i < fmod_ret->nr_nodes; i++) { 2893 if (invoke_bpf_prog(tjit, im, m, fmod_ret->nodes[i], true)) 2894 return -EINVAL; 2895 2896 /* 2897 * if (retval) 2898 * goto do_fexit; 2899 */ 2900 2901 /* ltg %r0,retval_off(%r15) */ 2902 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_0, REG_0, REG_15, 2903 tjit->retval_off); 2904 /* brcl 7,do_fexit */ 2905 EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit); 2906 } 2907 } 2908 2909 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2910 /* 2911 * retval = func_addr(args); 2912 */ 2913 2914 /* lmg %r2,%rN,reg_args_off(%r15) */ 2915 if (nr_reg_args) 2916 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2917 REG_2 + (nr_reg_args - 1), REG_15, 2918 tjit->reg_args_off); 2919 /* mvc stack_args_off(N,%r15),orig_stack_args_off(%r15) */ 2920 if (nr_stack_args) 2921 _EMIT6(0xd200f000 | 2922 (nr_stack_args * sizeof(u64) - 1) << 16 | 2923 tjit->stack_args_off, 2924 0xf000 | tjit->orig_stack_args_off); 2925 /* mvc tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2926 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), 2927 0xf000 | tjit->tccnt_off); 2928 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2929 if (nospec_uses_trampoline()) 2930 /* brasl %r14,__s390_indirect_jump_r8 */ 2931 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2932 __s390_indirect_jump_r8); 2933 else 2934 /* basr %r14,%r8 */ 2935 EMIT2(0x0d00, REG_14, REG_8); 2936 } else { 2937 /* brasl %r14,func_addr+S390X_PATCH_SIZE */ 2938 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2939 func_addr + S390X_PATCH_SIZE); 2940 } 2941 /* stg %r2,retval_off(%r15) */ 2942 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2943 tjit->retval_off); 2944 /* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */ 2945 _EMIT6(0xd203f000 | tjit->tccnt_off, 2946 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); 2947 } 2948 2949 /* Set the "is_return" flag for fsession. */ 2950 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 2951 if (fsession_cnt) 2952 emit_store_stack_imm64(jit, REG_W0, tjit->func_meta_off, 2953 func_meta); 2954 2955 /* do_fexit: */ 2956 tjit->do_fexit = jit->prg; 2957 if (invoke_bpf(tjit, im, m, fexit, false, func_meta, cookie_off)) 2958 return -EINVAL; 2959 2960 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2961 /* 2962 * __bpf_tramp_exit(im); 2963 */ 2964 2965 /* %r2 = im */ 2966 load_imm64(jit, REG_2, (u64)im); 2967 /* brasl %r14,__bpf_tramp_exit */ 2968 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_exit); 2969 } 2970 2971 /* lmg %r2,%rN,reg_args_off(%r15) */ 2972 if ((flags & BPF_TRAMP_F_RESTORE_REGS) && nr_reg_args) 2973 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2974 REG_2 + (nr_reg_args - 1), REG_15, 2975 tjit->reg_args_off); 2976 /* lgr %r1,%r8 */ 2977 if (!(flags & BPF_TRAMP_F_SKIP_FRAME) && 2978 (flags & BPF_TRAMP_F_ORIG_STACK)) 2979 EMIT4(0xb9040000, REG_1, REG_8); 2980 /* lmg %r7,%r8,r7_r8_off(%r15) */ 2981 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15, 2982 tjit->r7_r8_off); 2983 /* lg %r14,r14_off(%r15) */ 2984 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off); 2985 /* lg %r2,retval_off(%r15) */ 2986 if (flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET)) 2987 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15, 2988 tjit->retval_off); 2989 /* mvc stack_size+tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2990 _EMIT6(0xd203f000 | (tjit->stack_size + 2991 offsetof(struct prog_frame, tail_call_cnt)), 2992 0xf000 | tjit->tccnt_off); 2993 /* aghi %r15,stack_size */ 2994 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); 2995 if (flags & BPF_TRAMP_F_SKIP_FRAME) 2996 EMIT_JUMP_REG(14); 2997 else if (flags & BPF_TRAMP_F_ORIG_STACK) 2998 EMIT_JUMP_REG(1); 2999 else 3000 /* brcl 0xf,func_addr+S390X_PATCH_SIZE */ 3001 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 3002 func_addr + S390X_PATCH_SIZE); 3003 return 0; 3004 } 3005 3006 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 3007 struct bpf_tramp_nodes *tnodes, void *orig_call) 3008 { 3009 struct bpf_tramp_image im = {}; 3010 struct bpf_tramp_jit tjit; 3011 int ret; 3012 3013 memset(&tjit, 0, sizeof(tjit)); 3014 3015 ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, 3016 tnodes, orig_call); 3017 3018 return ret < 0 ? ret : tjit.common.prg; 3019 } 3020 3021 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 3022 void *image_end, const struct btf_func_model *m, 3023 u32 flags, struct bpf_tramp_nodes *tnodes, 3024 void *func_addr) 3025 { 3026 struct bpf_tramp_jit tjit; 3027 int ret; 3028 3029 /* Compute offsets, check whether the code fits. */ 3030 memset(&tjit, 0, sizeof(tjit)); 3031 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 3032 tnodes, func_addr); 3033 3034 if (ret < 0) 3035 return ret; 3036 if (tjit.common.prg > (char *)image_end - (char *)image) 3037 /* 3038 * Use the same error code as for exceeding 3039 * BPF_MAX_TRAMP_LINKS. 3040 */ 3041 return -E2BIG; 3042 3043 tjit.common.prg = 0; 3044 tjit.common.prg_buf = image; 3045 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 3046 tnodes, func_addr); 3047 3048 return ret < 0 ? ret : tjit.common.prg; 3049 } 3050 3051 bool bpf_jit_supports_subprog_tailcalls(void) 3052 { 3053 return true; 3054 } 3055 3056 bool bpf_jit_supports_arena(void) 3057 { 3058 return true; 3059 } 3060 3061 bool bpf_jit_supports_fsession(void) 3062 { 3063 return true; 3064 } 3065 3066 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 3067 { 3068 if (!in_arena) 3069 return true; 3070 switch (insn->code) { 3071 case BPF_LDX | BPF_MEMSX | BPF_B: 3072 case BPF_LDX | BPF_MEMSX | BPF_H: 3073 case BPF_LDX | BPF_MEMSX | BPF_W: 3074 return false; 3075 } 3076 return true; 3077 } 3078 3079 bool bpf_jit_supports_exceptions(void) 3080 { 3081 /* 3082 * Exceptions require unwinding support, which is always available, 3083 * because the kernel is always built with backchain. 3084 */ 3085 return true; 3086 } 3087 3088 void arch_bpf_stack_walk(bool (*consume_fn)(void *, u64, u64, u64), 3089 void *cookie) 3090 { 3091 unsigned long addr, prev_addr = 0; 3092 struct unwind_state state; 3093 3094 unwind_for_each_frame(&state, NULL, NULL, 0) { 3095 addr = unwind_get_return_address(&state); 3096 if (!addr) 3097 break; 3098 /* 3099 * addr is a return address and state.sp is the value of %r15 3100 * at this address. exception_cb needs %r15 at entry to the 3101 * function containing addr, so take the next state.sp. 3102 * 3103 * There is no bp, and the exception_cb prog does not need one 3104 * to perform a quasi-longjmp. The common code requires a 3105 * non-zero bp, so pass sp there as well. 3106 */ 3107 if (prev_addr && !consume_fn(cookie, prev_addr, state.sp, 3108 state.sp)) 3109 break; 3110 prev_addr = addr; 3111 } 3112 } 3113 3114 bool bpf_jit_supports_timed_may_goto(void) 3115 { 3116 return true; 3117 } 3118 3119 bool bpf_jit_inlines_helper_call(s32 imm) 3120 { 3121 switch (imm) { 3122 case BPF_FUNC_get_smp_processor_id: 3123 case BPF_FUNC_get_current_task: 3124 case BPF_FUNC_get_current_task_btf: 3125 return true; 3126 default: 3127 return false; 3128 } 3129 } 3130