1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * BPF Jit compiler for s390. 4 * 5 * Minimum build requirements: 6 * 7 * - HAVE_MARCH_Z196_FEATURES: laal, laalg 8 * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj 9 * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf 10 * - 64BIT 11 * 12 * Copyright IBM Corp. 2012,2015 13 * 14 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 15 * Michael Holzheu <holzheu@linux.vnet.ibm.com> 16 */ 17 18 #define pr_fmt(fmt) "bpf_jit: " fmt 19 20 #include <linux/netdevice.h> 21 #include <linux/filter.h> 22 #include <linux/init.h> 23 #include <linux/bpf.h> 24 #include <linux/cfi.h> 25 #include <linux/mm.h> 26 #include <linux/kernel.h> 27 #include <asm/cacheflush.h> 28 #include <asm/extable.h> 29 #include <asm/dis.h> 30 #include <asm/facility.h> 31 #include <asm/lowcore.h> 32 #include <asm/nospec-branch.h> 33 #include <asm/set_memory.h> 34 #include <asm/text-patching.h> 35 #include <asm/unwind.h> 36 37 struct bpf_jit { 38 u32 seen; /* Flags to remember seen eBPF instructions */ 39 u16 seen_regs; /* Mask to remember which registers are used */ 40 u32 *addrs; /* Array with relative instruction addresses */ 41 u8 *prg_buf; /* Start of program */ 42 int size; /* Size of program and literal pool */ 43 int size_prg; /* Size of program */ 44 int prg; /* Current position in program */ 45 int lit32_start; /* Start of 32-bit literal pool */ 46 int lit32; /* Current position in 32-bit literal pool */ 47 int lit64_start; /* Start of 64-bit literal pool */ 48 int lit64; /* Current position in 64-bit literal pool */ 49 int base_ip; /* Base address for literal pool */ 50 int exit_ip; /* Address of exit */ 51 int tail_call_start; /* Tail call start offset */ 52 int excnt; /* Number of exception table entries */ 53 int prologue_plt_ret; /* Return address for prologue hotpatch PLT */ 54 int prologue_plt; /* Start of prologue hotpatch PLT */ 55 int kern_arena; /* Pool offset of kernel arena address */ 56 u64 user_arena; /* User arena address */ 57 u32 frame_off; /* Offset of struct bpf_prog from %r15 */ 58 }; 59 60 #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */ 61 #define SEEN_LITERAL BIT(1) /* code uses literals */ 62 #define SEEN_FUNC BIT(2) /* calls C functions */ 63 #define SEEN_STACK (SEEN_FUNC | SEEN_MEM) 64 65 #define NVREGS 0xffc0 /* %r6-%r15 */ 66 67 /* 68 * s390 registers 69 */ 70 #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ 71 #define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ 72 #define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */ 73 #define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */ 74 #define REG_0 REG_W0 /* Register 0 */ 75 #define REG_1 REG_W1 /* Register 1 */ 76 #define REG_2 BPF_REG_1 /* Register 2 */ 77 #define REG_3 BPF_REG_2 /* Register 3 */ 78 #define REG_4 BPF_REG_3 /* Register 4 */ 79 #define REG_7 BPF_REG_6 /* Register 7 */ 80 #define REG_8 BPF_REG_7 /* Register 8 */ 81 #define REG_14 BPF_REG_0 /* Register 14 */ 82 83 /* 84 * Mapping of BPF registers to s390 registers 85 */ 86 static const int reg2hex[] = { 87 /* Return code */ 88 [BPF_REG_0] = 14, 89 /* Function parameters */ 90 [BPF_REG_1] = 2, 91 [BPF_REG_2] = 3, 92 [BPF_REG_3] = 4, 93 [BPF_REG_4] = 5, 94 [BPF_REG_5] = 6, 95 /* Call saved registers */ 96 [BPF_REG_6] = 7, 97 [BPF_REG_7] = 8, 98 [BPF_REG_8] = 9, 99 [BPF_REG_9] = 10, 100 /* BPF stack pointer */ 101 [BPF_REG_FP] = 13, 102 /* Register for blinding */ 103 [BPF_REG_AX] = 12, 104 /* Work registers for s390x backend */ 105 [REG_W0] = 0, 106 [REG_W1] = 1, 107 [REG_L] = 11, 108 [REG_15] = 15, 109 }; 110 111 static inline u32 reg(u32 dst_reg, u32 src_reg) 112 { 113 return reg2hex[dst_reg] << 4 | reg2hex[src_reg]; 114 } 115 116 static inline u32 reg_high(u32 reg) 117 { 118 return reg2hex[reg] << 4; 119 } 120 121 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 122 { 123 u32 r1 = reg2hex[b1]; 124 125 if (r1 >= 6 && r1 <= 15) 126 jit->seen_regs |= (1 << r1); 127 } 128 129 static s32 off_to_pcrel(struct bpf_jit *jit, u32 off) 130 { 131 return off - jit->prg; 132 } 133 134 static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr) 135 { 136 if (jit->prg_buf) 137 return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg); 138 return 0; 139 } 140 141 #define REG_SET_SEEN(b1) \ 142 ({ \ 143 reg_set_seen(jit, b1); \ 144 }) 145 146 /* 147 * EMIT macros for code generation 148 */ 149 150 #define _EMIT2(op) \ 151 ({ \ 152 if (jit->prg_buf) \ 153 *(u16 *) (jit->prg_buf + jit->prg) = (op); \ 154 jit->prg += 2; \ 155 }) 156 157 #define EMIT2(op, b1, b2) \ 158 ({ \ 159 _EMIT2((op) | reg(b1, b2)); \ 160 REG_SET_SEEN(b1); \ 161 REG_SET_SEEN(b2); \ 162 }) 163 164 #define _EMIT4(op) \ 165 ({ \ 166 if (jit->prg_buf) \ 167 *(u32 *) (jit->prg_buf + jit->prg) = (op); \ 168 jit->prg += 4; \ 169 }) 170 171 #define EMIT4(op, b1, b2) \ 172 ({ \ 173 _EMIT4((op) | reg(b1, b2)); \ 174 REG_SET_SEEN(b1); \ 175 REG_SET_SEEN(b2); \ 176 }) 177 178 #define EMIT4_RRF(op, b1, b2, b3) \ 179 ({ \ 180 _EMIT4((op) | reg_high(b3) << 8 | reg(b1, b2)); \ 181 REG_SET_SEEN(b1); \ 182 REG_SET_SEEN(b2); \ 183 REG_SET_SEEN(b3); \ 184 }) 185 186 #define _EMIT4_DISP(op, disp) \ 187 ({ \ 188 unsigned int __disp = (disp) & 0xfff; \ 189 _EMIT4((op) | __disp); \ 190 }) 191 192 #define EMIT4_DISP(op, b1, b2, disp) \ 193 ({ \ 194 _EMIT4_DISP((op) | reg_high(b1) << 16 | \ 195 reg_high(b2) << 8, (disp)); \ 196 REG_SET_SEEN(b1); \ 197 REG_SET_SEEN(b2); \ 198 }) 199 200 #define EMIT4_IMM(op, b1, imm) \ 201 ({ \ 202 unsigned int __imm = (imm) & 0xffff; \ 203 _EMIT4((op) | reg_high(b1) << 16 | __imm); \ 204 REG_SET_SEEN(b1); \ 205 }) 206 207 #define EMIT4_PCREL(op, pcrel) \ 208 ({ \ 209 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 210 _EMIT4((op) | __pcrel); \ 211 }) 212 213 #define EMIT4_PCREL_RIC(op, mask, target) \ 214 ({ \ 215 int __rel = off_to_pcrel(jit, target) / 2; \ 216 _EMIT4((op) | (mask) << 20 | (__rel & 0xffff)); \ 217 }) 218 219 #define _EMIT6(op1, op2) \ 220 ({ \ 221 if (jit->prg_buf) { \ 222 *(u32 *) (jit->prg_buf + jit->prg) = (op1); \ 223 *(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \ 224 } \ 225 jit->prg += 6; \ 226 }) 227 228 #define _EMIT6_DISP(op1, op2, disp) \ 229 ({ \ 230 unsigned int __disp = (disp) & 0xfff; \ 231 _EMIT6((op1) | __disp, op2); \ 232 }) 233 234 #define _EMIT6_DISP_LH(op1, op2, disp) \ 235 ({ \ 236 u32 _disp = (u32) (disp); \ 237 unsigned int __disp_h = _disp & 0xff000; \ 238 unsigned int __disp_l = _disp & 0x00fff; \ 239 _EMIT6((op1) | __disp_l, (op2) | __disp_h >> 4); \ 240 }) 241 242 #define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \ 243 ({ \ 244 _EMIT6_DISP_LH((op1) | reg(b1, b2) << 16 | \ 245 reg_high(b3) << 8, op2, disp); \ 246 REG_SET_SEEN(b1); \ 247 REG_SET_SEEN(b2); \ 248 REG_SET_SEEN(b3); \ 249 }) 250 251 #define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \ 252 ({ \ 253 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 254 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), \ 255 (op2) | (mask) << 12); \ 256 REG_SET_SEEN(b1); \ 257 REG_SET_SEEN(b2); \ 258 }) 259 260 #define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \ 261 ({ \ 262 unsigned int rel = off_to_pcrel(jit, target) / 2; \ 263 _EMIT6((op1) | (reg_high(b1) | (mask)) << 16 | \ 264 (rel & 0xffff), (op2) | ((imm) & 0xff) << 8); \ 265 REG_SET_SEEN(b1); \ 266 BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \ 267 }) 268 269 #define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \ 270 ({ \ 271 int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\ 272 _EMIT6((op1) | reg(b1, b2) << 16 | (rel & 0xffff), (op2) | (mask));\ 273 REG_SET_SEEN(b1); \ 274 REG_SET_SEEN(b2); \ 275 }) 276 277 static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel) 278 { 279 u32 pc32dbl = (s32)(pcrel / 2); 280 281 _EMIT6(op | pc32dbl >> 16, pc32dbl & 0xffff); 282 } 283 284 static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel) 285 { 286 emit6_pcrel_ril(jit, op | reg_high(b) << 16, pcrel); 287 REG_SET_SEEN(b); 288 } 289 290 #define EMIT6_PCREL_RILB(op, b, target) \ 291 emit6_pcrel_rilb(jit, op, b, off_to_pcrel(jit, target)) 292 293 #define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \ 294 emit6_pcrel_rilb(jit, op, b, ptr_to_pcrel(jit, target_ptr)) 295 296 static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel) 297 { 298 emit6_pcrel_ril(jit, op | mask << 20, pcrel); 299 } 300 301 #define EMIT6_PCREL_RILC(op, mask, target) \ 302 emit6_pcrel_rilc(jit, op, mask, off_to_pcrel(jit, target)) 303 304 #define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \ 305 emit6_pcrel_rilc(jit, op, mask, ptr_to_pcrel(jit, target_ptr)) 306 307 #define _EMIT6_IMM(op, imm) \ 308 ({ \ 309 unsigned int __imm = (imm); \ 310 _EMIT6((op) | (__imm >> 16), __imm & 0xffff); \ 311 }) 312 313 #define EMIT6_IMM(op, b1, imm) \ 314 ({ \ 315 _EMIT6_IMM((op) | reg_high(b1) << 16, imm); \ 316 REG_SET_SEEN(b1); \ 317 }) 318 319 #define _EMIT_CONST_U32(val) \ 320 ({ \ 321 unsigned int ret; \ 322 ret = jit->lit32; \ 323 if (jit->prg_buf) \ 324 *(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\ 325 jit->lit32 += 4; \ 326 ret; \ 327 }) 328 329 #define EMIT_CONST_U32(val) \ 330 ({ \ 331 jit->seen |= SEEN_LITERAL; \ 332 _EMIT_CONST_U32(val) - jit->base_ip; \ 333 }) 334 335 #define _EMIT_CONST_U64(val) \ 336 ({ \ 337 unsigned int ret; \ 338 ret = jit->lit64; \ 339 if (jit->prg_buf) \ 340 *(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\ 341 jit->lit64 += 8; \ 342 ret; \ 343 }) 344 345 #define EMIT_CONST_U64(val) \ 346 ({ \ 347 jit->seen |= SEEN_LITERAL; \ 348 _EMIT_CONST_U64(val) - jit->base_ip; \ 349 }) 350 351 #define EMIT_ZERO(b1) \ 352 ({ \ 353 if (!fp->aux->verifier_zext) { \ 354 /* llgfr %dst,%dst (zero extend to 64 bit) */ \ 355 EMIT4(0xb9160000, b1, b1); \ 356 REG_SET_SEEN(b1); \ 357 } \ 358 }) 359 360 static inline void emit_u32_data(const u32 data, struct bpf_jit *jit) 361 { 362 if (jit->prg_buf) 363 *(u32 *)(jit->prg_buf + jit->prg) = data; 364 jit->prg += 4; 365 } 366 367 static inline void emit_kcfi(u32 hash, struct bpf_jit *jit) 368 { 369 if (IS_ENABLED(CONFIG_CFI)) 370 emit_u32_data(hash, jit); 371 } 372 373 /* 374 * Return whether this is the first pass. The first pass is special, since we 375 * don't know any sizes yet, and thus must be conservative. 376 */ 377 static bool is_first_pass(struct bpf_jit *jit) 378 { 379 return jit->size == 0; 380 } 381 382 /* 383 * Return whether this is the code generation pass. The code generation pass is 384 * special, since we should change as little as possible. 385 */ 386 static bool is_codegen_pass(struct bpf_jit *jit) 387 { 388 return jit->prg_buf; 389 } 390 391 /* 392 * Return whether "rel" can be encoded as a short PC-relative offset 393 */ 394 static bool is_valid_rel(int rel) 395 { 396 return rel >= -65536 && rel <= 65534; 397 } 398 399 /* 400 * Return whether "off" can be reached using a short PC-relative offset 401 */ 402 static bool can_use_rel(struct bpf_jit *jit, int off) 403 { 404 return is_valid_rel(off - jit->prg); 405 } 406 407 /* 408 * Return whether given displacement can be encoded using 409 * Long-Displacement Facility 410 */ 411 static bool is_valid_ldisp(int disp) 412 { 413 return disp >= -524288 && disp <= 524287; 414 } 415 416 /* 417 * Return whether the next 32-bit literal pool entry can be referenced using 418 * Long-Displacement Facility 419 */ 420 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit) 421 { 422 return is_valid_ldisp(jit->lit32 - jit->base_ip); 423 } 424 425 /* 426 * Return whether the next 64-bit literal pool entry can be referenced using 427 * Long-Displacement Facility 428 */ 429 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit) 430 { 431 return is_valid_ldisp(jit->lit64 - jit->base_ip); 432 } 433 434 /* 435 * Fill whole space with illegal instructions 436 */ 437 static void jit_fill_hole(void *area, unsigned int size) 438 { 439 memset(area, 0, size); 440 } 441 442 /* 443 * Caller-allocated part of the frame. 444 * Thanks to packed stack, its otherwise unused initial part can be used for 445 * the BPF stack and for the next frame. 446 */ 447 struct prog_frame { 448 u64 unused[8]; 449 /* BPF stack starts here and grows towards 0 */ 450 u32 tail_call_cnt; 451 u32 pad; 452 u64 r6[10]; /* r6 - r15 */ 453 u64 backchain; 454 } __packed; 455 456 /* 457 * Save registers from "rs" (register start) to "re" (register end) on stack 458 */ 459 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) 460 { 461 u32 off = offsetof(struct prog_frame, r6) + (rs - 6) * 8; 462 463 if (rs == re) 464 /* stg %rs,off(%r15) */ 465 _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); 466 else 467 /* stmg %rs,%re,off(%r15) */ 468 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); 469 } 470 471 /* 472 * Restore registers from "rs" (register start) to "re" (register end) on stack 473 */ 474 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re) 475 { 476 u32 off = jit->frame_off + offsetof(struct prog_frame, r6) + (rs - 6) * 8; 477 478 if (rs == re) 479 /* lg %rs,off(%r15) */ 480 _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); 481 else 482 /* lmg %rs,%re,off(%r15) */ 483 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); 484 } 485 486 /* 487 * Return first seen register (from start) 488 */ 489 static int get_start(u16 seen_regs, int start) 490 { 491 int i; 492 493 for (i = start; i <= 15; i++) { 494 if (seen_regs & (1 << i)) 495 return i; 496 } 497 return 0; 498 } 499 500 /* 501 * Return last seen register (from start) (gap >= 2) 502 */ 503 static int get_end(u16 seen_regs, int start) 504 { 505 int i; 506 507 for (i = start; i < 15; i++) { 508 if (!(seen_regs & (3 << i))) 509 return i - 1; 510 } 511 return (seen_regs & (1 << 15)) ? 15 : 14; 512 } 513 514 #define REGS_SAVE 1 515 #define REGS_RESTORE 0 516 /* 517 * Save and restore clobbered registers (6-15) on stack. 518 * We save/restore registers in chunks with gap >= 2 registers. 519 */ 520 static void save_restore_regs(struct bpf_jit *jit, int op, u16 extra_regs) 521 { 522 u16 seen_regs = jit->seen_regs | extra_regs; 523 const int last = 15, save_restore_size = 6; 524 int re = 6, rs; 525 526 if (is_first_pass(jit)) { 527 /* 528 * We don't know yet which registers are used. Reserve space 529 * conservatively. 530 */ 531 jit->prg += (last - re + 1) * save_restore_size; 532 return; 533 } 534 535 do { 536 rs = get_start(seen_regs, re); 537 if (!rs) 538 break; 539 re = get_end(seen_regs, rs + 1); 540 if (op == REGS_SAVE) 541 save_regs(jit, rs, re); 542 else 543 restore_regs(jit, rs, re); 544 re++; 545 } while (re <= last); 546 } 547 548 static void bpf_skip(struct bpf_jit *jit, int size) 549 { 550 if (size >= 6 && !is_valid_rel(size)) { 551 /* brcl 0xf,size */ 552 EMIT6_PCREL_RILC(0xc0040000, 0xf, size); 553 size -= 6; 554 } else if (size >= 4 && is_valid_rel(size)) { 555 /* brc 0xf,size */ 556 EMIT4_PCREL(0xa7f40000, size); 557 size -= 4; 558 } 559 while (size >= 2) { 560 /* bcr 0,%0 */ 561 _EMIT2(0x0700); 562 size -= 2; 563 } 564 } 565 566 /* 567 * PLT for hotpatchable calls. The calling convention is the same as for the 568 * ftrace hotpatch trampolines: %r0 is return address, %r1 is clobbered. 569 */ 570 struct bpf_plt { 571 char code[16]; 572 void *ret; 573 void *target; 574 } __packed; 575 extern const struct bpf_plt bpf_plt; 576 asm( 577 ".pushsection .rodata\n" 578 " .balign 8\n" 579 "bpf_plt:\n" 580 " lgrl %r0,bpf_plt_ret\n" 581 " lgrl %r1,bpf_plt_target\n" 582 " br %r1\n" 583 " .balign 8\n" 584 "bpf_plt_ret: .quad 0\n" 585 "bpf_plt_target: .quad 0\n" 586 " .popsection\n" 587 ); 588 589 static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target) 590 { 591 memcpy(plt, &bpf_plt, sizeof(*plt)); 592 plt->ret = ret; 593 /* 594 * (target == NULL) implies that the branch to this PLT entry was 595 * patched and became a no-op. However, some CPU could have jumped 596 * to this PLT entry before patching and may be still executing it. 597 * 598 * Since the intention in this case is to make the PLT entry a no-op, 599 * make the target point to the return label instead of NULL. 600 */ 601 plt->target = target ?: ret; 602 } 603 604 /* 605 * Emit function prologue 606 * 607 * Save registers and create stack frame if necessary. 608 * Stack frame layout is described by struct prog_frame. 609 */ 610 static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp) 611 { 612 BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD); 613 614 emit_kcfi(bpf_is_subprog(fp) ? cfi_bpf_subprog_hash : cfi_bpf_hash, jit); 615 616 /* No-op for hotpatching */ 617 /* brcl 0,prologue_plt */ 618 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); 619 jit->prologue_plt_ret = jit->prg; 620 621 if (!bpf_is_subprog(fp)) { 622 /* Initialize the tail call counter in the main program. */ 623 /* xc tail_call_cnt(4,%r15),tail_call_cnt(%r15) */ 624 _EMIT6(0xd703f000 | offsetof(struct prog_frame, tail_call_cnt), 625 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); 626 } else { 627 /* 628 * Skip the tail call counter initialization in subprograms. 629 * Insert nops in order to have tail_call_start at a 630 * predictable offset. 631 */ 632 bpf_skip(jit, 6); 633 } 634 /* Tail calls have to skip above initialization */ 635 jit->tail_call_start = jit->prg - cfi_get_offset(); 636 if (fp->aux->exception_cb) { 637 /* 638 * Switch stack, the new address is in the 2nd parameter. 639 * 640 * Arrange the restoration of %r6-%r15 in the epilogue. 641 * Do not restore them now, the prog does not need them. 642 */ 643 /* lgr %r15,%r3 */ 644 EMIT4(0xb9040000, REG_15, REG_3); 645 jit->seen_regs |= NVREGS; 646 } else { 647 /* Save registers */ 648 save_restore_regs(jit, REGS_SAVE, 649 fp->aux->exception_boundary ? NVREGS : 0); 650 } 651 /* Setup literal pool */ 652 if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) { 653 if (!is_first_pass(jit) && 654 is_valid_ldisp(jit->size - (jit->prg + 2))) { 655 /* basr %l,0 */ 656 EMIT2(0x0d00, REG_L, REG_0); 657 jit->base_ip = jit->prg; 658 } else { 659 /* larl %l,lit32_start */ 660 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start); 661 jit->base_ip = jit->lit32_start; 662 } 663 } 664 /* Setup stack and backchain */ 665 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) { 666 /* lgr %w1,%r15 (backchain) */ 667 EMIT4(0xb9040000, REG_W1, REG_15); 668 /* la %bfp,unused_end(%r15) (BPF frame pointer) */ 669 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, 670 offsetofend(struct prog_frame, unused)); 671 /* aghi %r15,-frame_off */ 672 EMIT4_IMM(0xa70b0000, REG_15, -jit->frame_off); 673 /* stg %w1,backchain(%r15) */ 674 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, 675 REG_15, 676 offsetof(struct prog_frame, backchain)); 677 } 678 } 679 680 /* 681 * Jump using a register either directly or via an expoline thunk 682 */ 683 #define EMIT_JUMP_REG(reg) do { \ 684 if (nospec_uses_trampoline()) \ 685 /* brcl 0xf,__s390_indirect_jump_rN */ \ 686 EMIT6_PCREL_RILC_PTR(0xc0040000, 0x0f, \ 687 __s390_indirect_jump_r ## reg); \ 688 else \ 689 /* br %rN */ \ 690 _EMIT2(0x07f0 | reg); \ 691 } while (0) 692 693 /* 694 * Function epilogue 695 */ 696 static void bpf_jit_epilogue(struct bpf_jit *jit) 697 { 698 jit->exit_ip = jit->prg; 699 /* Load exit code: lgr %r2,%b0 */ 700 EMIT4(0xb9040000, REG_2, BPF_REG_0); 701 /* Restore registers */ 702 save_restore_regs(jit, REGS_RESTORE, 0); 703 EMIT_JUMP_REG(14); 704 705 jit->prg = ALIGN(jit->prg, 8); 706 jit->prologue_plt = jit->prg; 707 if (jit->prg_buf) 708 bpf_jit_plt((struct bpf_plt *)(jit->prg_buf + jit->prg), 709 jit->prg_buf + jit->prologue_plt_ret, NULL); 710 jit->prg += sizeof(struct bpf_plt); 711 } 712 713 bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) 714 { 715 regs->psw.addr = extable_fixup(x); 716 if (x->data != -1) 717 regs->gprs[x->data] = 0; 718 return true; 719 } 720 721 /* 722 * A single BPF probe instruction 723 */ 724 struct bpf_jit_probe { 725 int prg; /* JITed instruction offset */ 726 int nop_prg; /* JITed nop offset */ 727 int reg; /* Register to clear on exception */ 728 int arena_reg; /* Register to use for arena addressing */ 729 }; 730 731 static void bpf_jit_probe_init(struct bpf_jit_probe *probe) 732 { 733 probe->prg = -1; 734 probe->nop_prg = -1; 735 probe->reg = -1; 736 probe->arena_reg = REG_0; 737 } 738 739 /* 740 * Handlers of certain exceptions leave psw.addr pointing to the instruction 741 * directly after the failing one. Therefore, create two exception table 742 * entries and also add a nop in case two probing instructions come directly 743 * after each other. 744 */ 745 static void bpf_jit_probe_emit_nop(struct bpf_jit *jit, 746 struct bpf_jit_probe *probe) 747 { 748 if (probe->prg == -1 || probe->nop_prg != -1) 749 /* The probe is not armed or nop is already emitted. */ 750 return; 751 752 probe->nop_prg = jit->prg; 753 /* bcr 0,%0 */ 754 _EMIT2(0x0700); 755 } 756 757 static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, 758 struct bpf_jit_probe *probe) 759 { 760 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 761 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 762 BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 763 BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 764 return; 765 766 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 767 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { 768 /* lgrl %r1,kern_arena */ 769 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 770 probe->arena_reg = REG_W1; 771 } 772 probe->prg = jit->prg; 773 probe->reg = reg2hex[insn->dst_reg]; 774 } 775 776 static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, 777 struct bpf_jit_probe *probe) 778 { 779 if (BPF_MODE(insn->code) != BPF_PROBE_MEM32 && 780 BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 781 return; 782 783 /* lgrl %r1,kern_arena */ 784 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 785 probe->arena_reg = REG_W1; 786 probe->prg = jit->prg; 787 } 788 789 static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, 790 struct bpf_insn *insn, 791 struct bpf_jit_probe *probe) 792 { 793 int load_reg; 794 795 if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) 796 return; 797 798 /* lgrl %r1,kern_arena */ 799 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); 800 /* agr %r1,%dst */ 801 EMIT4(0xb9080000, REG_W1, insn->dst_reg); 802 probe->arena_reg = REG_W1; 803 probe->prg = jit->prg; 804 /* 805 * A read-modify-write carrying BPF_FETCH reads the old value into 806 * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on 807 * fault, the remaining atomics only write memory. 808 */ 809 load_reg = bpf_atomic_load_reg(insn); 810 if (load_reg >= 0) 811 probe->reg = reg2hex[load_reg]; 812 } 813 814 static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, 815 struct bpf_jit_probe *probe) 816 { 817 struct exception_table_entry *ex; 818 int i, prg; 819 s64 delta; 820 u8 *insn; 821 822 if (probe->prg == -1) 823 /* The probe is not armed. */ 824 return 0; 825 bpf_jit_probe_emit_nop(jit, probe); 826 if (!fp->aux->extable) 827 /* Do nothing during early JIT passes. */ 828 return 0; 829 insn = jit->prg_buf + probe->prg; 830 if (WARN_ON_ONCE(probe->prg + insn_length(*insn) != probe->nop_prg)) 831 /* JIT bug - gap between probe and nop instructions. */ 832 return -1; 833 for (i = 0; i < 2; i++) { 834 if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) 835 /* Verifier bug - not enough entries. */ 836 return -1; 837 ex = &fp->aux->extable[jit->excnt]; 838 /* Add extable entries for probe and nop instructions. */ 839 prg = i == 0 ? probe->prg : probe->nop_prg; 840 delta = jit->prg_buf + prg - (u8 *)&ex->insn; 841 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 842 /* JIT bug - code and extable must be close. */ 843 return -1; 844 ex->insn = delta; 845 /* 846 * Land on the current instruction. Note that the extable 847 * infrastructure ignores the fixup field; it is handled by 848 * ex_handler_bpf(). 849 */ 850 delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; 851 if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) 852 /* JIT bug - landing pad and extable must be close. */ 853 return -1; 854 ex->fixup = delta; 855 ex->type = EX_TYPE_BPF; 856 ex->data = probe->reg; 857 jit->excnt++; 858 } 859 return 0; 860 } 861 862 static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) 863 { 864 struct bpf_jit_probe probe; 865 866 bpf_jit_probe_init(&probe); 867 bpf_jit_probe_load_pre(jit, insn, &probe); 868 869 switch (BPF_SIZE(insn->code)) { 870 case BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 871 /* llgc %dst,off(%src,%arena) */ 872 EMIT6_DISP_LH(0xe3000000, 0x0090, insn->dst_reg, insn->src_reg, 873 probe.arena_reg, insn->off); 874 break; 875 case BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 876 /* llgh %dst,off(%src,%arena) */ 877 EMIT6_DISP_LH(0xe3000000, 0x0091, insn->dst_reg, insn->src_reg, 878 probe.arena_reg, insn->off); 879 break; 880 case BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 881 /* llgf %dst,off(%src,%arena) */ 882 EMIT6_DISP_LH(0xe3000000, 0x0016, insn->dst_reg, insn->src_reg, 883 probe.arena_reg, insn->off); 884 break; 885 case BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 886 /* lg %dst,off(%src,%arena) */ 887 EMIT6_DISP_LH(0xe3000000, 0x0004, insn->dst_reg, insn->src_reg, 888 probe.arena_reg, insn->off); 889 break; 890 } 891 892 return bpf_jit_probe_post(jit, fp, &probe); 893 } 894 895 static int emit_stx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) 896 { 897 struct bpf_jit_probe probe; 898 899 bpf_jit_probe_init(&probe); 900 bpf_jit_probe_store_pre(jit, insn, &probe); 901 902 switch (BPF_SIZE(insn->code)) { 903 case BPF_B: /* *(u8 *)(dst + off) = src_reg */ 904 /* stcy %src,off(%dst,%arena) */ 905 EMIT6_DISP_LH(0xe3000000, 0x0072, insn->src_reg, insn->dst_reg, 906 probe.arena_reg, insn->off); 907 break; 908 case BPF_H: /* (u16 *)(dst + off) = src */ 909 /* sthy %src,off(%dst,%arena) */ 910 EMIT6_DISP_LH(0xe3000000, 0x0070, insn->src_reg, insn->dst_reg, 911 probe.arena_reg, insn->off); 912 break; 913 case BPF_W: /* *(u32 *)(dst + off) = src */ 914 /* sty %src,off(%dst,%arena) */ 915 EMIT6_DISP_LH(0xe3000000, 0x0050, insn->src_reg, insn->dst_reg, 916 probe.arena_reg, insn->off); 917 break; 918 case BPF_DW: /* (u64 *)(dst + off) = src */ 919 /* stg %src,off(%dst,%arena) */ 920 EMIT6_DISP_LH(0xe3000000, 0x0024, insn->src_reg, insn->dst_reg, 921 probe.arena_reg, insn->off); 922 break; 923 } 924 925 return bpf_jit_probe_post(jit, fp, &probe); 926 } 927 928 /* 929 * Sign- or zero-extend the register if necessary 930 */ 931 static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) 932 { 933 switch (size) { 934 case 1: 935 if (flags & BTF_FMODEL_SIGNED_ARG) 936 /* lgbr %r,%r */ 937 EMIT4(0xb9060000, r, r); 938 else 939 /* llgcr %r,%r */ 940 EMIT4(0xb9840000, r, r); 941 return 0; 942 case 2: 943 if (flags & BTF_FMODEL_SIGNED_ARG) 944 /* lghr %r,%r */ 945 EMIT4(0xb9070000, r, r); 946 else 947 /* llghr %r,%r */ 948 EMIT4(0xb9850000, r, r); 949 return 0; 950 case 4: 951 if (flags & BTF_FMODEL_SIGNED_ARG) 952 /* lgfr %r,%r */ 953 EMIT4(0xb9140000, r, r); 954 else 955 /* llgfr %r,%r */ 956 EMIT4(0xb9160000, r, r); 957 return 0; 958 case 8: 959 return 0; 960 default: 961 return -1; 962 } 963 } 964 965 /* 966 * Compile one eBPF instruction into s390x code 967 * 968 * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of 969 * stack space for the large switch statement. 970 */ 971 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, 972 int i, bool extra_pass) 973 { 974 struct bpf_insn *insn = &fp->insnsi[i]; 975 s32 branch_oc_off = insn->off; 976 u32 dst_reg = insn->dst_reg; 977 u32 src_reg = insn->src_reg; 978 struct bpf_jit_probe probe; 979 int last, insn_count = 1; 980 u32 *addrs = jit->addrs; 981 s32 imm = insn->imm; 982 s16 off = insn->off; 983 unsigned int mask; 984 int err; 985 986 bpf_jit_probe_init(&probe); 987 988 switch (insn->code) { 989 /* 990 * BPF_MOV 991 */ 992 case BPF_ALU | BPF_MOV | BPF_X: 993 switch (insn->off) { 994 case 0: /* DST = (u32) SRC */ 995 /* llgfr %dst,%src */ 996 EMIT4(0xb9160000, dst_reg, src_reg); 997 if (insn_is_zext(&insn[1])) 998 insn_count = 2; 999 break; 1000 case 8: /* DST = (u32)(s8) SRC */ 1001 /* lbr %dst,%src */ 1002 EMIT4(0xb9260000, dst_reg, src_reg); 1003 /* llgfr %dst,%dst */ 1004 EMIT4(0xb9160000, dst_reg, dst_reg); 1005 break; 1006 case 16: /* DST = (u32)(s16) SRC */ 1007 /* lhr %dst,%src */ 1008 EMIT4(0xb9270000, dst_reg, src_reg); 1009 /* llgfr %dst,%dst */ 1010 EMIT4(0xb9160000, dst_reg, dst_reg); 1011 break; 1012 } 1013 break; 1014 case BPF_ALU64 | BPF_MOV | BPF_X: 1015 if (insn_is_cast_user(insn)) { 1016 int patch_brc; 1017 1018 /* ltgr %dst,%src */ 1019 EMIT4(0xb9020000, dst_reg, src_reg); 1020 /* brc 8,0f */ 1021 patch_brc = jit->prg; 1022 EMIT4_PCREL_RIC(0xa7040000, 8, 0); 1023 /* iihf %dst,user_arena>>32 */ 1024 EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32); 1025 /* 0: */ 1026 if (jit->prg_buf) 1027 *(u16 *)(jit->prg_buf + patch_brc + 2) = 1028 (jit->prg - patch_brc) >> 1; 1029 break; 1030 } 1031 switch (insn->off) { 1032 case 0: /* DST = SRC */ 1033 /* lgr %dst,%src */ 1034 EMIT4(0xb9040000, dst_reg, src_reg); 1035 break; 1036 case 8: /* DST = (s8) SRC */ 1037 /* lgbr %dst,%src */ 1038 EMIT4(0xb9060000, dst_reg, src_reg); 1039 break; 1040 case 16: /* DST = (s16) SRC */ 1041 /* lghr %dst,%src */ 1042 EMIT4(0xb9070000, dst_reg, src_reg); 1043 break; 1044 case 32: /* DST = (s32) SRC */ 1045 /* lgfr %dst,%src */ 1046 EMIT4(0xb9140000, dst_reg, src_reg); 1047 break; 1048 } 1049 break; 1050 case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */ 1051 /* llilf %dst,imm */ 1052 EMIT6_IMM(0xc00f0000, dst_reg, imm); 1053 if (insn_is_zext(&insn[1])) 1054 insn_count = 2; 1055 break; 1056 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */ 1057 /* lgfi %dst,imm */ 1058 EMIT6_IMM(0xc0010000, dst_reg, imm); 1059 break; 1060 /* 1061 * BPF_LD 64 1062 */ 1063 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 1064 { 1065 /* 16 byte instruction that uses two 'struct bpf_insn' */ 1066 u64 imm64; 1067 1068 imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; 1069 /* lgrl %dst,imm */ 1070 EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64)); 1071 insn_count = 2; 1072 break; 1073 } 1074 /* 1075 * BPF_ADD 1076 */ 1077 case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */ 1078 /* ar %dst,%src */ 1079 EMIT2(0x1a00, dst_reg, src_reg); 1080 EMIT_ZERO(dst_reg); 1081 break; 1082 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */ 1083 /* agr %dst,%src */ 1084 EMIT4(0xb9080000, dst_reg, src_reg); 1085 break; 1086 case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */ 1087 if (imm != 0) { 1088 /* alfi %dst,imm */ 1089 EMIT6_IMM(0xc20b0000, dst_reg, imm); 1090 } 1091 EMIT_ZERO(dst_reg); 1092 break; 1093 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */ 1094 if (!imm) 1095 break; 1096 /* agfi %dst,imm */ 1097 EMIT6_IMM(0xc2080000, dst_reg, imm); 1098 break; 1099 /* 1100 * BPF_SUB 1101 */ 1102 case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */ 1103 /* sr %dst,%src */ 1104 EMIT2(0x1b00, dst_reg, src_reg); 1105 EMIT_ZERO(dst_reg); 1106 break; 1107 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */ 1108 /* sgr %dst,%src */ 1109 EMIT4(0xb9090000, dst_reg, src_reg); 1110 break; 1111 case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */ 1112 if (imm != 0) { 1113 /* alfi %dst,-imm */ 1114 EMIT6_IMM(0xc20b0000, dst_reg, -imm); 1115 } 1116 EMIT_ZERO(dst_reg); 1117 break; 1118 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */ 1119 if (!imm) 1120 break; 1121 if (imm == -0x80000000) { 1122 /* algfi %dst,0x80000000 */ 1123 EMIT6_IMM(0xc20a0000, dst_reg, 0x80000000); 1124 } else { 1125 /* agfi %dst,-imm */ 1126 EMIT6_IMM(0xc2080000, dst_reg, -imm); 1127 } 1128 break; 1129 /* 1130 * BPF_MUL 1131 */ 1132 case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */ 1133 /* msr %dst,%src */ 1134 EMIT4(0xb2520000, dst_reg, src_reg); 1135 EMIT_ZERO(dst_reg); 1136 break; 1137 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */ 1138 /* msgr %dst,%src */ 1139 EMIT4(0xb90c0000, dst_reg, src_reg); 1140 break; 1141 case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */ 1142 if (imm != 1) { 1143 /* msfi %r5,imm */ 1144 EMIT6_IMM(0xc2010000, dst_reg, imm); 1145 } 1146 EMIT_ZERO(dst_reg); 1147 break; 1148 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */ 1149 if (imm == 1) 1150 break; 1151 /* msgfi %dst,imm */ 1152 EMIT6_IMM(0xc2000000, dst_reg, imm); 1153 break; 1154 /* 1155 * BPF_DIV / BPF_MOD 1156 */ 1157 case BPF_ALU | BPF_DIV | BPF_X: 1158 case BPF_ALU | BPF_MOD | BPF_X: 1159 { 1160 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1161 1162 switch (off) { 1163 case 0: /* dst = (u32) dst {/,%} (u32) src */ 1164 /* xr %w0,%w0 */ 1165 EMIT2(0x1700, REG_W0, REG_W0); 1166 /* lr %w1,%dst */ 1167 EMIT2(0x1800, REG_W1, dst_reg); 1168 /* dlr %w0,%src */ 1169 EMIT4(0xb9970000, REG_W0, src_reg); 1170 break; 1171 case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */ 1172 /* lgfr %r1,%dst */ 1173 EMIT4(0xb9140000, REG_W1, dst_reg); 1174 /* dsgfr %r0,%src */ 1175 EMIT4(0xb91d0000, REG_W0, src_reg); 1176 break; 1177 } 1178 /* llgfr %dst,%rc */ 1179 EMIT4(0xb9160000, dst_reg, rc_reg); 1180 if (insn_is_zext(&insn[1])) 1181 insn_count = 2; 1182 break; 1183 } 1184 case BPF_ALU64 | BPF_DIV | BPF_X: 1185 case BPF_ALU64 | BPF_MOD | BPF_X: 1186 { 1187 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1188 1189 switch (off) { 1190 case 0: /* dst = dst {/,%} src */ 1191 /* lghi %w0,0 */ 1192 EMIT4_IMM(0xa7090000, REG_W0, 0); 1193 /* lgr %w1,%dst */ 1194 EMIT4(0xb9040000, REG_W1, dst_reg); 1195 /* dlgr %w0,%src */ 1196 EMIT4(0xb9870000, REG_W0, src_reg); 1197 break; 1198 case 1: /* dst = (s64) dst {/,%} (s64) src */ 1199 /* lgr %w1,%dst */ 1200 EMIT4(0xb9040000, REG_W1, dst_reg); 1201 /* dsgr %w0,%src */ 1202 EMIT4(0xb90d0000, REG_W0, src_reg); 1203 break; 1204 } 1205 /* lgr %dst,%rc */ 1206 EMIT4(0xb9040000, dst_reg, rc_reg); 1207 break; 1208 } 1209 case BPF_ALU | BPF_DIV | BPF_K: 1210 case BPF_ALU | BPF_MOD | BPF_K: 1211 { 1212 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1213 1214 if (imm == 1) { 1215 if (BPF_OP(insn->code) == BPF_MOD) 1216 /* lghi %dst,0 */ 1217 EMIT4_IMM(0xa7090000, dst_reg, 0); 1218 else 1219 EMIT_ZERO(dst_reg); 1220 break; 1221 } 1222 if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) { 1223 switch (off) { 1224 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1225 /* xr %w0,%w0 */ 1226 EMIT2(0x1700, REG_W0, REG_W0); 1227 /* lr %w1,%dst */ 1228 EMIT2(0x1800, REG_W1, dst_reg); 1229 /* dl %w0,<d(imm)>(%l) */ 1230 EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, 1231 REG_L, EMIT_CONST_U32(imm)); 1232 break; 1233 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1234 /* lgfr %r1,%dst */ 1235 EMIT4(0xb9140000, REG_W1, dst_reg); 1236 /* dsgf %r0,<d(imm)>(%l) */ 1237 EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0, 1238 REG_L, EMIT_CONST_U32(imm)); 1239 break; 1240 } 1241 } else { 1242 switch (off) { 1243 case 0: /* dst = (u32) dst {/,%} (u32) imm */ 1244 /* xr %w0,%w0 */ 1245 EMIT2(0x1700, REG_W0, REG_W0); 1246 /* lr %w1,%dst */ 1247 EMIT2(0x1800, REG_W1, dst_reg); 1248 /* lrl %dst,imm */ 1249 EMIT6_PCREL_RILB(0xc40d0000, dst_reg, 1250 _EMIT_CONST_U32(imm)); 1251 jit->seen |= SEEN_LITERAL; 1252 /* dlr %w0,%dst */ 1253 EMIT4(0xb9970000, REG_W0, dst_reg); 1254 break; 1255 case 1: /* dst = (s32) dst {/,%} (s32) imm */ 1256 /* lgfr %w1,%dst */ 1257 EMIT4(0xb9140000, REG_W1, dst_reg); 1258 /* lgfrl %dst,imm */ 1259 EMIT6_PCREL_RILB(0xc40c0000, dst_reg, 1260 _EMIT_CONST_U32(imm)); 1261 jit->seen |= SEEN_LITERAL; 1262 /* dsgr %w0,%dst */ 1263 EMIT4(0xb90d0000, REG_W0, dst_reg); 1264 break; 1265 } 1266 } 1267 /* llgfr %dst,%rc */ 1268 EMIT4(0xb9160000, dst_reg, rc_reg); 1269 if (insn_is_zext(&insn[1])) 1270 insn_count = 2; 1271 break; 1272 } 1273 case BPF_ALU64 | BPF_DIV | BPF_K: 1274 case BPF_ALU64 | BPF_MOD | BPF_K: 1275 { 1276 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; 1277 1278 if (imm == 1) { 1279 if (BPF_OP(insn->code) == BPF_MOD) 1280 /* lhgi %dst,0 */ 1281 EMIT4_IMM(0xa7090000, dst_reg, 0); 1282 break; 1283 } 1284 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1285 switch (off) { 1286 case 0: /* dst = dst {/,%} imm */ 1287 /* lghi %w0,0 */ 1288 EMIT4_IMM(0xa7090000, REG_W0, 0); 1289 /* lgr %w1,%dst */ 1290 EMIT4(0xb9040000, REG_W1, dst_reg); 1291 /* dlg %w0,<d(imm)>(%l) */ 1292 EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, 1293 REG_L, EMIT_CONST_U64(imm)); 1294 break; 1295 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1296 /* lgr %w1,%dst */ 1297 EMIT4(0xb9040000, REG_W1, dst_reg); 1298 /* dsg %w0,<d(imm)>(%l) */ 1299 EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0, 1300 REG_L, EMIT_CONST_U64(imm)); 1301 break; 1302 } 1303 } else { 1304 switch (off) { 1305 case 0: /* dst = dst {/,%} imm */ 1306 /* lghi %w0,0 */ 1307 EMIT4_IMM(0xa7090000, REG_W0, 0); 1308 /* lgr %w1,%dst */ 1309 EMIT4(0xb9040000, REG_W1, dst_reg); 1310 /* lgrl %dst,imm */ 1311 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1312 _EMIT_CONST_U64(imm)); 1313 jit->seen |= SEEN_LITERAL; 1314 /* dlgr %w0,%dst */ 1315 EMIT4(0xb9870000, REG_W0, dst_reg); 1316 break; 1317 case 1: /* dst = (s64) dst {/,%} (s64) imm */ 1318 /* lgr %w1,%dst */ 1319 EMIT4(0xb9040000, REG_W1, dst_reg); 1320 /* lgrl %dst,imm */ 1321 EMIT6_PCREL_RILB(0xc4080000, dst_reg, 1322 _EMIT_CONST_U64(imm)); 1323 jit->seen |= SEEN_LITERAL; 1324 /* dsgr %w0,%dst */ 1325 EMIT4(0xb90d0000, REG_W0, dst_reg); 1326 break; 1327 } 1328 } 1329 /* lgr %dst,%rc */ 1330 EMIT4(0xb9040000, dst_reg, rc_reg); 1331 break; 1332 } 1333 /* 1334 * BPF_AND 1335 */ 1336 case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */ 1337 /* nr %dst,%src */ 1338 EMIT2(0x1400, dst_reg, src_reg); 1339 EMIT_ZERO(dst_reg); 1340 break; 1341 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 1342 /* ngr %dst,%src */ 1343 EMIT4(0xb9800000, dst_reg, src_reg); 1344 break; 1345 case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */ 1346 /* nilf %dst,imm */ 1347 EMIT6_IMM(0xc00b0000, dst_reg, imm); 1348 EMIT_ZERO(dst_reg); 1349 break; 1350 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 1351 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1352 /* ng %dst,<d(imm)>(%l) */ 1353 EMIT6_DISP_LH(0xe3000000, 0x0080, 1354 dst_reg, REG_0, REG_L, 1355 EMIT_CONST_U64(imm)); 1356 } else { 1357 /* lgrl %w0,imm */ 1358 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1359 _EMIT_CONST_U64(imm)); 1360 jit->seen |= SEEN_LITERAL; 1361 /* ngr %dst,%w0 */ 1362 EMIT4(0xb9800000, dst_reg, REG_W0); 1363 } 1364 break; 1365 /* 1366 * BPF_OR 1367 */ 1368 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 1369 /* or %dst,%src */ 1370 EMIT2(0x1600, dst_reg, src_reg); 1371 EMIT_ZERO(dst_reg); 1372 break; 1373 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 1374 /* ogr %dst,%src */ 1375 EMIT4(0xb9810000, dst_reg, src_reg); 1376 break; 1377 case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */ 1378 /* oilf %dst,imm */ 1379 EMIT6_IMM(0xc00d0000, dst_reg, imm); 1380 EMIT_ZERO(dst_reg); 1381 break; 1382 case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */ 1383 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1384 /* og %dst,<d(imm)>(%l) */ 1385 EMIT6_DISP_LH(0xe3000000, 0x0081, 1386 dst_reg, REG_0, REG_L, 1387 EMIT_CONST_U64(imm)); 1388 } else { 1389 /* lgrl %w0,imm */ 1390 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1391 _EMIT_CONST_U64(imm)); 1392 jit->seen |= SEEN_LITERAL; 1393 /* ogr %dst,%w0 */ 1394 EMIT4(0xb9810000, dst_reg, REG_W0); 1395 } 1396 break; 1397 /* 1398 * BPF_XOR 1399 */ 1400 case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */ 1401 /* xr %dst,%src */ 1402 EMIT2(0x1700, dst_reg, src_reg); 1403 EMIT_ZERO(dst_reg); 1404 break; 1405 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */ 1406 /* xgr %dst,%src */ 1407 EMIT4(0xb9820000, dst_reg, src_reg); 1408 break; 1409 case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */ 1410 if (imm != 0) { 1411 /* xilf %dst,imm */ 1412 EMIT6_IMM(0xc0070000, dst_reg, imm); 1413 } 1414 EMIT_ZERO(dst_reg); 1415 break; 1416 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */ 1417 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { 1418 /* xg %dst,<d(imm)>(%l) */ 1419 EMIT6_DISP_LH(0xe3000000, 0x0082, 1420 dst_reg, REG_0, REG_L, 1421 EMIT_CONST_U64(imm)); 1422 } else { 1423 /* lgrl %w0,imm */ 1424 EMIT6_PCREL_RILB(0xc4080000, REG_W0, 1425 _EMIT_CONST_U64(imm)); 1426 jit->seen |= SEEN_LITERAL; 1427 /* xgr %dst,%w0 */ 1428 EMIT4(0xb9820000, dst_reg, REG_W0); 1429 } 1430 break; 1431 /* 1432 * BPF_LSH 1433 */ 1434 case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */ 1435 /* sll %dst,0(%src) */ 1436 EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); 1437 EMIT_ZERO(dst_reg); 1438 break; 1439 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ 1440 /* sllg %dst,%dst,0(%src) */ 1441 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0); 1442 break; 1443 case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */ 1444 if (imm != 0) { 1445 /* sll %dst,imm(%r0) */ 1446 EMIT4_DISP(0x89000000, dst_reg, REG_0, imm); 1447 } 1448 EMIT_ZERO(dst_reg); 1449 break; 1450 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */ 1451 if (imm == 0) 1452 break; 1453 /* sllg %dst,%dst,imm(%r0) */ 1454 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm); 1455 break; 1456 /* 1457 * BPF_RSH 1458 */ 1459 case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */ 1460 /* srl %dst,0(%src) */ 1461 EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); 1462 EMIT_ZERO(dst_reg); 1463 break; 1464 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ 1465 /* srlg %dst,%dst,0(%src) */ 1466 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0); 1467 break; 1468 case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */ 1469 if (imm != 0) { 1470 /* srl %dst,imm(%r0) */ 1471 EMIT4_DISP(0x88000000, dst_reg, REG_0, imm); 1472 } 1473 EMIT_ZERO(dst_reg); 1474 break; 1475 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */ 1476 if (imm == 0) 1477 break; 1478 /* srlg %dst,%dst,imm(%r0) */ 1479 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm); 1480 break; 1481 /* 1482 * BPF_ARSH 1483 */ 1484 case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */ 1485 /* sra %dst,%dst,0(%src) */ 1486 EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); 1487 EMIT_ZERO(dst_reg); 1488 break; 1489 case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ 1490 /* srag %dst,%dst,0(%src) */ 1491 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); 1492 break; 1493 case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */ 1494 if (imm != 0) { 1495 /* sra %dst,imm(%r0) */ 1496 EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm); 1497 } 1498 EMIT_ZERO(dst_reg); 1499 break; 1500 case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */ 1501 if (imm == 0) 1502 break; 1503 /* srag %dst,%dst,imm(%r0) */ 1504 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm); 1505 break; 1506 /* 1507 * BPF_NEG 1508 */ 1509 case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */ 1510 /* lcr %dst,%dst */ 1511 EMIT2(0x1300, dst_reg, dst_reg); 1512 EMIT_ZERO(dst_reg); 1513 break; 1514 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 1515 /* lcgr %dst,%dst */ 1516 EMIT4(0xb9030000, dst_reg, dst_reg); 1517 break; 1518 /* 1519 * BPF_FROM_BE/LE 1520 */ 1521 case BPF_ALU | BPF_END | BPF_FROM_BE: 1522 /* s390 is big endian, therefore only clear high order bytes */ 1523 switch (imm) { 1524 case 16: /* dst = (u16) cpu_to_be16(dst) */ 1525 /* llghr %dst,%dst */ 1526 EMIT4(0xb9850000, dst_reg, dst_reg); 1527 if (insn_is_zext(&insn[1])) 1528 insn_count = 2; 1529 break; 1530 case 32: /* dst = (u32) cpu_to_be32(dst) */ 1531 if (!fp->aux->verifier_zext) 1532 /* llgfr %dst,%dst */ 1533 EMIT4(0xb9160000, dst_reg, dst_reg); 1534 break; 1535 case 64: /* dst = (u64) cpu_to_be64(dst) */ 1536 break; 1537 } 1538 break; 1539 case BPF_ALU | BPF_END | BPF_FROM_LE: 1540 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1541 switch (imm) { 1542 case 16: /* dst = (u16) cpu_to_le16(dst) */ 1543 /* lrvr %dst,%dst */ 1544 EMIT4(0xb91f0000, dst_reg, dst_reg); 1545 /* srl %dst,16(%r0) */ 1546 EMIT4_DISP(0x88000000, dst_reg, REG_0, 16); 1547 /* llghr %dst,%dst */ 1548 EMIT4(0xb9850000, dst_reg, dst_reg); 1549 if (insn_is_zext(&insn[1])) 1550 insn_count = 2; 1551 break; 1552 case 32: /* dst = (u32) cpu_to_le32(dst) */ 1553 /* lrvr %dst,%dst */ 1554 EMIT4(0xb91f0000, dst_reg, dst_reg); 1555 if (!fp->aux->verifier_zext) 1556 /* llgfr %dst,%dst */ 1557 EMIT4(0xb9160000, dst_reg, dst_reg); 1558 break; 1559 case 64: /* dst = (u64) cpu_to_le64(dst) */ 1560 /* lrvgr %dst,%dst */ 1561 EMIT4(0xb90f0000, dst_reg, dst_reg); 1562 break; 1563 } 1564 break; 1565 /* 1566 * BPF_NOSPEC (speculation barrier) 1567 */ 1568 case BPF_ST | BPF_NOSPEC: 1569 break; 1570 /* 1571 * BPF_ST(X) 1572 */ 1573 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ 1574 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1575 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 1576 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1577 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 1578 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1579 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 1580 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1581 err = emit_stx(jit, fp, insn); 1582 if (err < 0) 1583 return err; 1584 jit->seen |= SEEN_MEM; 1585 break; 1586 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 1587 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1588 /* lhi %w0,imm */ 1589 EMIT4_IMM(0xa7080000, REG_W0, (u8) imm); 1590 bpf_jit_probe_store_pre(jit, insn, &probe); 1591 /* stcy %w0,off(%dst,%arena) */ 1592 EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, 1593 probe.arena_reg, off); 1594 err = bpf_jit_probe_post(jit, fp, &probe); 1595 if (err < 0) 1596 return err; 1597 jit->seen |= SEEN_MEM; 1598 break; 1599 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 1600 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1601 /* lhi %w0,imm */ 1602 EMIT4_IMM(0xa7080000, REG_W0, (u16) imm); 1603 bpf_jit_probe_store_pre(jit, insn, &probe); 1604 /* sthy %w0,off(%dst,%arena) */ 1605 EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, 1606 probe.arena_reg, off); 1607 err = bpf_jit_probe_post(jit, fp, &probe); 1608 if (err < 0) 1609 return err; 1610 jit->seen |= SEEN_MEM; 1611 break; 1612 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 1613 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1614 /* llilf %w0,imm */ 1615 EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm); 1616 bpf_jit_probe_store_pre(jit, insn, &probe); 1617 /* sty %w0,off(%dst,%arena) */ 1618 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, 1619 probe.arena_reg, off); 1620 err = bpf_jit_probe_post(jit, fp, &probe); 1621 if (err < 0) 1622 return err; 1623 jit->seen |= SEEN_MEM; 1624 break; 1625 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 1626 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1627 /* lgfi %w0,imm */ 1628 EMIT6_IMM(0xc0010000, REG_W0, imm); 1629 bpf_jit_probe_store_pre(jit, insn, &probe); 1630 /* stg %w0,off(%dst,%arena) */ 1631 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, 1632 probe.arena_reg, off); 1633 err = bpf_jit_probe_post(jit, fp, &probe); 1634 if (err < 0) 1635 return err; 1636 jit->seen |= SEEN_MEM; 1637 break; 1638 /* 1639 * BPF_ATOMIC 1640 */ 1641 case BPF_STX | BPF_ATOMIC | BPF_B: 1642 case BPF_STX | BPF_ATOMIC | BPF_H: 1643 case BPF_STX | BPF_ATOMIC | BPF_DW: 1644 case BPF_STX | BPF_ATOMIC | BPF_W: 1645 case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: 1646 case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: 1647 case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: 1648 case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: 1649 { 1650 bool is32 = BPF_SIZE(insn->code) == BPF_W; 1651 1652 /* 1653 * Unlike loads and stores, s390 atomics have only a base 1654 * register, but no index register. For the non-arena case, 1655 * simply use %dst as a base. For the arena case, use the 1656 * work register %r1: first, load the arena base into it, 1657 * and then add %dst to it. 1658 */ 1659 probe.arena_reg = dst_reg; 1660 1661 switch (insn->imm) { 1662 #define EMIT_ATOMIC(op32, op64) do { \ 1663 bpf_jit_probe_atomic_pre(jit, insn, &probe); \ 1664 /* {op32|op64} {%w0|%src},%src,off(%arena) */ \ 1665 EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ 1666 (insn->imm & BPF_FETCH) ? src_reg : REG_W0, \ 1667 src_reg, probe.arena_reg, off); \ 1668 err = bpf_jit_probe_post(jit, fp, &probe); \ 1669 if (err < 0) \ 1670 return err; \ 1671 if (insn->imm & BPF_FETCH) { \ 1672 /* bcr 14,0 - see atomic_fetch_{add,and,or,xor}() */ \ 1673 _EMIT2(0x07e0); \ 1674 if (is32) \ 1675 EMIT_ZERO(src_reg); \ 1676 } \ 1677 } while (0) 1678 case BPF_ADD: 1679 case BPF_ADD | BPF_FETCH: 1680 /* {laal|laalg} */ 1681 EMIT_ATOMIC(0x00fa, 0x00ea); 1682 break; 1683 case BPF_AND: 1684 case BPF_AND | BPF_FETCH: 1685 /* {lan|lang} */ 1686 EMIT_ATOMIC(0x00f4, 0x00e4); 1687 break; 1688 case BPF_OR: 1689 case BPF_OR | BPF_FETCH: 1690 /* {lao|laog} */ 1691 EMIT_ATOMIC(0x00f6, 0x00e6); 1692 break; 1693 case BPF_XOR: 1694 case BPF_XOR | BPF_FETCH: 1695 /* {lax|laxg} */ 1696 EMIT_ATOMIC(0x00f7, 0x00e7); 1697 break; 1698 #undef EMIT_ATOMIC 1699 case BPF_XCHG: { 1700 struct bpf_jit_probe load_probe = probe; 1701 int loop_start; 1702 1703 bpf_jit_probe_atomic_pre(jit, insn, &load_probe); 1704 /* {ly|lg} %w0,off(%arena) */ 1705 EMIT6_DISP_LH(0xe3000000, 1706 is32 ? 0x0058 : 0x0004, REG_W0, REG_0, 1707 load_probe.arena_reg, off); 1708 bpf_jit_probe_emit_nop(jit, &load_probe); 1709 /* Reuse {ly|lg}'s arena_reg for {csy|csg}. */ 1710 if (load_probe.prg != -1) { 1711 probe.prg = jit->prg; 1712 probe.arena_reg = load_probe.arena_reg; 1713 probe.reg = load_probe.reg; 1714 } 1715 loop_start = jit->prg; 1716 /* 0: {csy|csg} %w0,%src,off(%arena) */ 1717 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1718 REG_W0, src_reg, probe.arena_reg, off); 1719 bpf_jit_probe_emit_nop(jit, &probe); 1720 /* brc 4,0b */ 1721 EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); 1722 /* {llgfr|lgr} %src,%w0 */ 1723 EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); 1724 /* Both probes should land here on exception. */ 1725 err = bpf_jit_probe_post(jit, fp, &load_probe); 1726 if (err < 0) 1727 return err; 1728 err = bpf_jit_probe_post(jit, fp, &probe); 1729 if (err < 0) 1730 return err; 1731 if (is32 && insn_is_zext(&insn[1])) 1732 insn_count = 2; 1733 break; 1734 } 1735 case BPF_CMPXCHG: 1736 bpf_jit_probe_atomic_pre(jit, insn, &probe); 1737 /* 0: {csy|csg} %b0,%src,off(%arena) */ 1738 EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1739 BPF_REG_0, src_reg, 1740 probe.arena_reg, off); 1741 err = bpf_jit_probe_post(jit, fp, &probe); 1742 if (err < 0) 1743 return err; 1744 break; 1745 case BPF_LOAD_ACQ: 1746 /* s390 has strong ordering, just use load */ 1747 err = emit_ldx(jit, fp, insn); 1748 if (err < 0) 1749 return err; 1750 break; 1751 case BPF_STORE_REL: 1752 /* s390 has strong ordering, just use store */ 1753 err = emit_stx(jit, fp, insn); 1754 if (err < 0) 1755 return err; 1756 break; 1757 default: 1758 pr_err("Unknown atomic operation %02x\n", insn->imm); 1759 return -1; 1760 } 1761 1762 jit->seen |= SEEN_MEM; 1763 break; 1764 } 1765 /* 1766 * BPF_LDX 1767 */ 1768 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 1769 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1770 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1771 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 1772 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1773 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1774 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 1775 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1776 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1777 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 1778 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1779 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1780 err = emit_ldx(jit, fp, insn); 1781 if (err < 0) 1782 return err; 1783 jit->seen |= SEEN_MEM; 1784 if (BPF_SIZE(insn->code) != BPF_DW && insn_is_zext(&insn[1])) 1785 insn_count = 2; 1786 break; 1787 case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ 1788 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1789 bpf_jit_probe_load_pre(jit, insn, &probe); 1790 /* lgb %dst,off(%src) */ 1791 EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off); 1792 err = bpf_jit_probe_post(jit, fp, &probe); 1793 if (err < 0) 1794 return err; 1795 jit->seen |= SEEN_MEM; 1796 break; 1797 case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ 1798 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1799 bpf_jit_probe_load_pre(jit, insn, &probe); 1800 /* lgh %dst,off(%src) */ 1801 EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off); 1802 err = bpf_jit_probe_post(jit, fp, &probe); 1803 if (err < 0) 1804 return err; 1805 jit->seen |= SEEN_MEM; 1806 break; 1807 case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ 1808 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1809 bpf_jit_probe_load_pre(jit, insn, &probe); 1810 /* lgf %dst,off(%src) */ 1811 jit->seen |= SEEN_MEM; 1812 EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off); 1813 err = bpf_jit_probe_post(jit, fp, &probe); 1814 if (err < 0) 1815 return err; 1816 break; 1817 /* 1818 * BPF_JMP / CALL 1819 */ 1820 case BPF_JMP | BPF_CALL: 1821 { 1822 const struct btf_func_model *m; 1823 bool func_addr_fixed; 1824 int j, ret; 1825 u64 func; 1826 1827 /* Implement helper call to bpf_get_smp_processor_id() inline */ 1828 if (insn->src_reg == 0 && 1829 insn->imm == BPF_FUNC_get_smp_processor_id) { 1830 const u32 *cpu_nr = &get_lowcore()->cpu_nr; 1831 1832 /* llgf %b0, cpu_nr */ 1833 EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0, 1834 (unsigned long)cpu_nr); 1835 break; 1836 } 1837 1838 /* Implement helper call to bpf_get_current_task/_btf() inline */ 1839 if (insn->src_reg == 0 && 1840 (insn->imm == BPF_FUNC_get_current_task || 1841 insn->imm == BPF_FUNC_get_current_task_btf)) { 1842 const u64 *current_task = 1843 &get_lowcore()->current_task; 1844 1845 /* lg %b0, current_task */ 1846 EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_0, REG_0, REG_0, 1847 (unsigned long)current_task); 1848 break; 1849 } 1850 1851 ret = bpf_jit_get_func_addr(fp, insn, extra_pass, 1852 &func, &func_addr_fixed); 1853 if (ret < 0) 1854 return -1; 1855 1856 REG_SET_SEEN(BPF_REG_5); 1857 jit->seen |= SEEN_FUNC; 1858 1859 /* 1860 * Copy the tail call counter to where the callee expects it. 1861 */ 1862 1863 if (insn->src_reg == BPF_PSEUDO_CALL) 1864 /* 1865 * mvc tail_call_cnt(4,%r15), 1866 * frame_off+tail_call_cnt(%r15) 1867 */ 1868 _EMIT6(0xd203f000 | offsetof(struct prog_frame, 1869 tail_call_cnt), 1870 0xf000 | (jit->frame_off + 1871 offsetof(struct prog_frame, 1872 tail_call_cnt))); 1873 1874 /* Sign-extend the kfunc arguments. */ 1875 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1876 m = bpf_jit_find_kfunc_model(fp, insn); 1877 if (!m) 1878 return -1; 1879 1880 for (j = 0; j < m->nr_args; j++) { 1881 if (sign_zero_extend(jit, BPF_REG_1 + j, 1882 m->arg_size[j], 1883 m->arg_flags[j])) 1884 return -1; 1885 } 1886 } 1887 1888 if ((void *)func == arch_bpf_timed_may_goto) { 1889 /* 1890 * arch_bpf_timed_may_goto() has a special ABI: the 1891 * parameters are in BPF_REG_AX and BPF_REG_10; the 1892 * return value is in BPF_REG_AX; and all GPRs except 1893 * REG_W0, REG_W1, and BPF_REG_AX are callee-saved. 1894 */ 1895 1896 /* brasl %r0,func */ 1897 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_0, (void *)func); 1898 } else { 1899 /* brasl %r14,func */ 1900 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, (void *)func); 1901 /* lgr %b0,%r2: load return value into %b0 */ 1902 EMIT4(0xb9040000, BPF_REG_0, REG_2); 1903 } 1904 1905 /* 1906 * Copy the potentially updated tail call counter back. 1907 */ 1908 1909 if (insn->src_reg == BPF_PSEUDO_CALL) 1910 /* 1911 * mvc frame_off+tail_call_cnt(%r15), 1912 * tail_call_cnt(4,%r15) 1913 */ 1914 _EMIT6(0xd203f000 | (jit->frame_off + 1915 offsetof(struct prog_frame, 1916 tail_call_cnt)), 1917 0xf000 | offsetof(struct prog_frame, 1918 tail_call_cnt)); 1919 1920 break; 1921 } 1922 case BPF_JMP | BPF_TAIL_CALL: { 1923 int patch_1_clrj, patch_2_clij, patch_3_brc; 1924 1925 /* 1926 * Implicit input: 1927 * B1: pointer to ctx 1928 * B2: pointer to bpf_array 1929 * B3: index in bpf_array 1930 * 1931 * if (index >= array->map.max_entries) 1932 * goto out; 1933 */ 1934 1935 /* llgf %w1,map.max_entries(%b2) */ 1936 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2, 1937 offsetof(struct bpf_array, map.max_entries)); 1938 /* if ((u32)%b3 >= (u32)%w1) goto out; */ 1939 /* clrj %b3,%w1,0xa,out */ 1940 patch_1_clrj = jit->prg; 1941 EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa, 1942 jit->prg); 1943 1944 /* 1945 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) 1946 * goto out; 1947 * 1948 * tail_call_cnt is read into %w0, which needs to be preserved 1949 * until it's incremented and flushed. 1950 */ 1951 1952 off = jit->frame_off + 1953 offsetof(struct prog_frame, tail_call_cnt); 1954 /* ly %w0,off(%r15) */ 1955 EMIT6_DISP_LH(0xe3000000, 0x0058, REG_W0, REG_0, REG_15, off); 1956 /* clij %w0,MAX_TAIL_CALL_CNT,0xa,out */ 1957 patch_2_clij = jit->prg; 1958 EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W0, MAX_TAIL_CALL_CNT, 1959 0xa, jit->prg); 1960 1961 /* 1962 * prog = array->ptrs[index]; 1963 * if (prog == NULL) 1964 * goto out; 1965 */ 1966 1967 /* llgfr %r1,%b3: %r1 = (u32) index */ 1968 EMIT4(0xb9160000, REG_1, BPF_REG_3); 1969 /* sllg %r1,%r1,3: %r1 *= 8 */ 1970 EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3); 1971 /* ltg %r1,prog(%b2,%r1) */ 1972 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2, 1973 REG_1, offsetof(struct bpf_array, ptrs)); 1974 /* brc 0x8,out */ 1975 patch_3_brc = jit->prg; 1976 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); 1977 1978 /* tail_call_cnt++; */ 1979 /* ahi %w0,1 */ 1980 EMIT4_IMM(0xa70a0000, REG_W0, 1); 1981 /* sty %w0,off(%r15) */ 1982 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, REG_0, REG_15, off); 1983 1984 /* 1985 * Restore registers before calling function 1986 */ 1987 save_restore_regs(jit, REGS_RESTORE, 0); 1988 1989 /* 1990 * goto *(prog->bpf_func + tail_call_start); 1991 */ 1992 1993 /* lg %r1,bpf_func(%r1) */ 1994 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0, 1995 offsetof(struct bpf_prog, bpf_func)); 1996 if (nospec_uses_trampoline()) { 1997 jit->seen |= SEEN_FUNC; 1998 /* aghi %r1,tail_call_start */ 1999 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start); 2000 /* brcl 0xf,__s390_indirect_jump_r1 */ 2001 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 2002 __s390_indirect_jump_r1); 2003 } else { 2004 /* bc 0xf,tail_call_start(%r1) */ 2005 _EMIT4(0x47f01000 + jit->tail_call_start); 2006 } 2007 /* out: */ 2008 if (jit->prg_buf) { 2009 *(u16 *)(jit->prg_buf + patch_1_clrj + 2) = 2010 (jit->prg - patch_1_clrj) >> 1; 2011 *(u16 *)(jit->prg_buf + patch_2_clij + 2) = 2012 (jit->prg - patch_2_clij) >> 1; 2013 *(u16 *)(jit->prg_buf + patch_3_brc + 2) = 2014 (jit->prg - patch_3_brc) >> 1; 2015 } 2016 break; 2017 } 2018 case BPF_JMP | BPF_EXIT: /* return b0 */ 2019 last = (i == fp->len - 1) ? 1 : 0; 2020 if (last) 2021 break; 2022 if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip)) 2023 /* brc 0xf, <exit> */ 2024 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip); 2025 else 2026 /* brcl 0xf, <exit> */ 2027 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip); 2028 break; 2029 /* 2030 * Branch relative (number of skipped instructions) to offset on 2031 * condition. 2032 * 2033 * Condition code to mask mapping: 2034 * 2035 * CC | Description | Mask 2036 * ------------------------------ 2037 * 0 | Operands equal | 8 2038 * 1 | First operand low | 4 2039 * 2 | First operand high | 2 2040 * 3 | Unused | 1 2041 * 2042 * For s390x relative branches: ip = ip + off_bytes 2043 * For BPF relative branches: insn = insn + off_insns + 1 2044 * 2045 * For example for s390x with offset 0 we jump to the branch 2046 * instruction itself (loop) and for BPF with offset 0 we 2047 * branch to the instruction behind the branch. 2048 */ 2049 case BPF_JMP32 | BPF_JA: /* if (true) */ 2050 branch_oc_off = imm; 2051 fallthrough; 2052 case BPF_JMP | BPF_JA: /* if (true) */ 2053 mask = 0xf000; /* j */ 2054 goto branch_oc; 2055 case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */ 2056 case BPF_JMP32 | BPF_JSGT | BPF_K: /* ((s32) dst > (s32) imm) */ 2057 mask = 0x2000; /* jh */ 2058 goto branch_ks; 2059 case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */ 2060 case BPF_JMP32 | BPF_JSLT | BPF_K: /* ((s32) dst < (s32) imm) */ 2061 mask = 0x4000; /* jl */ 2062 goto branch_ks; 2063 case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */ 2064 case BPF_JMP32 | BPF_JSGE | BPF_K: /* ((s32) dst >= (s32) imm) */ 2065 mask = 0xa000; /* jhe */ 2066 goto branch_ks; 2067 case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */ 2068 case BPF_JMP32 | BPF_JSLE | BPF_K: /* ((s32) dst <= (s32) imm) */ 2069 mask = 0xc000; /* jle */ 2070 goto branch_ks; 2071 case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */ 2072 case BPF_JMP32 | BPF_JGT | BPF_K: /* ((u32) dst_reg > (u32) imm) */ 2073 mask = 0x2000; /* jh */ 2074 goto branch_ku; 2075 case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */ 2076 case BPF_JMP32 | BPF_JLT | BPF_K: /* ((u32) dst_reg < (u32) imm) */ 2077 mask = 0x4000; /* jl */ 2078 goto branch_ku; 2079 case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */ 2080 case BPF_JMP32 | BPF_JGE | BPF_K: /* ((u32) dst_reg >= (u32) imm) */ 2081 mask = 0xa000; /* jhe */ 2082 goto branch_ku; 2083 case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */ 2084 case BPF_JMP32 | BPF_JLE | BPF_K: /* ((u32) dst_reg <= (u32) imm) */ 2085 mask = 0xc000; /* jle */ 2086 goto branch_ku; 2087 case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */ 2088 case BPF_JMP32 | BPF_JNE | BPF_K: /* ((u32) dst_reg != (u32) imm) */ 2089 mask = 0x7000; /* jne */ 2090 goto branch_ku; 2091 case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */ 2092 case BPF_JMP32 | BPF_JEQ | BPF_K: /* ((u32) dst_reg == (u32) imm) */ 2093 mask = 0x8000; /* je */ 2094 goto branch_ku; 2095 case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */ 2096 case BPF_JMP32 | BPF_JSET | BPF_K: /* ((u32) dst_reg & (u32) imm) */ 2097 mask = 0x7000; /* jnz */ 2098 if (BPF_CLASS(insn->code) == BPF_JMP32) { 2099 /* llilf %w1,imm (load zero extend imm) */ 2100 EMIT6_IMM(0xc00f0000, REG_W1, imm); 2101 /* nr %w1,%dst */ 2102 EMIT2(0x1400, REG_W1, dst_reg); 2103 } else { 2104 /* lgfi %w1,imm (load sign extend imm) */ 2105 EMIT6_IMM(0xc0010000, REG_W1, imm); 2106 /* ngr %w1,%dst */ 2107 EMIT4(0xb9800000, REG_W1, dst_reg); 2108 } 2109 goto branch_oc; 2110 2111 case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */ 2112 case BPF_JMP32 | BPF_JSGT | BPF_X: /* ((s32) dst > (s32) src) */ 2113 mask = 0x2000; /* jh */ 2114 goto branch_xs; 2115 case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */ 2116 case BPF_JMP32 | BPF_JSLT | BPF_X: /* ((s32) dst < (s32) src) */ 2117 mask = 0x4000; /* jl */ 2118 goto branch_xs; 2119 case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */ 2120 case BPF_JMP32 | BPF_JSGE | BPF_X: /* ((s32) dst >= (s32) src) */ 2121 mask = 0xa000; /* jhe */ 2122 goto branch_xs; 2123 case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */ 2124 case BPF_JMP32 | BPF_JSLE | BPF_X: /* ((s32) dst <= (s32) src) */ 2125 mask = 0xc000; /* jle */ 2126 goto branch_xs; 2127 case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */ 2128 case BPF_JMP32 | BPF_JGT | BPF_X: /* ((u32) dst > (u32) src) */ 2129 mask = 0x2000; /* jh */ 2130 goto branch_xu; 2131 case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */ 2132 case BPF_JMP32 | BPF_JLT | BPF_X: /* ((u32) dst < (u32) src) */ 2133 mask = 0x4000; /* jl */ 2134 goto branch_xu; 2135 case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */ 2136 case BPF_JMP32 | BPF_JGE | BPF_X: /* ((u32) dst >= (u32) src) */ 2137 mask = 0xa000; /* jhe */ 2138 goto branch_xu; 2139 case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */ 2140 case BPF_JMP32 | BPF_JLE | BPF_X: /* ((u32) dst <= (u32) src) */ 2141 mask = 0xc000; /* jle */ 2142 goto branch_xu; 2143 case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */ 2144 case BPF_JMP32 | BPF_JNE | BPF_X: /* ((u32) dst != (u32) src) */ 2145 mask = 0x7000; /* jne */ 2146 goto branch_xu; 2147 case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */ 2148 case BPF_JMP32 | BPF_JEQ | BPF_X: /* ((u32) dst == (u32) src) */ 2149 mask = 0x8000; /* je */ 2150 goto branch_xu; 2151 case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */ 2152 case BPF_JMP32 | BPF_JSET | BPF_X: /* ((u32) dst & (u32) src) */ 2153 { 2154 bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2155 2156 mask = 0x7000; /* jnz */ 2157 /* nrk or ngrk %w1,%dst,%src */ 2158 EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000), 2159 REG_W1, dst_reg, src_reg); 2160 goto branch_oc; 2161 branch_ks: 2162 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2163 /* cfi or cgfi %dst,imm */ 2164 EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000, 2165 dst_reg, imm); 2166 if (!is_first_pass(jit) && 2167 can_use_rel(jit, addrs[i + off + 1])) { 2168 /* brc mask,off */ 2169 EMIT4_PCREL_RIC(0xa7040000, 2170 mask >> 12, addrs[i + off + 1]); 2171 } else { 2172 /* brcl mask,off */ 2173 EMIT6_PCREL_RILC(0xc0040000, 2174 mask >> 12, addrs[i + off + 1]); 2175 } 2176 break; 2177 branch_ku: 2178 /* lgfi %w1,imm (load sign extend imm) */ 2179 src_reg = REG_1; 2180 EMIT6_IMM(0xc0010000, src_reg, imm); 2181 goto branch_xu; 2182 branch_xs: 2183 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2184 if (!is_first_pass(jit) && 2185 can_use_rel(jit, addrs[i + off + 1])) { 2186 /* crj or cgrj %dst,%src,mask,off */ 2187 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064), 2188 dst_reg, src_reg, i, off, mask); 2189 } else { 2190 /* cr or cgr %dst,%src */ 2191 if (is_jmp32) 2192 EMIT2(0x1900, dst_reg, src_reg); 2193 else 2194 EMIT4(0xb9200000, dst_reg, src_reg); 2195 /* brcl mask,off */ 2196 EMIT6_PCREL_RILC(0xc0040000, 2197 mask >> 12, addrs[i + off + 1]); 2198 } 2199 break; 2200 branch_xu: 2201 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 2202 if (!is_first_pass(jit) && 2203 can_use_rel(jit, addrs[i + off + 1])) { 2204 /* clrj or clgrj %dst,%src,mask,off */ 2205 EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065), 2206 dst_reg, src_reg, i, off, mask); 2207 } else { 2208 /* clr or clgr %dst,%src */ 2209 if (is_jmp32) 2210 EMIT2(0x1500, dst_reg, src_reg); 2211 else 2212 EMIT4(0xb9210000, dst_reg, src_reg); 2213 /* brcl mask,off */ 2214 EMIT6_PCREL_RILC(0xc0040000, 2215 mask >> 12, addrs[i + off + 1]); 2216 } 2217 break; 2218 branch_oc: 2219 if (!is_first_pass(jit) && 2220 can_use_rel(jit, addrs[i + branch_oc_off + 1])) { 2221 /* brc mask,off */ 2222 EMIT4_PCREL_RIC(0xa7040000, 2223 mask >> 12, 2224 addrs[i + branch_oc_off + 1]); 2225 } else { 2226 /* brcl mask,off */ 2227 EMIT6_PCREL_RILC(0xc0040000, 2228 mask >> 12, 2229 addrs[i + branch_oc_off + 1]); 2230 } 2231 break; 2232 } 2233 default: /* too complex, give up */ 2234 pr_err("Unknown opcode %02x\n", insn->code); 2235 return -1; 2236 } 2237 2238 return insn_count; 2239 } 2240 2241 /* 2242 * Return whether new i-th instruction address does not violate any invariant 2243 */ 2244 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i) 2245 { 2246 /* On the first pass anything goes */ 2247 if (is_first_pass(jit)) 2248 return true; 2249 2250 /* The codegen pass must not change anything */ 2251 if (is_codegen_pass(jit)) 2252 return jit->addrs[i] == jit->prg; 2253 2254 /* Passes in between must not increase code size */ 2255 return jit->addrs[i] >= jit->prg; 2256 } 2257 2258 /* 2259 * Update the address of i-th instruction 2260 */ 2261 static int bpf_set_addr(struct bpf_jit *jit, int i) 2262 { 2263 int delta; 2264 2265 if (is_codegen_pass(jit)) { 2266 delta = jit->prg - jit->addrs[i]; 2267 if (delta < 0) 2268 bpf_skip(jit, -delta); 2269 } 2270 if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i))) 2271 return -1; 2272 jit->addrs[i] = jit->prg; 2273 return 0; 2274 } 2275 2276 /* 2277 * Compile eBPF program into s390x code 2278 */ 2279 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, 2280 bool extra_pass) 2281 { 2282 int i, insn_count, lit32_size, lit64_size; 2283 u64 kern_arena; 2284 2285 jit->lit32 = jit->lit32_start; 2286 jit->lit64 = jit->lit64_start; 2287 jit->prg = 0; 2288 jit->excnt = 0; 2289 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) 2290 jit->frame_off = sizeof(struct prog_frame) - 2291 offsetofend(struct prog_frame, unused) + 2292 round_up(fp->aux->stack_depth, 8); 2293 else 2294 jit->frame_off = 0; 2295 2296 kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); 2297 if (kern_arena) 2298 jit->kern_arena = _EMIT_CONST_U64(kern_arena); 2299 jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); 2300 2301 bpf_jit_prologue(jit, fp); 2302 if (bpf_set_addr(jit, 0) < 0) 2303 return -1; 2304 for (i = 0; i < fp->len; i += insn_count) { 2305 insn_count = bpf_jit_insn(jit, fp, i, extra_pass); 2306 if (insn_count < 0) 2307 return -1; 2308 /* Next instruction address */ 2309 if (bpf_set_addr(jit, i + insn_count) < 0) 2310 return -1; 2311 } 2312 bpf_jit_epilogue(jit); 2313 2314 lit32_size = jit->lit32 - jit->lit32_start; 2315 lit64_size = jit->lit64 - jit->lit64_start; 2316 jit->lit32_start = jit->prg; 2317 if (lit32_size) 2318 jit->lit32_start = ALIGN(jit->lit32_start, 4); 2319 jit->lit64_start = jit->lit32_start + lit32_size; 2320 if (lit64_size) 2321 jit->lit64_start = ALIGN(jit->lit64_start, 8); 2322 jit->size = jit->lit64_start + lit64_size; 2323 jit->size_prg = jit->prg; 2324 2325 if (WARN_ON_ONCE(fp->aux->extable && 2326 jit->excnt != fp->aux->num_exentries)) 2327 /* Verifier bug - too many entries. */ 2328 return -1; 2329 2330 return 0; 2331 } 2332 2333 bool bpf_jit_needs_zext(void) 2334 { 2335 return true; 2336 } 2337 2338 struct s390_jit_data { 2339 struct bpf_binary_header *header; 2340 struct bpf_jit ctx; 2341 int pass; 2342 }; 2343 2344 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, 2345 struct bpf_prog *fp) 2346 { 2347 struct bpf_binary_header *header; 2348 struct bpf_insn *insn; 2349 u32 extable_size; 2350 u32 code_size; 2351 int i; 2352 2353 for (i = 0; i < fp->len; i++) { 2354 insn = &fp->insnsi[i]; 2355 2356 if (BPF_CLASS(insn->code) == BPF_STX && 2357 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC && 2358 (BPF_SIZE(insn->code) == BPF_DW || 2359 BPF_SIZE(insn->code) == BPF_W) && 2360 insn->imm == BPF_XCHG) 2361 /* 2362 * bpf_jit_insn() emits a load and a compare-and-swap, 2363 * both of which need to be probed. 2364 */ 2365 fp->aux->num_exentries += 1; 2366 } 2367 /* We need two entries per insn. */ 2368 fp->aux->num_exentries *= 2; 2369 2370 code_size = roundup(jit->size, 2371 __alignof__(struct exception_table_entry)); 2372 extable_size = fp->aux->num_exentries * 2373 sizeof(struct exception_table_entry); 2374 header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf, 2375 8, jit_fill_hole); 2376 if (!header) 2377 return NULL; 2378 fp->aux->extable = (struct exception_table_entry *) 2379 (jit->prg_buf + code_size); 2380 return header; 2381 } 2382 2383 /* 2384 * Compile eBPF program "fp" 2385 */ 2386 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 2387 { 2388 struct bpf_binary_header *header; 2389 struct s390_jit_data *jit_data; 2390 bool extra_pass = false; 2391 struct bpf_jit jit; 2392 int pass; 2393 2394 if (!fp->jit_requested) 2395 return fp; 2396 2397 jit_data = fp->aux->jit_data; 2398 if (!jit_data) { 2399 jit_data = kzalloc_obj(*jit_data); 2400 if (!jit_data) 2401 return fp; 2402 fp->aux->jit_data = jit_data; 2403 } 2404 if (jit_data->ctx.addrs) { 2405 jit = jit_data->ctx; 2406 header = jit_data->header; 2407 extra_pass = true; 2408 pass = jit_data->pass + 1; 2409 goto skip_init_ctx; 2410 } 2411 2412 memset(&jit, 0, sizeof(jit)); 2413 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); 2414 if (jit.addrs == NULL) 2415 goto out_err; 2416 /* 2417 * Three initial passes: 2418 * - 1/2: Determine clobbered registers 2419 * - 3: Calculate program size and addrs array 2420 */ 2421 for (pass = 1; pass <= 3; pass++) { 2422 if (bpf_jit_prog(&jit, fp, extra_pass)) 2423 goto out_err; 2424 } 2425 /* 2426 * Final pass: Allocate and generate program 2427 */ 2428 header = bpf_jit_alloc(&jit, fp); 2429 if (!header) 2430 goto out_err; 2431 skip_init_ctx: 2432 if (bpf_jit_prog(&jit, fp, extra_pass)) { 2433 bpf_jit_binary_free(header); 2434 goto out_err; 2435 } 2436 if (bpf_jit_enable > 1) { 2437 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); 2438 print_fn_code(jit.prg_buf, jit.size_prg); 2439 } 2440 if (!fp->is_func || extra_pass) { 2441 if (bpf_jit_binary_lock_ro(header)) { 2442 bpf_jit_binary_free(header); 2443 goto out_err; 2444 } 2445 } else { 2446 jit_data->header = header; 2447 jit_data->ctx = jit; 2448 jit_data->pass = pass; 2449 } 2450 fp->bpf_func = (void *)jit.prg_buf + cfi_get_offset(); 2451 fp->jited = 1; 2452 fp->jited_len = jit.size - cfi_get_offset(); 2453 2454 if (!fp->is_func || extra_pass) { 2455 for (int i = 0; i < fp->len; i++) 2456 jit.addrs[i] -= cfi_get_offset(); 2457 bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); 2458 free_addrs: 2459 kvfree(jit.addrs); 2460 kfree(jit_data); 2461 fp->aux->jit_data = NULL; 2462 } 2463 2464 return fp; 2465 2466 out_err: 2467 if (extra_pass) { 2468 fp->bpf_func = NULL; 2469 fp->jited = 0; 2470 fp->jited_len = 0; 2471 } 2472 goto free_addrs; 2473 } 2474 2475 bool bpf_jit_supports_kfunc_call(void) 2476 { 2477 return true; 2478 } 2479 2480 bool bpf_jit_supports_far_kfunc_call(void) 2481 { 2482 return true; 2483 } 2484 2485 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 2486 enum bpf_text_poke_type new_t, void *old_addr, 2487 void *new_addr) 2488 { 2489 struct bpf_plt expected_plt, current_plt, new_plt, *plt; 2490 struct { 2491 u16 opc; 2492 s32 disp; 2493 } __packed insn; 2494 char *ret; 2495 int err; 2496 2497 /* Verify the branch to be patched. */ 2498 err = copy_from_kernel_nofault(&insn, ip, sizeof(insn)); 2499 if (err < 0) 2500 return err; 2501 if (insn.opc != (0xc004 | (old_addr ? 0xf0 : 0))) 2502 return -EINVAL; 2503 2504 if ((new_t == BPF_MOD_JUMP || old_t == BPF_MOD_JUMP) && 2505 insn.disp == ((char *)new_addr - (char *)ip) >> 1) { 2506 /* 2507 * The branch already points to the destination, 2508 * there is no PLT. 2509 */ 2510 } else { 2511 /* Verify the PLT. */ 2512 plt = ip + (insn.disp << 1); 2513 err = copy_from_kernel_nofault(¤t_plt, plt, 2514 sizeof(current_plt)); 2515 if (err < 0) 2516 return err; 2517 ret = (char *)ip + 6; 2518 bpf_jit_plt(&expected_plt, ret, old_addr); 2519 if (memcmp(¤t_plt, &expected_plt, sizeof(current_plt))) 2520 return -EINVAL; 2521 /* Adjust the call address. */ 2522 bpf_jit_plt(&new_plt, ret, new_addr); 2523 s390_kernel_write(&plt->target, &new_plt.target, 2524 sizeof(void *)); 2525 } 2526 2527 /* Adjust the mask of the branch. */ 2528 insn.opc = 0xc004 | (new_addr ? 0xf0 : 0); 2529 s390_kernel_write((char *)ip + 1, (char *)&insn.opc + 1, 1); 2530 2531 /* Make the new code visible to the other CPUs. */ 2532 text_poke_sync_lock(); 2533 2534 return 0; 2535 } 2536 2537 struct bpf_tramp_jit { 2538 struct bpf_jit common; 2539 int orig_stack_args_off;/* Offset of arguments placed on stack by the 2540 * func_addr's original caller 2541 */ 2542 int stack_size; /* Trampoline stack size */ 2543 int backchain_off; /* Offset of backchain */ 2544 int stack_args_off; /* Offset of stack arguments for calling 2545 * func_addr, has to be at the top 2546 */ 2547 int reg_args_off; /* Offset of register arguments for calling 2548 * func_addr 2549 */ 2550 int ip_off; /* For bpf_get_func_ip(), has to be at 2551 * (ctx - 16) 2552 */ 2553 int func_meta_off; /* For bpf_get_func_arg_cnt()/fsession, has 2554 * to be at (ctx - 8) 2555 */ 2556 int bpf_args_off; /* Offset of BPF_PROG context, which consists 2557 * of BPF arguments followed by return value 2558 */ 2559 int retval_off; /* Offset of return value (see above) */ 2560 int r7_r8_off; /* Offset of saved %r7 and %r8, which are used 2561 * for __bpf_prog_enter() return value and 2562 * func_addr respectively 2563 */ 2564 int run_ctx_off; /* Offset of struct bpf_tramp_run_ctx */ 2565 int tccnt_off; /* Offset of saved tailcall counter */ 2566 int r14_off; /* Offset of saved %r14, has to be at the 2567 * bottom */ 2568 int do_fexit; /* do_fexit: label */ 2569 }; 2570 2571 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) 2572 { 2573 /* llihf %dst_reg,val_hi */ 2574 EMIT6_IMM(0xc00e0000, dst_reg, (val >> 32)); 2575 /* oilf %rdst_reg,val_lo */ 2576 EMIT6_IMM(0xc00d0000, dst_reg, val); 2577 } 2578 2579 static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm) 2580 { 2581 load_imm64(jit, tmp_reg, imm); 2582 /* stg %tmp_reg,stack_off(%r15) */ 2583 EMIT6_DISP_LH(0xe3000000, 0x0024, tmp_reg, REG_0, REG_15, stack_off); 2584 } 2585 2586 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, 2587 const struct btf_func_model *m, 2588 struct bpf_tramp_node *node, bool save_ret) 2589 { 2590 struct bpf_jit *jit = &tjit->common; 2591 int cookie_off = tjit->run_ctx_off + 2592 offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 2593 struct bpf_prog *p = node->link->prog; 2594 int patch; 2595 2596 /* 2597 * run_ctx.cookie = node->cookie; 2598 */ 2599 2600 emit_store_stack_imm64(jit, REG_W0, cookie_off, node->cookie); 2601 2602 /* 2603 * if ((start = __bpf_prog_enter(p, &run_ctx)) == 0) 2604 * goto skip; 2605 */ 2606 2607 /* %r2 = p */ 2608 load_imm64(jit, REG_2, (u64)p); 2609 /* la %r3,run_ctx_off(%r15) */ 2610 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); 2611 /* brasl %r14,__bpf_prog_enter */ 2612 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_enter(p)); 2613 /* ltgr %r7,%r2 */ 2614 EMIT4(0xb9020000, REG_7, REG_2); 2615 /* brcl 8,skip */ 2616 patch = jit->prg; 2617 EMIT6_PCREL_RILC(0xc0040000, 8, 0); 2618 2619 /* 2620 * retval = bpf_func(args, p->insnsi); 2621 */ 2622 2623 /* la %r2,bpf_args_off(%r15) */ 2624 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); 2625 /* %r3 = p->insnsi */ 2626 if (!p->jited) 2627 load_imm64(jit, REG_3, (u64)p->insnsi); 2628 /* brasl %r14,p->bpf_func */ 2629 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func); 2630 /* stg %r2,retval_off(%r15) */ 2631 if (save_ret) { 2632 if (sign_zero_extend(jit, REG_2, m->ret_size, m->ret_flags)) 2633 return -1; 2634 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2635 tjit->retval_off); 2636 } 2637 2638 /* skip: */ 2639 if (jit->prg_buf) 2640 *(u32 *)&jit->prg_buf[patch + 2] = (jit->prg - patch) >> 1; 2641 2642 /* 2643 * __bpf_prog_exit(p, start, &run_ctx); 2644 */ 2645 2646 /* %r2 = p */ 2647 load_imm64(jit, REG_2, (u64)p); 2648 /* lgr %r3,%r7 */ 2649 EMIT4(0xb9040000, REG_3, REG_7); 2650 /* la %r4,run_ctx_off(%r15) */ 2651 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); 2652 /* brasl %r14,__bpf_prog_exit */ 2653 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p)); 2654 2655 return 0; 2656 } 2657 2658 static int invoke_bpf(struct bpf_tramp_jit *tjit, 2659 const struct btf_func_model *m, 2660 struct bpf_tramp_nodes *tn, bool save_ret, 2661 u64 func_meta, int cookie_off) 2662 { 2663 int i, cur_cookie = (tjit->bpf_args_off - cookie_off) / sizeof(u64); 2664 struct bpf_jit *jit = &tjit->common; 2665 2666 for (i = 0; i < tn->nr_nodes; i++) { 2667 if (bpf_prog_calls_session_cookie(tn->nodes[i])) { 2668 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); 2669 2670 emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); 2671 cur_cookie--; 2672 } 2673 if (invoke_bpf_prog(tjit, m, tn->nodes[i], save_ret)) 2674 return -EINVAL; 2675 } 2676 2677 return 0; 2678 } 2679 2680 static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size) 2681 { 2682 int stack_offset = tjit->stack_size; 2683 2684 tjit->stack_size += size; 2685 return stack_offset; 2686 } 2687 2688 /* ABI uses %r2 - %r6 for parameter passing. */ 2689 #define MAX_NR_REG_ARGS 5 2690 2691 /* The "L" field of the "mvc" instruction is 8 bits. */ 2692 #define MAX_MVC_SIZE 256 2693 #define MAX_NR_STACK_ARGS (MAX_MVC_SIZE / sizeof(u64)) 2694 2695 /* -mfentry generates a 6-byte nop on s390x. */ 2696 #define S390X_PATCH_SIZE 6 2697 2698 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 2699 struct bpf_tramp_jit *tjit, 2700 const struct btf_func_model *m, 2701 u32 flags, 2702 struct bpf_tramp_nodes *tnodes, 2703 void *func_addr) 2704 { 2705 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 2706 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 2707 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 2708 int nr_bpf_args, nr_reg_args, nr_stack_args; 2709 int cookie_cnt, cookie_off, fsession_cnt; 2710 struct bpf_jit *jit = &tjit->common; 2711 int arg, bpf_arg_off; 2712 u64 func_meta; 2713 int i, j; 2714 2715 /* Support as many stack arguments as "mvc" instruction can handle. */ 2716 nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS); 2717 nr_stack_args = m->nr_args - nr_reg_args; 2718 if (nr_stack_args > MAX_NR_STACK_ARGS) 2719 return -ENOTSUPP; 2720 2721 /* Return to %r14 in the struct_ops case. */ 2722 if (flags & BPF_TRAMP_F_INDIRECT) { 2723 flags |= BPF_TRAMP_F_SKIP_FRAME; 2724 emit_kcfi(cfi_get_func_hash(func_addr), jit); 2725 } 2726 2727 /* 2728 * Compute how many arguments we need to pass to BPF programs. 2729 * BPF ABI mirrors that of x86_64: arguments that are 16 bytes or 2730 * smaller are packed into 1 or 2 registers; larger arguments are 2731 * passed via pointers. 2732 * In s390x ABI, arguments that are 8 bytes or smaller are packed into 2733 * a register; larger arguments are passed via pointers. 2734 * We need to deal with this difference. 2735 */ 2736 nr_bpf_args = 0; 2737 for (i = 0; i < m->nr_args; i++) { 2738 if (m->arg_size[i] <= 8) 2739 nr_bpf_args += 1; 2740 else if (m->arg_size[i] <= 16) 2741 nr_bpf_args += 2; 2742 else 2743 return -ENOTSUPP; 2744 } 2745 2746 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 2747 fsession_cnt = bpf_fsession_cnt(tnodes); 2748 2749 /* 2750 * Calculate the stack layout. 2751 */ 2752 2753 /* 2754 * Allocate STACK_FRAME_OVERHEAD bytes for the callees. As the s390x 2755 * ABI requires, put our backchain at the end of the allocated memory. 2756 */ 2757 tjit->stack_size = STACK_FRAME_OVERHEAD; 2758 tjit->backchain_off = tjit->stack_size - sizeof(u64); 2759 tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64)); 2760 tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64)); 2761 cookie_off = alloc_stack(tjit, cookie_cnt * sizeof(u64)); 2762 tjit->ip_off = alloc_stack(tjit, sizeof(u64)); 2763 tjit->func_meta_off = alloc_stack(tjit, sizeof(u64)); 2764 tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64)); 2765 tjit->retval_off = alloc_stack(tjit, sizeof(u64)); 2766 tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64)); 2767 tjit->run_ctx_off = alloc_stack(tjit, 2768 sizeof(struct bpf_tramp_run_ctx)); 2769 tjit->tccnt_off = alloc_stack(tjit, sizeof(u64)); 2770 tjit->r14_off = alloc_stack(tjit, sizeof(u64) * 2); 2771 /* 2772 * In accordance with the s390x ABI, the caller has allocated 2773 * STACK_FRAME_OVERHEAD bytes for us. 8 of them contain the caller's 2774 * backchain, and the rest we can use. 2775 */ 2776 tjit->stack_size -= STACK_FRAME_OVERHEAD - sizeof(u64); 2777 tjit->orig_stack_args_off = tjit->stack_size + STACK_FRAME_OVERHEAD; 2778 2779 /* lgr %r1,%r15 */ 2780 EMIT4(0xb9040000, REG_1, REG_15); 2781 /* aghi %r15,-stack_size */ 2782 EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size); 2783 /* stg %r1,backchain_off(%r15) */ 2784 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15, 2785 tjit->backchain_off); 2786 /* mvc tccnt_off(4,%r15),stack_size+tail_call_cnt(%r15) */ 2787 _EMIT6(0xd203f000 | tjit->tccnt_off, 2788 0xf000 | (tjit->stack_size + 2789 offsetof(struct prog_frame, tail_call_cnt))); 2790 /* stmg %r2,%rN,fwd_reg_args_off(%r15) */ 2791 if (nr_reg_args) 2792 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2, 2793 REG_2 + (nr_reg_args - 1), REG_15, 2794 tjit->reg_args_off); 2795 for (i = 0, j = 0; i < m->nr_args; i++) { 2796 if (i < MAX_NR_REG_ARGS) 2797 arg = REG_2 + i; 2798 else 2799 arg = tjit->orig_stack_args_off + 2800 (i - MAX_NR_REG_ARGS) * sizeof(u64); 2801 bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64); 2802 if (m->arg_size[i] <= 8) { 2803 if (i < MAX_NR_REG_ARGS) 2804 /* stg %arg,bpf_arg_off(%r15) */ 2805 EMIT6_DISP_LH(0xe3000000, 0x0024, arg, 2806 REG_0, REG_15, bpf_arg_off); 2807 else 2808 /* mvc bpf_arg_off(8,%r15),arg(%r15) */ 2809 _EMIT6(0xd207f000 | bpf_arg_off, 2810 0xf000 | arg); 2811 j += 1; 2812 } else { 2813 if (i < MAX_NR_REG_ARGS) { 2814 /* mvc bpf_arg_off(16,%r15),0(%arg) */ 2815 _EMIT6(0xd20ff000 | bpf_arg_off, 2816 reg2hex[arg] << 12); 2817 } else { 2818 /* lg %r1,arg(%r15) */ 2819 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_0, 2820 REG_15, arg); 2821 /* mvc bpf_arg_off(16,%r15),0(%r1) */ 2822 _EMIT6(0xd20ff000 | bpf_arg_off, 0x1000); 2823 } 2824 j += 2; 2825 } 2826 } 2827 /* stmg %r7,%r8,r7_r8_off(%r15) */ 2828 EMIT6_DISP_LH(0xeb000000, 0x0024, REG_7, REG_8, REG_15, 2829 tjit->r7_r8_off); 2830 /* stg %r14,r14_off(%r15) */ 2831 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off); 2832 2833 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2834 /* 2835 * The ftrace trampoline puts the return address (which is the 2836 * address of the original function + S390X_PATCH_SIZE) into 2837 * %r0; see ftrace_shared_hotpatch_trampoline_br and 2838 * ftrace_init_nop() for details. 2839 */ 2840 2841 /* lgr %r8,%r0 */ 2842 EMIT4(0xb9040000, REG_8, REG_0); 2843 } 2844 2845 /* 2846 * ip = func_addr; 2847 * arg_cnt = m->nr_args; 2848 */ 2849 2850 if (flags & BPF_TRAMP_F_IP_ARG) 2851 emit_store_stack_imm64(jit, REG_0, tjit->ip_off, (u64)func_addr); 2852 func_meta = nr_bpf_args; 2853 /* lghi %r0,func_meta */ 2854 EMIT4_IMM(0xa7090000, REG_0, func_meta); 2855 /* stg %r0,func_meta_off(%r15) */ 2856 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15, 2857 tjit->func_meta_off); 2858 2859 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2860 /* 2861 * __bpf_tramp_enter(im); 2862 */ 2863 2864 /* %r2 = im */ 2865 load_imm64(jit, REG_2, (u64)im); 2866 /* brasl %r14,__bpf_tramp_enter */ 2867 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter); 2868 } 2869 2870 if (fsession_cnt) { 2871 /* Clear all the session cookies' value. */ 2872 for (i = 0; i < cookie_cnt; i++) 2873 emit_store_stack_imm64(jit, REG_0, cookie_off + 8 * i, 0); 2874 /* Clear the return value to make sure fentry always gets 0. */ 2875 emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); 2876 } 2877 2878 if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, 2879 func_meta, cookie_off)) 2880 return -EINVAL; 2881 2882 if (fmod_ret->nr_nodes) { 2883 /* 2884 * retval = 0; 2885 */ 2886 2887 /* xc retval_off(8,%r15),retval_off(%r15) */ 2888 _EMIT6(0xd707f000 | tjit->retval_off, 2889 0xf000 | tjit->retval_off); 2890 2891 for (i = 0; i < fmod_ret->nr_nodes; i++) { 2892 if (invoke_bpf_prog(tjit, m, fmod_ret->nodes[i], true)) 2893 return -EINVAL; 2894 2895 /* 2896 * if (retval) 2897 * goto do_fexit; 2898 */ 2899 2900 /* ltg %r0,retval_off(%r15) */ 2901 EMIT6_DISP_LH(0xe3000000, 0x0002, REG_0, REG_0, REG_15, 2902 tjit->retval_off); 2903 /* brcl 7,do_fexit */ 2904 EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit); 2905 } 2906 } 2907 2908 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2909 /* 2910 * retval = func_addr(args); 2911 */ 2912 2913 /* lmg %r2,%rN,reg_args_off(%r15) */ 2914 if (nr_reg_args) 2915 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2916 REG_2 + (nr_reg_args - 1), REG_15, 2917 tjit->reg_args_off); 2918 /* mvc stack_args_off(N,%r15),orig_stack_args_off(%r15) */ 2919 if (nr_stack_args) 2920 _EMIT6(0xd200f000 | 2921 (nr_stack_args * sizeof(u64) - 1) << 16 | 2922 tjit->stack_args_off, 2923 0xf000 | tjit->orig_stack_args_off); 2924 /* mvc tail_call_cnt(4,%r15),tccnt_off(%r15) */ 2925 _EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt), 2926 0xf000 | tjit->tccnt_off); 2927 if (flags & BPF_TRAMP_F_ORIG_STACK) { 2928 if (nospec_uses_trampoline()) 2929 /* brasl %r14,__s390_indirect_jump_r8 */ 2930 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2931 __s390_indirect_jump_r8); 2932 else 2933 /* basr %r14,%r8 */ 2934 EMIT2(0x0d00, REG_14, REG_8); 2935 } else { 2936 /* brasl %r14,func_addr+S390X_PATCH_SIZE */ 2937 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, 2938 func_addr + S390X_PATCH_SIZE); 2939 } 2940 /* stg %r2,retval_off(%r15) */ 2941 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, 2942 tjit->retval_off); 2943 /* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */ 2944 _EMIT6(0xd203f000 | tjit->tccnt_off, 2945 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); 2946 2947 im->ip_after_call = jit->prg_buf + jit->prg; 2948 2949 /* 2950 * The following nop will be patched by bpf_tramp_image_put(). 2951 */ 2952 2953 /* brcl 0,im->ip_epilogue */ 2954 EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); 2955 } 2956 2957 /* Set the "is_return" flag for fsession. */ 2958 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 2959 if (fsession_cnt) 2960 emit_store_stack_imm64(jit, REG_W0, tjit->func_meta_off, 2961 func_meta); 2962 2963 /* do_fexit: */ 2964 tjit->do_fexit = jit->prg; 2965 if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off)) 2966 return -EINVAL; 2967 2968 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2969 im->ip_epilogue = jit->prg_buf + jit->prg; 2970 2971 /* 2972 * __bpf_tramp_exit(im); 2973 */ 2974 2975 /* %r2 = im */ 2976 load_imm64(jit, REG_2, (u64)im); 2977 /* brasl %r14,__bpf_tramp_exit */ 2978 EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_exit); 2979 } 2980 2981 /* lmg %r2,%rN,reg_args_off(%r15) */ 2982 if ((flags & BPF_TRAMP_F_RESTORE_REGS) && nr_reg_args) 2983 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_2, 2984 REG_2 + (nr_reg_args - 1), REG_15, 2985 tjit->reg_args_off); 2986 /* lgr %r1,%r8 */ 2987 if (!(flags & BPF_TRAMP_F_SKIP_FRAME) && 2988 (flags & BPF_TRAMP_F_ORIG_STACK)) 2989 EMIT4(0xb9040000, REG_1, REG_8); 2990 /* lmg %r7,%r8,r7_r8_off(%r15) */ 2991 EMIT6_DISP_LH(0xeb000000, 0x0004, REG_7, REG_8, REG_15, 2992 tjit->r7_r8_off); 2993 /* lg %r14,r14_off(%r15) */ 2994 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off); 2995 /* lg %r2,retval_off(%r15) */ 2996 if (flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET)) 2997 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15, 2998 tjit->retval_off); 2999 /* mvc stack_size+tail_call_cnt(4,%r15),tccnt_off(%r15) */ 3000 _EMIT6(0xd203f000 | (tjit->stack_size + 3001 offsetof(struct prog_frame, tail_call_cnt)), 3002 0xf000 | tjit->tccnt_off); 3003 /* aghi %r15,stack_size */ 3004 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); 3005 if (flags & BPF_TRAMP_F_SKIP_FRAME) 3006 EMIT_JUMP_REG(14); 3007 else if (flags & BPF_TRAMP_F_ORIG_STACK) 3008 EMIT_JUMP_REG(1); 3009 else 3010 /* brcl 0xf,func_addr+S390X_PATCH_SIZE */ 3011 EMIT6_PCREL_RILC_PTR(0xc0040000, 0xf, 3012 func_addr + S390X_PATCH_SIZE); 3013 return 0; 3014 } 3015 3016 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 3017 struct bpf_tramp_nodes *tnodes, void *orig_call) 3018 { 3019 struct bpf_tramp_image im; 3020 struct bpf_tramp_jit tjit; 3021 int ret; 3022 3023 memset(&tjit, 0, sizeof(tjit)); 3024 3025 ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, 3026 tnodes, orig_call); 3027 3028 return ret < 0 ? ret : tjit.common.prg; 3029 } 3030 3031 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 3032 void *image_end, const struct btf_func_model *m, 3033 u32 flags, struct bpf_tramp_nodes *tnodes, 3034 void *func_addr) 3035 { 3036 struct bpf_tramp_jit tjit; 3037 int ret; 3038 3039 /* Compute offsets, check whether the code fits. */ 3040 memset(&tjit, 0, sizeof(tjit)); 3041 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 3042 tnodes, func_addr); 3043 3044 if (ret < 0) 3045 return ret; 3046 if (tjit.common.prg > (char *)image_end - (char *)image) 3047 /* 3048 * Use the same error code as for exceeding 3049 * BPF_MAX_TRAMP_LINKS. 3050 */ 3051 return -E2BIG; 3052 3053 tjit.common.prg = 0; 3054 tjit.common.prg_buf = image; 3055 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, 3056 tnodes, func_addr); 3057 3058 return ret < 0 ? ret : tjit.common.prg; 3059 } 3060 3061 bool bpf_jit_supports_subprog_tailcalls(void) 3062 { 3063 return true; 3064 } 3065 3066 bool bpf_jit_supports_arena(void) 3067 { 3068 return true; 3069 } 3070 3071 bool bpf_jit_supports_fsession(void) 3072 { 3073 return true; 3074 } 3075 3076 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 3077 { 3078 if (!in_arena) 3079 return true; 3080 switch (insn->code) { 3081 case BPF_LDX | BPF_MEMSX | BPF_B: 3082 case BPF_LDX | BPF_MEMSX | BPF_H: 3083 case BPF_LDX | BPF_MEMSX | BPF_W: 3084 return false; 3085 } 3086 return true; 3087 } 3088 3089 bool bpf_jit_supports_exceptions(void) 3090 { 3091 /* 3092 * Exceptions require unwinding support, which is always available, 3093 * because the kernel is always built with backchain. 3094 */ 3095 return true; 3096 } 3097 3098 void arch_bpf_stack_walk(bool (*consume_fn)(void *, u64, u64, u64), 3099 void *cookie) 3100 { 3101 unsigned long addr, prev_addr = 0; 3102 struct unwind_state state; 3103 3104 unwind_for_each_frame(&state, NULL, NULL, 0) { 3105 addr = unwind_get_return_address(&state); 3106 if (!addr) 3107 break; 3108 /* 3109 * addr is a return address and state.sp is the value of %r15 3110 * at this address. exception_cb needs %r15 at entry to the 3111 * function containing addr, so take the next state.sp. 3112 * 3113 * There is no bp, and the exception_cb prog does not need one 3114 * to perform a quasi-longjmp. The common code requires a 3115 * non-zero bp, so pass sp there as well. 3116 */ 3117 if (prev_addr && !consume_fn(cookie, prev_addr, state.sp, 3118 state.sp)) 3119 break; 3120 prev_addr = addr; 3121 } 3122 } 3123 3124 bool bpf_jit_supports_timed_may_goto(void) 3125 { 3126 return true; 3127 } 3128 3129 bool bpf_jit_inlines_helper_call(s32 imm) 3130 { 3131 switch (imm) { 3132 case BPF_FUNC_get_smp_processor_id: 3133 case BPF_FUNC_get_current_task: 3134 case BPF_FUNC_get_current_task_btf: 3135 return true; 3136 default: 3137 return false; 3138 } 3139 } 3140